diff --git a/.github/workflows/bot-label-lgtm.yaml b/.github/workflows/bot-label-lgtm.yaml
deleted file mode 100644
index 27f50375cdc..00000000000
--- a/.github/workflows/bot-label-lgtm.yaml
+++ /dev/null
@@ -1,48 +0,0 @@
-# This workflow adds the community approval label ("lgtm") to pull requests. It
-# does *not* indicate maintainer approval. This a way to visually highlight that
-# someone in the world thinks the pull request is ready for further review. This
-# event is triggered by a pull request approval, or simply a comment that
-# contains the text "lgtm".
-# Webhook events: Issue comments, Pull request reviews
-name: Community approval
-on:
- repository_dispatch:
- # From: issue_comment, pull_request_review
- types: [created, edited, submitted]
-
-jobs:
- lgtm-comment:
- # Check the comment. contains() is case-insensitive.
- if: >-
- ${{ github.actor == 'tfdocsbot' &&
- contains(github.event.client_payload.comment.body, 'LGTM') }}
- runs-on: ubuntu-latest
- steps:
- - name: Add label
- env:
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- ISSUE_URL: ${{ github.event.client_payload.comment.issue_url }}
- run: |
- curl -X POST \
- -H "Accept: application/vnd.github.v3+json" \
- -H "Authorization: token $GITHUB_TOKEN" \
- "${ISSUE_URL}/labels" \
- --data '{"labels":["lgtm"]}'
-
- review-approval:
- # Check the pull request review.
- if: >-
- ${{ github.actor == 'tfdocsbot' &&
- contains(github.event.client_payload.review.state, 'approved') }}
- runs-on: ubuntu-latest
- steps:
- - name: Add label
- env:
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- ISSUE_URL: ${{ github.event.client_payload.pull_request.issue_url }}
- run: |
- curl -X POST \
- -H "Accept: application/vnd.github.v3+json" \
- -H "Authorization: token $GITHUB_TOKEN" \
- "${ISSUE_URL}/labels" \
- --data '{"labels":["lgtm"]}'
diff --git a/.github/workflows/bot-nightly.yaml b/.github/workflows/bot-nightly.yaml
deleted file mode 100644
index a0595c74a0b..00000000000
--- a/.github/workflows/bot-nightly.yaml
+++ /dev/null
@@ -1,65 +0,0 @@
-# Nightly jobs run by a bot collaborator.
-name: Nightly jobs
-on:
- repository_dispatch:
- types: [nightly]
-
-jobs:
- snapshot-source:
- name: Update Keras guides
- if : ${{ github.actor == 'tfdocsbot' }}
- runs-on: ubuntu-latest
- steps:
- - uses: actions/checkout@v2
- with:
- repository: keras-team/keras-io
- path: keras-io
- - uses: actions/checkout@v2
- with:
- # tensorflow/docs branch to save generated notebooks.
- ref: snapshot-keras
- path: docs
- - name: Set up repo
- run: |
- # Set commit author.
- git config --global user.name "$GITHUB_ACTOR"
- git config --global user.email "$GITHUB_ACTOR@users.noreply.github.com"
- - name: Set up Python
- uses: actions/setup-python@v2
- with:
- python-version: '3.8'
- - name: Install requirements
- run: |
- python3 -m pip install -U pip
- python3 -m pip install -U -r keras-io/requirements.txt
- python3 -m pip install -U git+https://github.com/tensorflow/docs
- - name: Generate Keras notebooks
- run: |
- # Autogen requires formated code
- echo "[${GITHUB_WORKFLOW}] Format Python files ..."
- python -m black keras-io/guides/
- mkdir -p keras-io/tf # Make sure output dir exists.
- cd keras-io/scripts/
- echo "[${GITHUB_WORKFLOW}] Generate Keras guides ..."
- python3 autogen.py generate_tf_guides
- echo "[${GITHUB_WORKFLOW}] Format notebooks ..."
- python3 -m tensorflow_docs.tools.nbfmt ../tf/
- - name: Sync docs repo
- env:
- KERAS_GUIDES_DIR: site/en/guide/keras/
- run: |
- rsync --archive --del --checksum ./keras-io/tf/ "./docs/${KERAS_GUIDES_DIR}"
- cd docs
- if [[ -z $(git status -s | grep "$KERAS_GUIDES_DIR") ]]; then
- echo "[${GITHUB_WORKFLOW}] No Keras guides updated, exiting."
- exit 0
- fi
- # Match timestamp format to other snapshot messages.
- fmt_rfc7231="%a, %d %b %Y %H:%M:%S %Z"
- TIMESTAMP_STR=$(TZ=GMT date +"$fmt_rfc7231")
-
- git add "./${KERAS_GUIDES_DIR}"
- git commit -m "Keras guides snapshot: ${TIMESTAMP_STR}"
- # Push to current branch.
- echo "[${GITHUB_WORKFLOW}] Push changes to repo ..."
- git push origin
diff --git a/.github/workflows/bot-pr-fix.yaml b/.github/workflows/bot-pr-fix.yaml
deleted file mode 100644
index a8ead3aa7ae..00000000000
--- a/.github/workflows/bot-pr-fix.yaml
+++ /dev/null
@@ -1,48 +0,0 @@
-# Automatically add commits to fix pull requests. This workflow must initiate
-# from an authenticated bot repo collaborator. Check for opt-out label.
-# Webhook events: Pull requests
-name: Auto-fix pull request
-on:
- repository_dispatch:
- types: [opened, synchronize]
-
-jobs:
- nbfmt:
- # Check for opt-out label.
- if: >-
- ${{ github.actor == 'tfdocsbot' &&
- !contains(github.event.client_payload.pull_request.labels.*.name, 'nbfmt-disable') }}
- runs-on: ubuntu-latest
- steps:
- - name: Set up Python
- uses: actions/setup-python@v2
- - name: Install tensorflow-docs
- run: python3 -m pip install -U git+https://github.com/tensorflow/docs
- - name: Fetch pull request branch
- uses: actions/checkout@v2
- with:
- # Head repo is the user's fork. Ref is the branch name.
- repository: ${{ github.event.client_payload.pull_request.head.repo.full_name }}
- ref: ${{ github.event.client_payload.pull_request.head.ref }}
- - name: Fetch base master branch
- run: git fetch -u "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY" master:master
- - name: Format notebooks
- run: |
- # Only want notebooks modified in this pull request.
- readarray -t changed_files < <(git diff --name-only master | grep '\.ipynb$' || true)
- if [[ ${#changed_files[@]} == 0 ]]; then
- echo "No notebooks modified in this pull request."
- exit 0
- fi
- python3 -m tensorflow_docs.tools.nbfmt "${changed_files[@]}"
-
- if [[ -z $(git ls-files --modified) ]]; then
- echo "Notebooks already formatted."
- exit 0
- fi
- # Set author and commit.
- git config --global user.name "$GITHUB_ACTOR"
- git config --global user.email "$GITHUB_ACTOR@users.noreply.github.com"
- git commit -am "nbfmt"
- # Push to the pull request branch submitted by head.
- git push
diff --git a/.github/workflows/bot-pr-new.yaml b/.github/workflows/bot-pr-new.yaml
index 7f2c6164832..13724cc14f0 100644
--- a/.github/workflows/bot-pr-new.yaml
+++ b/.github/workflows/bot-pr-new.yaml
@@ -6,8 +6,15 @@ on:
repository_dispatch:
types: [opened, reopened]
+permissions:
+ contents: read # to fetch code (actions/checkout)
+
jobs:
comment-welcome:
+ permissions:
+ contents: read # to fetch code (actions/checkout)
+ pull-requests: write # to comment on pull-request
+
if: ${{ github.actor == 'tfdocsbot' }}
runs-on: ubuntu-latest
steps:
@@ -15,7 +22,7 @@ jobs:
uses: actions/checkout@v2
with:
repository: ${{ github.event.client_payload.pull_request.head.repo.full_name }}
- ref: ${{ github.event.client_payload.pull_request.head.ref }}
+ ref: ${{ github.event.client_payload.pull_request.head.sha }}
- name: Fetch base master branch
run: git fetch -u "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY" master:master
- name: Create message
diff --git a/.github/workflows/stale.yaml b/.github/workflows/stale.yaml
new file mode 100644
index 00000000000..0ca76b0677e
--- /dev/null
+++ b/.github/workflows/stale.yaml
@@ -0,0 +1,49 @@
+# This workflow warns and then closes issues and PRs that have had no activity for a specified amount of time.
+#
+# You can adjust the behavior by modifying this file.
+# For more information, see:
+# https://github.com/actions/stale
+name: Mark stale issues and pull requests
+
+on:
+ schedule:
+ # Scheduled to run at 1.30 UTC everyday
+ - cron: '30 1 * * *'
+ workflow_dispatch:
+
+jobs:
+ stale:
+
+ runs-on: ubuntu-latest
+ permissions:
+ issues: write
+ pull-requests: write
+
+ steps:
+ - uses: actions/stale@v9
+ with:
+ repo-token: ${{ secrets.GITHUB_TOKEN }}
+ days-before-issue-stale: 14
+ days-before-issue-close: 14
+ stale-issue-label: "status:stale"
+ close-issue-reason: not_planned
+ any-of-labels: "awaiting-contributor-response,cla:no"
+ stale-issue-message: >
+ Marking this issue as stale since it has been open for 14 days with no activity.
+ This issue will be closed if no further activity occurs.
+ close-issue-message: >
+ This issue was closed because it has been inactive for 28 days.
+ Please post a new issue if you need further assistance. Thanks!
+ days-before-pr-stale: 14
+ days-before-pr-close: 14
+ stale-pr-label: "status:stale"
+ stale-pr-message: >
+ Marking this pull request as stale since it has been open for 14 days with no activity.
+ This PR will be closed if no further activity occurs.
+ close-pr-message: >
+ This pull request was closed because it has been inactive for 28 days.
+ Please open a new pull request if you need further assistance. Thanks!
+ # Label that can be assigned to issues to exclude them from being marked as stale
+ exempt-issue-labels: 'override-stale'
+ # Label that can be assigned to PRs to exclude them from being marked as stale
+ exempt-pr-labels: "override-stale"
diff --git a/CODEOWNERS b/CODEOWNERS
index 94c3b5d8d2e..d4d2932d8bc 100644
--- a/CODEOWNERS
+++ b/CODEOWNERS
@@ -2,10 +2,13 @@
# Last matching pattern takes precedence.
# Default owners for everything in repo.
-* @MarkDaoust @8bitmp3
+* @tensorflow/docs-team
# Install
/site/en/install/ @haifeng-jin @MarkDaoust @8bitmp3
# Community
/site/en/community/ @ewilderj @theadactyl @joanafilipa
+
+# Hub
+/site/en/hub @gustheman
\ No newline at end of file
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 1559b721f51..6f301eab782 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -6,9 +6,7 @@ This guide shows how to make contributions to [tensorflow.org](https://www.tenso
See the
[TensorFlow docs contributor guide](https://www.tensorflow.org/community/contribute/docs)
-for guidance. For questions, the
-[docs@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs)
-mailing list is available.
+for guidance. For questions, check out [TensorFlow Forum](https://discuss.tensorflow.org/).
Questions about TensorFlow usage are better addressed on
[Stack Overflow](https://stackoverflow.com/questions/tagged/tensorflow) or the
diff --git a/LICENSE b/LICENSE
index 4862420c023..08026f1ac8e 100644
--- a/LICENSE
+++ b/LICENSE
@@ -201,3 +201,28 @@ Copyright 2018 The TensorFlow Authors. All rights reserved.
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
+
+
+---------------------------
+
+Where indicated, some files are also distributed under the MIT License:
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining a
+copy of this software and associated documentation files (the "Software"),
+to deal in the Software without restriction, including without limitation
+the rights to use, copy, modify, merge, publish, distribute, sublicense,
+and/or sell copies of the Software, and to permit persons to whom the
+Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
\ No newline at end of file
diff --git a/README.md b/README.md
index 7b94ce5f90f..66b6d3fb065 100644
--- a/README.md
+++ b/README.md
@@ -16,7 +16,7 @@ To file a docs issue, use the issue tracker in the
[tensorflow/tensorflow](https://github.com/tensorflow/tensorflow/issues/new?template=20-documentation-issue.md) repo.
And join the TensorFlow documentation contributors on the
-[docs@tensorflow.org mailing list](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs).
+[TensorFlow Forum](https://discuss.tensorflow.org/).
## Community translations
diff --git a/setup.py b/setup.py
index 3a726be637b..404479668b3 100644
--- a/setup.py
+++ b/setup.py
@@ -14,6 +14,7 @@
# ==============================================================================
"""tensorflow_docs is a package for generating python api-reference docs."""
+import datetime
import subprocess
import sys
@@ -21,7 +22,22 @@
from setuptools import setup
project_name = 'tensorflow-docs'
-version = '0.0.0.dev0'
+
+
+def get_version() -> str:
+ ts = int(
+ subprocess.check_output(['git', 'log', '-1', '--format=%ct', 'tools'])
+ .decode('utf-8')
+ .strip()
+ )
+ dt = datetime.datetime.utcfromtimestamp(ts)
+ sec = 60 * 60 * dt.hour + 60 * dt.minute + dt.second
+
+ # calver.org
+ return f'{dt.year}.{dt.month}.{dt.day}.{sec}'
+
+
+version = get_version()
DOCLINES = __doc__.split('\n')
@@ -29,21 +45,11 @@
'astor',
'absl-py',
'jinja2',
- # TODO(b/182876485): Protobuf 3.20 results in linker errors on Windows
- # Protobuf 4.0 is binary incompatible with what C++ TF uses.
- # We need ~1 quarter to update properly.
- # See also: https://github.com/tensorflow/tensorflow/issues/53234
- # See also: https://github.com/protocolbuffers/protobuf/issues/9954
- # See also: https://github.com/tensorflow/tensorflow/issues/56077
- # This is a temporary patch for now, to patch previous TF releases.
- 'protobuf >= 3.12.0, < 3.20',
+ 'nbformat',
+ 'protobuf>=3.12',
'pyyaml',
]
-# Dataclasses is in-built from py >=3.7. This version is a backport for py 3.6.
-if (sys.version_info.major, sys.version_info.minor) == (3, 6):
- REQUIRED_PKGS.append('dataclasses')
-
VIS_REQUIRE = [
'numpy',
'PILLOW',
@@ -53,6 +59,7 @@
# https://setuptools.readthedocs.io/en/latest/setuptools.html#new-and-changed-setup-keywords
setup(
name=project_name,
+ python_requires='>=3.9',
version=version,
description=DOCLINES[0],
long_description='\n'.join(DOCLINES[2:]),
diff --git a/site/en/README.md b/site/en/README.md
index 21bdd58fb9d..28dc0cce7d4 100644
--- a/site/en/README.md
+++ b/site/en/README.md
@@ -39,7 +39,7 @@ tensorflow.org project | GitHub docs location
[/ranking](https://www.tensorflow.org/ranking) | https://github.com/tensorflow/ranking/tree/master/docs
[/recommenders](https://www.tensorflow.org/recommenders) | https://github.com/tensorflow/recommenders/tree/main/docs
[/responsible_ai/fairness_indicators](https://www.tensorflow.org/responsible_ai/fairness_indicators/guide) | https://github.com/tensorflow/fairness-indicators/tree/master/g3doc
-[/responsible_ai/model_card_toolkit](https://www.tensorflow.org/responsible_ai/model_card_toolkit/guide) | https://github.com/tensorflow/model-card-toolkit/tree/master/model_card_toolkit/documentation
+[/responsible_ai/model_card_toolkit](https://www.tensorflow.org/responsible_ai/model_card_toolkit/guide) | https://github.com/tensorflow/model-card-toolkit/tree/main/model_card_toolkit/documentation
[/responsible_ai/model_remediation](https://www.tensorflow.org/responsible_ai/model_remediation) | https://github.com/tensorflow/model-remediation/tree/master/docs
[/responsible_ai/privacy](https://www.tensorflow.org/responsible_ai/privacy/guide) | https://github.com/tensorflow/privacy/tree/master/g3doc
[/tensorboard](https://www.tensorflow.org/tensorboard) | https://github.com/tensorflow/tensorboard/tree/master/docs
diff --git a/site/en/community/contribute/code.md b/site/en/community/contribute/code.md
index 790e1c72e71..2f71f12d7fe 100644
--- a/site/en/community/contribute/code.md
+++ b/site/en/community/contribute/code.md
@@ -2,7 +2,7 @@
Whether you are adding a loss function, improving test coverage, or writing an
RFC for a major design change, this portion of the contributor guide will help
-you get started. Thank you for work and interest in improving TensorFlow.
+you get started. Thank you for your work and interest in improving TensorFlow.
## Before you get started
@@ -34,7 +34,7 @@ workflow, and for the core devs to become acquainted with the contributor.
If you are interested in recruiting a team to help tackle a large-scale problem
or a new feature, please email the
-[developers@ group](https://groups.google.com/a/tensorflow.org/forum/#!forum/developers)
+[developers@ group](https://groups.google.com/a/tensorflow.org/g/developers)
and review our current list of RFCs.
## Code review
diff --git a/site/en/community/contribute/docs.md b/site/en/community/contribute/docs.md
index 29b2b5c9550..34b1619ca5d 100644
--- a/site/en/community/contribute/docs.md
+++ b/site/en/community/contribute/docs.md
@@ -32,7 +32,7 @@ To participate in the TensorFlow docs community:
For details, use the [TensorFlow API docs contributor guide](docs_ref.md). This
shows you how to find the
-[source file](https://www.tensorflow.org/code/tensorflow/python/)
+[source file](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/)
and edit the symbol's
docstring.
Many API reference pages on tensorflow.org include a link to the source file
@@ -53,9 +53,9 @@ main
tensorflow/tensorflow
repo. The reference documentation is generated from code comments
and docstrings in the source code for
-Python,
-C++, and
-Java.
+Python,
+C++, and
+Java.
Previous versions of the TensorFlow documentation are available as
[rX.x branches](https://github.com/tensorflow/docs/branches) in the TensorFlow
diff --git a/site/en/community/contribute/docs_ref.md b/site/en/community/contribute/docs_ref.md
index fbf207a47f1..41fce4dde40 100644
--- a/site/en/community/contribute/docs_ref.md
+++ b/site/en/community/contribute/docs_ref.md
@@ -8,7 +8,7 @@ TensorFlow uses [DocTest](https://docs.python.org/3/library/doctest.html) to
test code snippets in Python docstrings. The snippet must be executable Python
code. To enable testing, prepend the line with `>>>` (three left-angle
brackets). For example, here's a excerpt from the `tf.concat` function in the
-[array_ops.py](https://www.tensorflow.org/code/tensorflow/python/ops/array_ops.py)
+[array_ops.py](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/ops/array_ops.py)
source file:
```
@@ -178,7 +178,7 @@ There are two ways to test the code in the docstring locally:
* If you are only changing the docstring of a class/function/method, then you
can test it by passing that file's path to
- [tf_doctest.py](https://www.tensorflow.org/code/tensorflow/tools/docs/tf_doctest.py).
+ [tf_doctest.py](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/tools/docs/tf_doctest.py).
For example:
diff --git a/site/en/community/contribute/docs_style.md b/site/en/community/contribute/docs_style.md
index eba78afa896..10f18e52699 100644
--- a/site/en/community/contribute/docs_style.md
+++ b/site/en/community/contribute/docs_style.md
@@ -63,10 +63,10 @@ repository like this:
* \[Basics\]\(../../guide/basics.ipynb\) produces
[Basics](../../guide/basics.ipynb).
-This is the prefered approach because this way the links on
+This is the preferred approach because this way the links on
[tensorflow.org](https://www.tensorflow.org),
-[GitHub](https://github.com/tensorflow/docs){:.external} and
-[Colab](https://github.com/tensorflow/docs/tree/master/site/en/guide/bazics.ipynb){:.external}
+[GitHub](https://github.com/tensorflow/docs) and
+[Colab](https://github.com/tensorflow/docs/tree/master/site/en/guide/bazics.ipynb)
all work. Also, the reader stays in the same site when they click a link.
Note: You should include the file extension—such as `.ipynb` or `.md`—for
@@ -83,10 +83,10 @@ To link to source code, use a link starting with
by the file name starting at the GitHub root.
When linking off of [tensorflow.org](https://www.tensorflow.org), include a
-`{:.external}` on the Markdown link so that the "external link" symbol is shown.
+`` on the Markdown link so that the "external link" symbol is shown.
-* `[GitHub](https://github.com/tensorflow/docs){:.external}` produces
- [GitHub](https://github.com/tensorflow/docs){:.external}
+* `[GitHub](https://github.com/tensorflow/docs)` produces
+ [GitHub](https://github.com/tensorflow/docs)
Do not include URI query parameters in the link:
diff --git a/site/en/community/mailing-lists.md b/site/en/community/mailing-lists.md
index 54c5d57ab91..35bfb218ba1 100644
--- a/site/en/community/mailing-lists.md
+++ b/site/en/community/mailing-lists.md
@@ -5,7 +5,7 @@ note that if you're looking for help using TensorFlow,
[TensorFlow Forum](https://discuss.tensorflow.org/),
[Stack Overflow](https://stackoverflow.com/questions/tagged/tensorflow), and
[GitHub issues](https://github.com/tensorflow/tensorflow/issues) are the best
-initial places to look.
+initial places to look. To receive a roundup of updates from the TensorFlow team each quarter, subscribe to the [TensorFlow newsletter](https://services.google.com/fb/forms/tensorflow/).
## General TensorFlow lists and forums
diff --git a/site/en/community/sig_playbook.md b/site/en/community/sig_playbook.md
index 75e277d3d96..6ec7a554a5b 100644
--- a/site/en/community/sig_playbook.md
+++ b/site/en/community/sig_playbook.md
@@ -55,7 +55,7 @@ must demonstrate:
application area)
* Two or more contributors willing to act as group leads, existence of other
contributors, and evidence of demand for the group
-* Resources it will initially require (usually, mailing list and regular VC
+* Resources it will initially require (usually, mailing list and regular video conference
call.)
Approval for the group will be given by a decision of the TF Community Team,
diff --git a/site/en/guide/_index.yaml b/site/en/guide/_index.yaml
index f9d873dbf2a..e39dd37ead5 100644
--- a/site/en/guide/_index.yaml
+++ b/site/en/guide/_index.yaml
@@ -95,10 +95,11 @@ landing_page:
items:
- list:
- description: >
-
+ A TFX serving system for ML models, designed for high-performance in
+ production environments.
+ path: /tfx/guide/serving
icon:
icon_name: chevron_right
foreground: theme
@@ -147,7 +148,24 @@ landing_page:
icon_name: chevron_right
foreground: theme
background: grey
+ - description: >
+
+ The TensorFlow Model Optimization Toolkit is a suite of tools for
+ optimizing ML models for deployment and execution.
+ path: /model_optimization
icon:
icon_name: chevron_right
foreground: theme
@@ -192,14 +210,6 @@ landing_page:
icon_name: chevron_right
foreground: theme
background: grey
- - description: >
-
Dataset, streaming, and file system extensions, maintained by SIG IO.
diff --git a/site/en/guide/_toc.yaml b/site/en/guide/_toc.yaml
index 1f98745faee..92e5d6a80c3 100644
--- a/site/en/guide/_toc.yaml
+++ b/site/en/guide/_toc.yaml
@@ -19,8 +19,28 @@ toc:
path: /guide/basic_training_loops
- heading: "Keras"
+- title: "Overview"
+ path: /guide/keras
- include: /guide/keras/_toc.yaml
+- heading: "Build with Core"
+ status: new
+- title: "Overview"
+ path: /guide/core/index
+- title: "Quickstart for Core"
+ path: /guide/core/quickstart_core
+- title: "Logistic regression"
+ path: /guide/core/logistic_regression_core
+- title: "Multilayer perceptrons"
+ path: /guide/core/mlp_core
+- title: "Matrix approximation"
+ path: /guide/core/matrix_core
+- title: "Custom optimizers"
+ path: /guide/core/optimizers_core
+- title: "DTensor with Core APIs"
+ path: /guide/core/distribution
+ status: experimental
+
- heading: "TensorFlow in depth"
- title: "Tensor slicing"
path: /guide/tensor_slicing
@@ -35,6 +55,9 @@ toc:
- title: "NumPy API"
status: experimental
path: /guide/tf_numpy
+- title: "NumPy API Type Promotion"
+ status: nightly
+ path: /guide/tf_numpy_type_promotion
- title: "DTensor concepts"
path: /guide/dtensor_overview
status: experimental
@@ -56,11 +79,14 @@ toc:
- title: "Analyze pipeline performance"
path: /guide/data_performance_analysis
-- heading: "Save a model"
+- heading: "Import and export"
- title: "Checkpoint"
path: /guide/checkpoint
- title: "SavedModel"
path: /guide/saved_model
+- title: "Import a JAX model using JAX2TF"
+ status: new
+ path: /guide/jax2tf
- heading: "Accelerators"
- title: "Distributed training"
@@ -86,10 +112,12 @@ toc:
status: new
- title: "Overview"
path: /tfmodels
-- title: "Example: Image classification"
- path: /tfmodels/vision/image_classification
- title: "Training with Orbit"
path: /tfmodels/orbit
+- title: "TFModels - NLP"
+ path: /tfmodels/nlp
+ status: external
+- include: /tfmodels/vision/_toc.yaml
- heading: "Estimators"
status: deprecated
diff --git a/site/en/guide/basics.ipynb b/site/en/guide/basics.ipynb
index 7e8a556e0a7..5457f162a0e 100644
--- a/site/en/guide/basics.ipynb
+++ b/site/en/guide/basics.ipynb
@@ -698,7 +698,6 @@
" self.w_q = tf.Variable(rand_init[0])\n",
" self.w_l = tf.Variable(rand_init[1])\n",
" self.b = tf.Variable(rand_init[2])\n",
- " self.vars = [self.w_q, self.w_l, self.b]\n",
" \n",
" @tf.function\n",
" def __call__(self, x):\n",
@@ -762,9 +761,9 @@
"source": [
"Now, define a loss for your model:\n",
"\n",
- "Given that this model is intended to predict continuous values, the mean squared error (MSE) is a good choice for the loss function. The MSE is defined as the mean of the squared differences between the predicted values and the ground truth. \n",
+ "Given that this model is intended to predict continuous values, the mean squared error (MSE) is a good choice for the loss function. Given a vector of predictions, $\\hat{y}$, and a vector of true targets, $y$, the MSE is defined as the mean of the squared differences between the predicted values and the ground truth.\n",
"\n",
- "$MSE = \\frac{1}{n}\\sum_{i=1}^{n}({y_{pred\\_i}}-y_i)^2$"
+ "$MSE = \\frac{1}{m}\\sum_{i=1}^{m}(\\hat{y}_i -y_i)^2$"
]
},
{
@@ -785,7 +784,7 @@
"id": "7EWyDu3zot2w"
},
"source": [
- "Write a basic training loop for the model. The loop will make use of the MSE loss function and its gradients with respect to the input in order to iteratively update the model's parameters. Using mini-batches for training provides both memory efficienciy and faster convergence. The `tf.data.Dataset` API has useful functions for batching and shuffling."
+ "Write a basic training loop for the model. The loop will make use of the MSE loss function and its gradients with respect to the input in order to iteratively update the model's parameters. Using mini-batches for training provides both memory efficiency and faster convergence. The `tf.data.Dataset` API has useful functions for batching and shuffling."
]
},
{
@@ -820,8 +819,8 @@
" with tf.GradientTape() as tape:\n",
" batch_loss = mse_loss(quad_model(x_batch), y_batch)\n",
" # Update parameters with respect to the gradient calculations\n",
- " grads = tape.gradient(batch_loss, quad_model.vars)\n",
- " for g,v in zip(grads, quad_model.vars):\n",
+ " grads = tape.gradient(batch_loss, quad_model.variables)\n",
+ " for g,v in zip(grads, quad_model.variables):\n",
" v.assign_sub(learning_rate*g)\n",
" # Keep track of model loss per epoch\n",
" loss = mse_loss(quad_model(x), y)\n",
@@ -905,7 +904,7 @@
" batch_size=32,\n",
" verbose=0)\n",
"\n",
- "new_model.save('./my_new_model')"
+ "new_model.save('./my_new_model.keras')"
]
},
{
@@ -957,7 +956,6 @@
"colab": {
"collapsed_sections": [],
"name": "basics.ipynb",
- "provenance": [],
"toc_visible": true
},
"kernelspec": {
diff --git a/site/en/guide/core/distribution.ipynb b/site/en/guide/core/distribution.ipynb
new file mode 100644
index 00000000000..c7f13b2f4db
--- /dev/null
+++ b/site/en/guide/core/distribution.ipynb
@@ -0,0 +1,700 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "FhGuhbZ6M5tl"
+ },
+ "source": [
+ "##### Copyright 2022 The TensorFlow Authors."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "cellView": "form",
+ "id": "AwOEIRJC6Une"
+ },
+ "outputs": [],
+ "source": [
+ "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+ "# you may not use this file except in compliance with the License.\n",
+ "# You may obtain a copy of the License at\n",
+ "#\n",
+ "# https://www.apache.org/licenses/LICENSE-2.0\n",
+ "#\n",
+ "# Unless required by applicable law or agreed to in writing, software\n",
+ "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+ "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+ "# See the License for the specific language governing permissions and\n",
+ "# limitations under the License."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "EIdT9iu_Z4Rb"
+ },
+ "source": [
+ "# Distributed training with Core APIs and DTensor"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "bBIlTPscrIT9"
+ },
+ "source": [
+ "
"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "SjAxxRpBzVYg"
+ },
+ "source": [
+ "## Introduction\n",
+ "\n",
+ "This notebook uses the [TensorFlow Core low-level APIs](https://www.tensorflow.org/guide/core) and [DTensor](https://www.tensorflow.org/guide/dtensor_overview) to demonstrate a data parallel distributed training example. Visit the [Core APIs overview](https://www.tensorflow.org/guide/core) to learn more about TensorFlow Core and its intended use cases. Refer to the [DTensor Overview](https://www.tensorflow.org/guide/dtensor_overview) guide and [Distributed Training with DTensors](https://www.tensorflow.org/tutorials/distribute/dtensor_ml_tutorial) tutorial to learn more about DTensor.\n",
+ "\n",
+ "This example uses the same model and optimizer shown in the [multilayer perceptrons](https://www.tensorflow.org/guide/core/mlp_core) tutorial. See this tutorial first to get comfortable with writing an end-to-end machine learning workflow with the Core APIs.\n",
+ "\n",
+ "Note: DTensor is still an experimental TensorFlow API which means that its features are available for testing, and it is intended for use in test environments only."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "d_OFkG0dyWCp"
+ },
+ "source": [
+ "## Overview of data parallel training with DTensor\n",
+ "\n",
+ "Before building an MLP that supports distribution, take a moment to explore the fundamentals of DTensor for data parallel training.\n",
+ "\n",
+ "DTensor allows you to run distributed training across devices to improve efficiency, reliability and scalability. DTensor distributes the program and tensors according to the sharding directives through a procedure called Single program, multiple data (SPMD) expansion. A variable of a `DTensor` aware layer is created as `dtensor.DVariable`, and the constructors of `DTensor` aware layer objects take additional `Layout` inputs in addition to the usual layer parameters.\n",
+ "\n",
+ "The main ideas for data parallel training are as follows:\n",
+ " - Model variables are replicated on N devices each.\n",
+ " - A global batch is split into N per-replica batches.\n",
+ " - Each per-replica batch is trained on the replica device.\n",
+ " - The gradient is reduced before weight up data is collectively performed on all replicas.\n",
+ " - Data parallel training provides nearly linear speed with respect to the number of devices"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "nchsZfwEVtVs"
+ },
+ "source": [
+ "## Setup\n",
+ "\n",
+ "DTensor is part of TensorFlow 2.9.0 release."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "latuqlI_Yvoo"
+ },
+ "outputs": [],
+ "source": [
+ "#!pip install --quiet --upgrade --pre tensorflow"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "1rRo8oNqZ-Rj"
+ },
+ "outputs": [],
+ "source": [
+ "import matplotlib\n",
+ "from matplotlib import pyplot as plt\n",
+ "# Preset Matplotlib figure sizes.\n",
+ "matplotlib.rcParams['figure.figsize'] = [9, 6]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "9xQKvCJ85kCQ"
+ },
+ "outputs": [],
+ "source": [
+ "import tensorflow as tf\n",
+ "import tensorflow_datasets as tfds\n",
+ "from tensorflow.experimental import dtensor\n",
+ "print(tf.__version__)\n",
+ "# Set random seed for reproducible results \n",
+ "tf.random.set_seed(22)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "vDH9-sy4sfPf"
+ },
+ "source": [
+ "Configure 8 virtual CPUs for this experiment. DTensor can also be used with GPU or TPU devices. Given that this notebook uses virtual devices, the speedup gained from distributed training is not noticeable. "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "H2iM-6J4s2D6"
+ },
+ "outputs": [],
+ "source": [
+ "def configure_virtual_cpus(ncpu):\n",
+ " phy_devices = tf.config.list_physical_devices('CPU')\n",
+ " tf.config.set_logical_device_configuration(phy_devices[0], [\n",
+ " tf.config.LogicalDeviceConfiguration(),\n",
+ " ] * ncpu)\n",
+ "\n",
+ "configure_virtual_cpus(8)\n",
+ "\n",
+ "DEVICES = [f'CPU:{i}' for i in range(8)]\n",
+ "devices = tf.config.list_logical_devices('CPU')\n",
+ "device_names = [d.name for d in devices]\n",
+ "device_names"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "F_72b0LCNbjx"
+ },
+ "source": [
+ "## The MNIST Dataset\n",
+ "\n",
+ "The dataset is available from [TensorFlow Datasets](https://www.tensorflow.org/datasets/catalog/mnist). Split the data into training and testing sets. Only use 5000 examples for training and testing to save time."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "8h4fV_JCfPIX"
+ },
+ "outputs": [],
+ "source": [
+ "train_data, test_data = tfds.load(\"mnist\", split=['train[:5000]', 'test[:5000]'], batch_size=128, as_supervised=True)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "twkJ35YB6tSi"
+ },
+ "source": [
+ "### Preprocessing the data\n",
+ "\n",
+ "Preprocess the data by reshaping it to be 2-dimensional and by rescaling it to fit into the unit interval, [0,1]."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "6Cmjhg0xCqbz"
+ },
+ "outputs": [],
+ "source": [
+ "def preprocess(x, y):\n",
+ " # Reshaping the data\n",
+ " x = tf.reshape(x, shape=[-1, 784])\n",
+ " # Rescaling the data\n",
+ " x = x/255\n",
+ " return x, y\n",
+ "\n",
+ "train_data, test_data = train_data.map(preprocess), test_data.map(preprocess)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "6o3CrycBXA2s"
+ },
+ "source": [
+ "## Build the MLP \n",
+ "\n",
+ "Build an MLP model with DTensor aware layers."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "OHW6Yvg2yS6H"
+ },
+ "source": [
+ "### The dense layer\n",
+ "\n",
+ "Start by creating a dense layer module that supports DTensor. The `dtensor.call_with_layout` function can be used to call a function that takes in a DTensor input and produces a DTensor output. This is useful for initializing a DTensor variable, `dtensor.DVariable`, with a TensorFlow supported function."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "IM0yJos25FG5"
+ },
+ "outputs": [],
+ "source": [
+ "class DenseLayer(tf.Module):\n",
+ "\n",
+ " def __init__(self, in_dim, out_dim, weight_layout, activation=tf.identity):\n",
+ " super().__init__()\n",
+ " # Initialize dimensions and the activation function\n",
+ " self.in_dim, self.out_dim = in_dim, out_dim\n",
+ " self.activation = activation\n",
+ "\n",
+ " # Initialize the DTensor weights using the Xavier scheme\n",
+ " uniform_initializer = tf.function(tf.random.stateless_uniform)\n",
+ " xavier_lim = tf.sqrt(6.)/tf.sqrt(tf.cast(self.in_dim + self.out_dim, tf.float32))\n",
+ " self.w = dtensor.DVariable(\n",
+ " dtensor.call_with_layout(\n",
+ " uniform_initializer, weight_layout,\n",
+ " shape=(self.in_dim, self.out_dim), seed=(22, 23),\n",
+ " minval=-xavier_lim, maxval=xavier_lim))\n",
+ " \n",
+ " # Initialize the bias with the zeros\n",
+ " bias_layout = weight_layout.delete([0])\n",
+ " self.b = dtensor.DVariable(\n",
+ " dtensor.call_with_layout(tf.zeros, bias_layout, shape=[out_dim]))\n",
+ "\n",
+ " def __call__(self, x):\n",
+ " # Compute the forward pass\n",
+ " z = tf.add(tf.matmul(x, self.w), self.b)\n",
+ " return self.activation(z)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "X-7MzpjgyHg6"
+ },
+ "source": [
+ "### The MLP sequential model\n",
+ "\n",
+ "Now create an MLP module that executes the dense layers sequentially."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "6XisRWiCyHAb"
+ },
+ "outputs": [],
+ "source": [
+ "class MLP(tf.Module):\n",
+ "\n",
+ " def __init__(self, layers):\n",
+ " self.layers = layers\n",
+ " \n",
+ " def __call__(self, x, preds=False): \n",
+ " # Execute the model's layers sequentially\n",
+ " for layer in self.layers:\n",
+ " x = layer(x)\n",
+ " return x"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "r5HZJ0kv-V3v"
+ },
+ "source": [
+ "Performing \"data-parallel\" training with DTensor is equivalent to `tf.distribute.MirroredStrategy`. To do this each device will run the same model on a shard of the data batch. So you'll need the following:\n",
+ "\n",
+ "* A `dtensor.Mesh` with a single `\"batch\"` dimension\n",
+ "* A `dtensor.Layout` for all the weights that replicates them across the mesh (using `dtensor.UNSHARDED` for each axis)\n",
+ "* A `dtensor.Layout` for the data that splits the batch dimension across the mesh\n",
+ "\n",
+ "\n",
+ "\n",
+ "\n",
+ "\n",
+ "Create a DTensor mesh that consists of a single batch dimension, where each device becomes a replica that receives a shard from the global batch. Use this mesh to instantiate an MLP mode with the following architecture:\n",
+ "\n",
+ "Forward Pass: ReLU(784 x 700) x ReLU(700 x 500) x Softmax(500 x 10)\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "VmlACuki3oPi"
+ },
+ "outputs": [],
+ "source": [
+ "mesh = dtensor.create_mesh([(\"batch\", 8)], devices=DEVICES)\n",
+ "weight_layout = dtensor.Layout([dtensor.UNSHARDED, dtensor.UNSHARDED], mesh)\n",
+ "\n",
+ "input_size = 784\n",
+ "hidden_layer_1_size = 700\n",
+ "hidden_layer_2_size = 500\n",
+ "hidden_layer_2_size = 10\n",
+ "\n",
+ "mlp_model = MLP([\n",
+ " DenseLayer(in_dim=input_size, out_dim=hidden_layer_1_size, \n",
+ " weight_layout=weight_layout,\n",
+ " activation=tf.nn.relu),\n",
+ " DenseLayer(in_dim=hidden_layer_1_size , out_dim=hidden_layer_2_size,\n",
+ " weight_layout=weight_layout,\n",
+ " activation=tf.nn.relu),\n",
+ " DenseLayer(in_dim=hidden_layer_2_size, out_dim=hidden_layer_2_size, \n",
+ " weight_layout=weight_layout)])"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "tyBATDoRmDkg"
+ },
+ "source": [
+ "### Training metrics\n",
+ "\n",
+ "Use the cross-entropy loss function and accuracy metric for training."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "rskOYA7FVCwg"
+ },
+ "outputs": [],
+ "source": [
+ "def cross_entropy_loss(y_pred, y):\n",
+ " # Compute cross entropy loss with a sparse operation\n",
+ " sparse_ce = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=y_pred)\n",
+ " return tf.reduce_mean(sparse_ce)\n",
+ "\n",
+ "def accuracy(y_pred, y):\n",
+ " # Compute accuracy after extracting class predictions\n",
+ " class_preds = tf.argmax(y_pred, axis=1)\n",
+ " is_equal = tf.equal(y, class_preds)\n",
+ " return tf.reduce_mean(tf.cast(is_equal, tf.float32))"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "JSiNRhTOnKZr"
+ },
+ "source": [
+ "### Optimizer\n",
+ "\n",
+ "Using an optimizer can result in significantly faster convergence compared to standard gradient descent. The Adam optimizer is implemented below and has been configured to be compatible with DTensor. In order to use Keras optimizers with DTensor, refer to the experimental`tf.keras.dtensor.experimental.optimizers` module."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "-9kIAI_lfXDS"
+ },
+ "outputs": [],
+ "source": [
+ "class Adam(tf.Module):\n",
+ "\n",
+ " def __init__(self, model_vars, learning_rate=1e-3, beta_1=0.9, beta_2=0.999, ep=1e-7):\n",
+ " # Initialize optimizer parameters and variable slots\n",
+ " self.model_vars = model_vars\n",
+ " self.beta_1 = beta_1\n",
+ " self.beta_2 = beta_2\n",
+ " self.learning_rate = learning_rate\n",
+ " self.ep = ep\n",
+ " self.t = 1.\n",
+ " self.v_dvar, self.s_dvar = [], []\n",
+ " # Initialize optimizer variable slots\n",
+ " for var in model_vars:\n",
+ " v = dtensor.DVariable(dtensor.call_with_layout(tf.zeros, var.layout, shape=var.shape))\n",
+ " s = dtensor.DVariable(dtensor.call_with_layout(tf.zeros, var.layout, shape=var.shape))\n",
+ " self.v_dvar.append(v)\n",
+ " self.s_dvar.append(s)\n",
+ "\n",
+ " def apply_gradients(self, grads):\n",
+ " # Update the model variables given their gradients\n",
+ " for i, (d_var, var) in enumerate(zip(grads, self.model_vars)):\n",
+ " self.v_dvar[i].assign(self.beta_1*self.v_dvar[i] + (1-self.beta_1)*d_var)\n",
+ " self.s_dvar[i].assign(self.beta_2*self.s_dvar[i] + (1-self.beta_2)*tf.square(d_var))\n",
+ " v_dvar_bc = self.v_dvar[i]/(1-(self.beta_1**self.t))\n",
+ " s_dvar_bc = self.s_dvar[i]/(1-(self.beta_2**self.t))\n",
+ " var.assign_sub(self.learning_rate*(v_dvar_bc/(tf.sqrt(s_dvar_bc) + self.ep)))\n",
+ " self.t += 1.\n",
+ " return "
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "w54b7GtLfn1j"
+ },
+ "source": [
+ "### Data packing\n",
+ "\n",
+ "Start by writing a helper function for transferring data to the device. This function should use `dtensor.pack` to send (and only send) the shard of the global batch that is intended for a replica to the device backing the replica. For simplicity, assume a single-client application.\n",
+ "\n",
+ "Next, write a function that uses this helper function to pack the training data batches into DTensors sharded along the batch (first) axis. This ensures that DTensor evenly distributes the training data to the 'batch' mesh dimension. Note that in DTensor, the batch size always refers to the global batch size; therefore, the batch size should be chosen such that it can be divided evenly by the size of the batch mesh dimension. Additional DTensor APIs to simplify `tf.data` integration are planned, so please stay tuned."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "3Rx82djZ6ITm"
+ },
+ "outputs": [],
+ "source": [
+ "def repack_local_tensor(x, layout):\n",
+ " # Repacks a local Tensor-like to a DTensor with layout\n",
+ " # This function assumes a single-client application\n",
+ " x = tf.convert_to_tensor(x)\n",
+ " sharded_dims = []\n",
+ "\n",
+ " # For every sharded dimension, use tf.split to split the along the dimension.\n",
+ " # The result is a nested list of split-tensors in queue[0].\n",
+ " queue = [x]\n",
+ " for axis, dim in enumerate(layout.sharding_specs):\n",
+ " if dim == dtensor.UNSHARDED:\n",
+ " continue\n",
+ " num_splits = layout.shape[axis]\n",
+ " queue = tf.nest.map_structure(lambda x: tf.split(x, num_splits, axis=axis), queue)\n",
+ " sharded_dims.append(dim)\n",
+ "\n",
+ " # Now you can build the list of component tensors by looking up the location in\n",
+ " # the nested list of split-tensors created in queue[0].\n",
+ " components = []\n",
+ " for locations in layout.mesh.local_device_locations():\n",
+ " t = queue[0]\n",
+ " for dim in sharded_dims:\n",
+ " split_index = locations[dim] # Only valid on single-client mesh.\n",
+ " t = t[split_index]\n",
+ " components.append(t)\n",
+ "\n",
+ " return dtensor.pack(components, layout)\n",
+ "\n",
+ "def repack_batch(x, y, mesh):\n",
+ " # Pack training data batches into DTensors along the batch axis\n",
+ " x = repack_local_tensor(x, layout=dtensor.Layout(['batch', dtensor.UNSHARDED], mesh))\n",
+ " y = repack_local_tensor(y, layout=dtensor.Layout(['batch'], mesh))\n",
+ " return x, y"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "osEK3rqpYfKd"
+ },
+ "source": [
+ "### Training\n",
+ "\n",
+ "Write a traceable function that executes a single training step given a batch of data. This function does not require any special DTensor annotations. Also write a function that executes a test step and returns the appropriate performance metrics."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "ZICEsDGuSbDD"
+ },
+ "outputs": [],
+ "source": [
+ "@tf.function\n",
+ "def train_step(model, x_batch, y_batch, loss, metric, optimizer):\n",
+ " # Execute a single training step\n",
+ " with tf.GradientTape() as tape:\n",
+ " y_pred = model(x_batch)\n",
+ " batch_loss = loss(y_pred, y_batch)\n",
+ " # Compute gradients and update the model's parameters\n",
+ " grads = tape.gradient(batch_loss, model.trainable_variables)\n",
+ " optimizer.apply_gradients(grads)\n",
+ " # Return batch loss and accuracy\n",
+ " batch_acc = metric(y_pred, y_batch)\n",
+ " return batch_loss, batch_acc\n",
+ "\n",
+ "@tf.function\n",
+ "def test_step(model, x_batch, y_batch, loss, metric):\n",
+ " # Execute a single testing step\n",
+ " y_pred = model(x_batch)\n",
+ " batch_loss = loss(y_pred, y_batch)\n",
+ " batch_acc = metric(y_pred, y_batch)\n",
+ " return batch_loss, batch_acc"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "RjIDVTwwX-Mr"
+ },
+ "source": [
+ "Now, train the MLP model for 3 epochs with a batch size of 128."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "oC85kuZgmh3q"
+ },
+ "outputs": [],
+ "source": [
+ "# Initialize the training loop parameters and structures\n",
+ "epochs = 3\n",
+ "batch_size = 128\n",
+ "train_losses, test_losses = [], []\n",
+ "train_accs, test_accs = [], []\n",
+ "optimizer = Adam(mlp_model.trainable_variables)\n",
+ "\n",
+ "# Format training loop\n",
+ "for epoch in range(epochs):\n",
+ " batch_losses_train, batch_accs_train = [], []\n",
+ " batch_losses_test, batch_accs_test = [], []\n",
+ "\n",
+ " # Iterate through training data\n",
+ " for x_batch, y_batch in train_data:\n",
+ " x_batch, y_batch = repack_batch(x_batch, y_batch, mesh)\n",
+ " batch_loss, batch_acc = train_step(mlp_model, x_batch, y_batch, cross_entropy_loss, accuracy, optimizer)\n",
+ " # Keep track of batch-level training performance\n",
+ " batch_losses_train.append(batch_loss)\n",
+ " batch_accs_train.append(batch_acc)\n",
+ "\n",
+ " # Iterate through testing data\n",
+ " for x_batch, y_batch in test_data:\n",
+ " x_batch, y_batch = repack_batch(x_batch, y_batch, mesh)\n",
+ " batch_loss, batch_acc = test_step(mlp_model, x_batch, y_batch, cross_entropy_loss, accuracy)\n",
+ " # Keep track of batch-level testing\n",
+ " batch_losses_test.append(batch_loss)\n",
+ " batch_accs_test.append(batch_acc)\n",
+ "\n",
+ "# Keep track of epoch-level model performance\n",
+ " train_loss, train_acc = tf.reduce_mean(batch_losses_train), tf.reduce_mean(batch_accs_train)\n",
+ " test_loss, test_acc = tf.reduce_mean(batch_losses_test), tf.reduce_mean(batch_accs_test)\n",
+ " train_losses.append(train_loss)\n",
+ " train_accs.append(train_acc)\n",
+ " test_losses.append(test_loss)\n",
+ " test_accs.append(test_acc)\n",
+ " print(f\"Epoch: {epoch}\")\n",
+ " print(f\"Training loss: {train_loss.numpy():.3f}, Training accuracy: {train_acc.numpy():.3f}\")\n",
+ " print(f\"Testing loss: {test_loss.numpy():.3f}, Testing accuracy: {test_acc.numpy():.3f}\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "j_RVmt43G12R"
+ },
+ "source": [
+ "### Performance evaluation\n",
+ "\n",
+ "Start by writing a plotting function to visualize the model's loss and accuracy during training. "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "VXTCYVtNDjAM"
+ },
+ "outputs": [],
+ "source": [
+ "def plot_metrics(train_metric, test_metric, metric_type):\n",
+ " # Visualize metrics vs training Epochs\n",
+ " plt.figure()\n",
+ " plt.plot(range(len(train_metric)), train_metric, label = f\"Training {metric_type}\")\n",
+ " plt.plot(range(len(test_metric)), test_metric, label = f\"Testing {metric_type}\")\n",
+ " plt.xlabel(\"Epochs\")\n",
+ " plt.ylabel(metric_type)\n",
+ " plt.legend()\n",
+ " plt.title(f\"{metric_type} vs Training Epochs\");"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "407qok7q2JIO"
+ },
+ "outputs": [],
+ "source": [
+ "plot_metrics(train_losses, test_losses, \"Cross entropy loss\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "8H_TgxV92NfX"
+ },
+ "outputs": [],
+ "source": [
+ "plot_metrics(train_accs, test_accs, \"Accuracy\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "DHO_u-3w4YRF"
+ },
+ "source": [
+ "## Saving your model\n",
+ "\n",
+ "The integration of `tf.saved_model` and DTensor is still under development. As of TensorFlow 2.9.0, tf.saved_model only accepts DTensor models with fully replicated variables. As a workaround, you can convert a DTensor model to a fully replicated one by reloading a checkpoint. However, after a model is saved, all DTensor annotations are lost and the saved signatures can only be used with regular Tensors. This tutorial will be updated to showcase the integration once it is solidified.\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "VFLfEH4ManbW"
+ },
+ "source": [
+ "## Conclusion\n",
+ "\n",
+ "This notebook provided an overview of distributed training with DTensor and the TensorFlow Core APIs. Here are a few more tips that may help:\n",
+ "\n",
+ "- The [TensorFlow Core APIs](https://www.tensorflow.org/guide/core) can be used to build highly-configurable machine learning workflows with support for distributed training.\n",
+ "- The [DTensor concepts](https://www.tensorflow.org/guide/dtensor_overview) guide and [Distributed training with DTensors](https://www.tensorflow.org/tutorials/distribute/dtensor_ml_tutorial) tutorial contain the most up-to-date information about DTensor and its integrations.\n",
+ "\n",
+ "For more examples of using the TensorFlow Core APIs, check out the [guide](https://www.tensorflow.org/guide/core). If you want to learn more about loading and preparing data, see the tutorials on [image data loading](https://www.tensorflow.org/tutorials/load_data/images) or [CSV data loading](https://www.tensorflow.org/tutorials/load_data/csv)."
+ ]
+ }
+ ],
+ "metadata": {
+ "colab": {
+ "collapsed_sections": [
+ "FhGuhbZ6M5tl"
+ ],
+ "name": "distribution.ipynb",
+ "toc_visible": true
+ },
+ "kernelspec": {
+ "display_name": "Python 3",
+ "name": "python3"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}
diff --git a/site/en/guide/core/index.md b/site/en/guide/core/index.md
new file mode 100644
index 00000000000..7f44e11b018
--- /dev/null
+++ b/site/en/guide/core/index.md
@@ -0,0 +1,112 @@
+# TensorFlow Core APIs overview
+
+The TensorFlow Core APIs provide a set of comprehensive, composable, and
+extensible low-level APIs for high-performance (distributed and accelerated)
+computation, primarily aimed at building machine learning (ML) models as well as
+authoring ML workflow tools and frameworks within the TensorFlow platform. These
+APIs provide a foundation for creating highly configurable models with
+fine-grained control and new frameworks from the ground up.
+
+The Core APIs can be used as an alternative to high-level machine learning APIs
+like Keras. These high-level APIs are best suited for general machine learning
+needs. They offer a variety of modules that abstract away the complexities of ML
+while also offering functionalities for customization through subclassing. If
+you are looking for an overview of TensorFlow using Keras, see the Quickstarts
+and Keras sections in the [tutorials](https://www.tensorflow.org/tutorials).
+
+## Who should use the Core APIs
+
+The TensorFlow Core low-level APIs are designed with the following ML developers
+in mind:
+
+* Researchers building complex models with high levels of configurability
+* Developers interested in using TensorFlow as a high-performance scientific
+ computing platform
+* Framework authors building tools on top of the TensorFlow platform
+* High-level API users interested in:
+ * Adding additional functionalities to their machine learning workflows
+ such as custom layers, losses, models, and optimizers
+ * Learning more about the inner workings of their models
+
+## Core API applications
+
+The TensorFlow Core APIs provide access to low level functionality within the
+TensorFlow ecosystem. This API provides more flexibility and control for
+building ML models, applications, and tools, compared to high-level APIs, such
+as Keras.
+
+### Build models and workflows
+
+The Core APIs are most commonly used to build highly customizable and optimized
+machine learning models and workflows. Here are some of the ways that the
+TensorFlow Core APIs can improve your machine learning models and workflow
+development:
+
+
+
+* Building non-traditional models or layers that do not fully fit the
+ structures supported by high-level APIs
+* Building custom layers, losses, models, and optimizers within Keras
+* Implementing new optimization techniques to expedite convergence during
+ training
+* Creating custom metrics for performance evaluation
+* Designing highly-configurable training loops with support for features like
+ batching, cross-validation, and distribution strategies
+
+### Build frameworks and tools
+
+The TensorFlow Core APIs can also serve as the building blocks for new
+high-level frameworks. Here are some examples of tools and frameworks that are
+created with the low-level APIs:
+
+
+* [Keras](https://keras.io): deep learning for humans
+* [TensorFlow Model Optimization Toolkit](https://www.tensorflow.org/model_optimization):
+ a suite of tools to optimize ML models for deployment and execution
+* [TensorFlow Graphics](https://www.tensorflow.org/graphics): a library for
+ making useful graphics functions widely accessible
+
+### Build for scientific computing
+
+The TensorFlow Core APIs can also be applied outside the realm of machine
+learning. Here are a few general-purpose use cases of TensorFlow for scientific
+computing:
+
+
+* Physics simulations for solid mechanics and
+ [fluid dynamics](https://arxiv.org/abs/2108.11076) problems
+* Graphics rendering applications like
+ [ray tracing](https://github.com/BachiLi/redner)
+* Solving
+ [constrained optimization problems](https://github.com/google-research/tensorflow_constrained_optimization/blob/master/README.md)
+
+## Core API components
+
+Here are some of the fundamental components that comprise TensorFlow Core’s low-
+level APIs. Note that this is not an all-encompassing list:
+
+
+
+* Data structures : `tf.Tensor`, `tf.Variable`, `tf.TensorArray`
+* Primitive APIs: `tf.shape`,
+ [slicing](https://www.tensorflow.org/guide/tensor_slicing), `tf.concat`,
+ `tf.bitwise`
+* Numerical: `tf.math`, `tf.linalg`, `tf.random`
+* Functional components: `tf.function`, `tf.GradientTape`
+* Distribution: [DTensor](https://www.tensorflow.org/guide/dtensor_overview)
+* Export: `tf.saved_model`
+
+## Next steps
+
+The *Build with Core* documentation provides tutorials of basic machine learning
+concepts from scratch. The tutorials in this section help you get comfortable
+with writing low-level code with Core APIs that you can then apply to more
+complex use cases of your own.
+
+Note: You should not use the Core APIs to simply re-implement high-level APIs,
+and it is possible to use high-level APIs, such as Keras, with the Core APIs.
+
+To get started using and learning more about the Core APIs, check out the
+[Quickstart for TensorFlow Core](https://www.tensorflow.org/guide/core/quickstart_core).
diff --git a/site/en/guide/core/logistic_regression_core.ipynb b/site/en/guide/core/logistic_regression_core.ipynb
new file mode 100644
index 00000000000..5a9af324ad5
--- /dev/null
+++ b/site/en/guide/core/logistic_regression_core.ipynb
@@ -0,0 +1,935 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "FhGuhbZ6M5tl"
+ },
+ "source": [
+ "##### Copyright 2022 The TensorFlow Authors."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "cellView": "form",
+ "id": "AwOEIRJC6Une"
+ },
+ "outputs": [],
+ "source": [
+ "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+ "# you may not use this file except in compliance with the License.\n",
+ "# You may obtain a copy of the License at\n",
+ "#\n",
+ "# https://www.apache.org/licenses/LICENSE-2.0\n",
+ "#\n",
+ "# Unless required by applicable law or agreed to in writing, software\n",
+ "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+ "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+ "# See the License for the specific language governing permissions and\n",
+ "# limitations under the License."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "EIdT9iu_Z4Rb"
+ },
+ "source": [
+ "# Logistic regression for binary classification with Core APIs"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "bBIlTPscrIT9"
+ },
+ "source": [
+ "
"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "DauaqJ7WhIhO"
+ },
+ "source": [
+ "This guide demonstrates how to use the [TensorFlow Core low-level APIs](https://www.tensorflow.org/guide/core) to perform [binary classification](https://developers.google.com/machine-learning/glossary#binary_classification) with [logistic regression](https://developers.google.com/machine-learning/crash-course/logistic-regression/). It uses the [Wisconsin Breast Cancer Dataset](https://archive.ics.uci.edu/ml/datasets/breast+cancer+wisconsin+(original)) for tumor classification.\n",
+ "\n",
+ "[Logistic regression](https://developers.google.com/machine-learning/crash-course/logistic-regression/) is one of the most popular algorithms for binary classification. Given a set of examples with features, the goal of logistic regression is to output values between 0 and 1, which can be interpreted as the probabilities of each example belonging to a particular class. "
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "nchsZfwEVtVs"
+ },
+ "source": [
+ "## Setup\n",
+ "\n",
+ "This tutorial uses [pandas](https://pandas.pydata.org) for reading a CSV file into a [DataFrame](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.html), [seaborn](https://seaborn.pydata.org) for plotting a pairwise relationship in a dataset, [Scikit-learn](https://scikit-learn.org/) for computing a confusion matrix, and [matplotlib](https://matplotlib.org/) for creating visualizations."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "5lZoUK6AVTos"
+ },
+ "outputs": [],
+ "source": [
+ "!pip install -q seaborn"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "1rRo8oNqZ-Rj"
+ },
+ "outputs": [],
+ "source": [
+ "import tensorflow as tf\n",
+ "import pandas as pd\n",
+ "import matplotlib\n",
+ "from matplotlib import pyplot as plt\n",
+ "import seaborn as sns\n",
+ "import sklearn.metrics as sk_metrics\n",
+ "import tempfile\n",
+ "import os\n",
+ "\n",
+ "# Preset matplotlib figure sizes.\n",
+ "matplotlib.rcParams['figure.figsize'] = [9, 6]\n",
+ "\n",
+ "print(tf.__version__)\n",
+ "# To make the results reproducible, set the random seed value.\n",
+ "tf.random.set_seed(22)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "gFh9ne3FZ-On"
+ },
+ "source": [
+ "## Load the data\n",
+ "\n",
+ "Next, load the [Wisconsin Breast Cancer Dataset](https://archive.ics.uci.edu/ml/datasets/breast+cancer+wisconsin+(original)) from the [UCI Machine Learning Repository](https://archive.ics.uci.edu/ml/). This dataset contains various features such as a tumor's radius, texture, and concavity."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "CiX2FI4gZtTt"
+ },
+ "outputs": [],
+ "source": [
+ "url = 'https://archive.ics.uci.edu/ml/machine-learning-databases/breast-cancer-wisconsin/wdbc.data'\n",
+ "\n",
+ "features = ['radius', 'texture', 'perimeter', 'area', 'smoothness', 'compactness',\n",
+ " 'concavity', 'concave_poinits', 'symmetry', 'fractal_dimension']\n",
+ "column_names = ['id', 'diagnosis']\n",
+ "\n",
+ "for attr in ['mean', 'ste', 'largest']:\n",
+ " for feature in features:\n",
+ " column_names.append(feature + \"_\" + attr)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "A3VR1aTP92nV"
+ },
+ "source": [
+ "Read the dataset into a pandas [DataFrame]() using [`pandas.read_csv`](https://pandas.pydata.org/docs/reference/api/pandas.read_csv.html):"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "uvR2Bzb691lJ"
+ },
+ "outputs": [],
+ "source": [
+ "dataset = pd.read_csv(url, names=column_names)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "YB9eq6Zq-IZ4"
+ },
+ "outputs": [],
+ "source": [
+ "dataset.info()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "0_Z1V6Dg-La_"
+ },
+ "source": [
+ "Display the first five rows:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "hWxktwbv-KPp"
+ },
+ "outputs": [],
+ "source": [
+ "dataset.head()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "s4-Wn2jzVC1W"
+ },
+ "source": [
+ "Split the dataset into training and test sets using [`pandas.DataFrame.sample`](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.sample.html), [`pandas.DataFrame.drop`](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.drop.html) and [`pandas.DataFrame.iloc`](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.iloc.html). Make sure to split the features from the target labels. The test set is used to evaluate your model's generalizability to unseen data."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "m2O60B-IVG9Q"
+ },
+ "outputs": [],
+ "source": [
+ "train_dataset = dataset.sample(frac=0.75, random_state=1)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "i06vHFv_QB24"
+ },
+ "outputs": [],
+ "source": [
+ "len(train_dataset)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "19JaochhaQ3m"
+ },
+ "outputs": [],
+ "source": [
+ "test_dataset = dataset.drop(train_dataset.index)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "LmHRcbAfaSag"
+ },
+ "outputs": [],
+ "source": [
+ "len(test_dataset)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "w6JxBhBc_wwO"
+ },
+ "outputs": [],
+ "source": [
+ "# The `id` column can be dropped since each row is unique\n",
+ "x_train, y_train = train_dataset.iloc[:, 2:], train_dataset.iloc[:, 1]\n",
+ "x_test, y_test = test_dataset.iloc[:, 2:], test_dataset.iloc[:, 1]"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "3MWuJTKEDM-f"
+ },
+ "source": [
+ "## Preprocess the data\n",
+ "\n",
+ "This dataset contains the mean, standard error, and largest values for each of the 10 tumor measurements collected per example. The `\"diagnosis\"` target column is a categorical variable with `'M'` indicating a malignant tumor and `'B'` indicating a benign tumor diagnosis. This column needs to be converted into a numerical binary format for model training.\n",
+ "\n",
+ "The [`pandas.Series.map`](https://pandas.pydata.org/docs/reference/api/pandas.Series.map.html) function is useful for mapping binary values to the categories.\n",
+ "\n",
+ "The dataset should also be converted to a tensor with the `tf.convert_to_tensor` function after the preprocessing is complete."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "JEJHhN65a2VV"
+ },
+ "outputs": [],
+ "source": [
+ "y_train, y_test = y_train.map({'B': 0, 'M': 1}), y_test.map({'B': 0, 'M': 1})\n",
+ "x_train, y_train = tf.convert_to_tensor(x_train, dtype=tf.float32), tf.convert_to_tensor(y_train, dtype=tf.float32)\n",
+ "x_test, y_test = tf.convert_to_tensor(x_test, dtype=tf.float32), tf.convert_to_tensor(y_test, dtype=tf.float32)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "J4ubs136WLNp"
+ },
+ "source": [
+ "Use [`seaborn.pairplot`](https://seaborn.pydata.org/generated/seaborn.pairplot.html) to review the joint distribution of a few pairs of mean-based features from the training set and observe how they relate to the target:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "oRKO_x8gWKv-"
+ },
+ "outputs": [],
+ "source": [
+ "sns.pairplot(train_dataset.iloc[:, 1:6], hue = 'diagnosis', diag_kind='kde');"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "5YOG5iKYKW_3"
+ },
+ "source": [
+ "This pairplot demonstrates that certain features such as radius, perimeter and area are highly correlated. This is expected since the tumor radius is directly involved in the computation of both perimeter and area. Additionally, note that malignant diagnoses seem to be more right-skewed for many of the features.\n",
+ "\n",
+ "Make sure to also check the overall statistics. Note how each feature covers a vastly different range of values."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "yi2FzC3T21jR"
+ },
+ "outputs": [],
+ "source": [
+ "train_dataset.describe().transpose()[:10]"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "_8pDCIFjMla8"
+ },
+ "source": [
+ "Given the inconsistent ranges, it is beneficial to standardize the data such that each feature has a zero mean and unit variance. This process is called [normalization](https://developers.google.com/machine-learning/glossary#normalization)."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "FrzKNFNjLQDl"
+ },
+ "outputs": [],
+ "source": [
+ "class Normalize(tf.Module):\n",
+ " def __init__(self, x):\n",
+ " # Initialize the mean and standard deviation for normalization\n",
+ " self.mean = tf.Variable(tf.math.reduce_mean(x, axis=0))\n",
+ " self.std = tf.Variable(tf.math.reduce_std(x, axis=0))\n",
+ "\n",
+ " def norm(self, x):\n",
+ " # Normalize the input\n",
+ " return (x - self.mean)/self.std\n",
+ "\n",
+ " def unnorm(self, x):\n",
+ " # Unnormalize the input\n",
+ " return (x * self.std) + self.mean\n",
+ "\n",
+ "norm_x = Normalize(x_train)\n",
+ "x_train_norm, x_test_norm = norm_x.norm(x_train), norm_x.norm(x_test)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "6o3CrycBXA2s"
+ },
+ "source": [
+ "## Logistic regression\n",
+ "\n",
+ "Before building a logistic regression model, it is crucial to understand the method's differences compared to traditional linear regression.\n",
+ "\n",
+ "### Logistic regression fundamentals\n",
+ "\n",
+ "Linear regression returns a linear combination of its inputs; this output is unbounded. The output of a [logistic regression](https://developers.google.com/machine-learning/glossary#logistic_regression) is in the `(0, 1)` range. For each example, it represents the probability that the example belongs to the _positive_ class.\n",
+ "\n",
+ "Logistic regression maps the continuous outputs of traditional linear regression, `(-∞, ∞)`, to probabilities, `(0, 1)`. This transformation is also symmetric so that flipping the sign of the linear output results in the inverse of the original probability.\n",
+ "\n",
+ "Let $Y$ denote the probability of being in class `1` (the tumor is malignant). The desired mapping can be achieved by interpreting the linear regression output as the [log odds](https://developers.google.com/machine-learning/glossary#log-odds) ratio of being in class `1` as opposed to class `0`:\n",
+ "\n",
+ "$$\\ln(\\frac{Y}{1-Y}) = wX + b$$\n",
+ "\n",
+ "By setting $wX + b = z$, this equation can then be solved for $Y$:\n",
+ "\n",
+ "$$Y = \\frac{e^{z}}{1 + e^{z}} = \\frac{1}{1 + e^{-z}}$$\n",
+ "\n",
+ "The expression $\\frac{1}{1 + e^{-z}}$ is known as the [sigmoid function](https://developers.google.com/machine-learning/glossary#sigmoid_function) $\\sigma(z)$. Hence, the equation for logistic regression can be written as $Y = \\sigma(wX + b)$.\n",
+ "\n",
+ "The dataset in this tutorial deals with a high-dimensional feature matrix. Therefore, the above equation must be rewritten in a matrix vector form as follows:\n",
+ "\n",
+ "$${\\mathrm{Y}} = \\sigma({\\mathrm{X}}w + b)$$\n",
+ "\n",
+ "where:\n",
+ "\n",
+ "* $\\underset{m\\times 1}{\\mathrm{Y}}$: a target vector\n",
+ "* $\\underset{m\\times n}{\\mathrm{X}}$: a feature matrix\n",
+ "* $\\underset{n\\times 1}w$: a weight vector\n",
+ "* $b$: a bias\n",
+ "* $\\sigma$: a sigmoid function applied to each element of the output vector\n",
+ "\n",
+ "Start by visualizing the sigmoid function, which transforms the linear output, `(-∞, ∞)`, to fall between `0` and `1`. The sigmoid function is available in `tf.math.sigmoid`."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "ThHaV_RmucZl"
+ },
+ "outputs": [],
+ "source": [
+ "x = tf.linspace(-10, 10, 500)\n",
+ "x = tf.cast(x, tf.float32)\n",
+ "f = lambda x : (1/20)*x + 0.6\n",
+ "plt.plot(x, tf.math.sigmoid(x))\n",
+ "plt.ylim((-0.1,1.1))\n",
+ "plt.title(\"Sigmoid function\");"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "VMXEhrZuKECV"
+ },
+ "source": [
+ "### The log loss function\n",
+ "\n",
+ "The [log loss](https://developers.google.com/machine-learning/glossary#Log_Loss), or binary cross-entropy loss, is the ideal loss function for a binary classification problem with logistic regression. For each example, the log loss quantifies the similarity between a predicted probability and the example's true value. It is determined by the following equation:\n",
+ "\n",
+ "$$L = -\\frac{1}{m}\\sum_{i=1}^{m}y_i\\cdot\\log(\\hat{y}_i) + (1- y_i)\\cdot\\log(1 - \\hat{y}_i)$$\n",
+ "\n",
+ "where:\n",
+ "\n",
+ "* $\\hat{y}$: a vector of predicted probabilities\n",
+ "* $y$: a vector of true targets\n",
+ "\n",
+ "You can use the `tf.nn.sigmoid_cross_entropy_with_logits` function to compute the log loss. This function automatically applies the sigmoid activation to the regression output:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "JVBInnSqS36W"
+ },
+ "outputs": [],
+ "source": [
+ "def log_loss(y_pred, y):\n",
+ " # Compute the log loss function\n",
+ " ce = tf.nn.sigmoid_cross_entropy_with_logits(labels=y, logits=y_pred)\n",
+ " return tf.reduce_mean(ce)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "Q_mutLj0KNUb"
+ },
+ "source": [
+ "### The gradient descent update rule\n",
+ "\n",
+ "The TensorFlow Core APIs support automatic differentiation with `tf.GradientTape`. If you are curious about the mathematics behind the logistic regression [gradient updates](https://developers.google.com/machine-learning/glossary#gradient_descent), here is a short explanation:\n",
+ "\n",
+ "In the above equation for the log loss, recall that each $\\hat{y}_i$ can be rewritten in terms of the inputs as $\\sigma({\\mathrm{X_i}}w + b)$.\n",
+ "\n",
+ "The goal is to find a $w^*$ and $b^*$ that minimize the log loss:\n",
+ "\n",
+ "$$L = -\\frac{1}{m}\\sum_{i=1}^{m}y_i\\cdot\\log(\\sigma({\\mathrm{X_i}}w + b)) + (1- y_i)\\cdot\\log(1 - \\sigma({\\mathrm{X_i}}w + b))$$\n",
+ "\n",
+ "By taking the gradient $L$ with respect to $w$, you get the following:\n",
+ "\n",
+ "$$\\frac{\\partial L}{\\partial w} = \\frac{1}{m}(\\sigma({\\mathrm{X}}w + b) - y)X$$\n",
+ "\n",
+ "By taking the gradient $L$ with respect to $b$, you get the following:\n",
+ "\n",
+ "$$\\frac{\\partial L}{\\partial b} = \\frac{1}{m}\\sum_{i=1}^{m}\\sigma({\\mathrm{X_i}}w + b) - y_i$$"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "uTCndUecKZho"
+ },
+ "source": [
+ "Now, build the logistic regression model."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "c0sXM7qLlKfZ"
+ },
+ "outputs": [],
+ "source": [
+ "class LogisticRegression(tf.Module):\n",
+ "\n",
+ " def __init__(self):\n",
+ " self.built = False\n",
+ " \n",
+ " def __call__(self, x, train=True):\n",
+ " # Initialize the model parameters on the first call\n",
+ " if not self.built:\n",
+ " # Randomly generate the weights and the bias term\n",
+ " rand_w = tf.random.uniform(shape=[x.shape[-1], 1], seed=22)\n",
+ " rand_b = tf.random.uniform(shape=[], seed=22)\n",
+ " self.w = tf.Variable(rand_w)\n",
+ " self.b = tf.Variable(rand_b)\n",
+ " self.built = True\n",
+ " # Compute the model output\n",
+ " z = tf.add(tf.matmul(x, self.w), self.b)\n",
+ " z = tf.squeeze(z, axis=1)\n",
+ " if train:\n",
+ " return z\n",
+ " return tf.sigmoid(z)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "eObQu9fDnXGL"
+ },
+ "source": [
+ "To validate, make sure the untrained model outputs values in the range of `(0, 1)` for a small subset of the training data."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "5bIovC0Z4QHJ"
+ },
+ "outputs": [],
+ "source": [
+ "log_reg = LogisticRegression()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "QJ2ievISyf0p"
+ },
+ "outputs": [],
+ "source": [
+ "y_pred = log_reg(x_train_norm[:5], train=False)\n",
+ "y_pred.numpy()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "PribnwDHUksC"
+ },
+ "source": [
+ "Next, write an accuracy function to calculate the proportion of correct classifications during training. In order to retrieve the classifications from the predicted probabilities, set a threshold for which all probabilities higher than the threshold belong to class `1`. This is a configurable hyperparameter that can be set to `0.5` as a default."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "ssnVcKg7oMe6"
+ },
+ "outputs": [],
+ "source": [
+ "def predict_class(y_pred, thresh=0.5):\n",
+ " # Return a tensor with `1` if `y_pred` > `0.5`, and `0` otherwise\n",
+ " return tf.cast(y_pred > thresh, tf.float32)\n",
+ "\n",
+ "def accuracy(y_pred, y):\n",
+ " # Return the proportion of matches between `y_pred` and `y`\n",
+ " y_pred = tf.math.sigmoid(y_pred)\n",
+ " y_pred_class = predict_class(y_pred)\n",
+ " check_equal = tf.cast(y_pred_class == y,tf.float32)\n",
+ " acc_val = tf.reduce_mean(check_equal)\n",
+ " return acc_val"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "J_0KHQ25_2dF"
+ },
+ "source": [
+ "### Train the model\n",
+ "\n",
+ "Using mini-batches for training provides both memory efficiency and faster convergence. The `tf.data.Dataset` API has useful functions for batching and shuffling. The API enables you to build complex input pipelines from simple, reusable pieces. "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "vJD7-4U0etqa"
+ },
+ "outputs": [],
+ "source": [
+ "batch_size = 64\n",
+ "train_dataset = tf.data.Dataset.from_tensor_slices((x_train_norm, y_train))\n",
+ "train_dataset = train_dataset.shuffle(buffer_size=x_train.shape[0]).batch(batch_size)\n",
+ "test_dataset = tf.data.Dataset.from_tensor_slices((x_test_norm, y_test))\n",
+ "test_dataset = test_dataset.shuffle(buffer_size=x_test.shape[0]).batch(batch_size)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "sLiWZZPBSDip"
+ },
+ "source": [
+ "Now write a training loop for the logistic regression model. The loop utilizes the log loss function and its gradients with respect to the input in order to iteratively update the model's parameters."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "jNC3D1DGsGgK"
+ },
+ "outputs": [],
+ "source": [
+ "# Set training parameters\n",
+ "epochs = 200\n",
+ "learning_rate = 0.01\n",
+ "train_losses, test_losses = [], []\n",
+ "train_accs, test_accs = [], []\n",
+ "\n",
+ "# Set up the training loop and begin training\n",
+ "for epoch in range(epochs):\n",
+ " batch_losses_train, batch_accs_train = [], []\n",
+ " batch_losses_test, batch_accs_test = [], []\n",
+ "\n",
+ " # Iterate over the training data\n",
+ " for x_batch, y_batch in train_dataset:\n",
+ " with tf.GradientTape() as tape:\n",
+ " y_pred_batch = log_reg(x_batch)\n",
+ " batch_loss = log_loss(y_pred_batch, y_batch)\n",
+ " batch_acc = accuracy(y_pred_batch, y_batch)\n",
+ " # Update the parameters with respect to the gradient calculations\n",
+ " grads = tape.gradient(batch_loss, log_reg.variables)\n",
+ " for g,v in zip(grads, log_reg.variables):\n",
+ " v.assign_sub(learning_rate * g)\n",
+ " # Keep track of batch-level training performance\n",
+ " batch_losses_train.append(batch_loss)\n",
+ " batch_accs_train.append(batch_acc)\n",
+ "\n",
+ " # Iterate over the testing data\n",
+ " for x_batch, y_batch in test_dataset:\n",
+ " y_pred_batch = log_reg(x_batch)\n",
+ " batch_loss = log_loss(y_pred_batch, y_batch)\n",
+ " batch_acc = accuracy(y_pred_batch, y_batch)\n",
+ " # Keep track of batch-level testing performance\n",
+ " batch_losses_test.append(batch_loss)\n",
+ " batch_accs_test.append(batch_acc)\n",
+ "\n",
+ " # Keep track of epoch-level model performance\n",
+ " train_loss, train_acc = tf.reduce_mean(batch_losses_train), tf.reduce_mean(batch_accs_train)\n",
+ " test_loss, test_acc = tf.reduce_mean(batch_losses_test), tf.reduce_mean(batch_accs_test)\n",
+ " train_losses.append(train_loss)\n",
+ " train_accs.append(train_acc)\n",
+ " test_losses.append(test_loss)\n",
+ " test_accs.append(test_acc)\n",
+ " if epoch % 20 == 0:\n",
+ " print(f\"Epoch: {epoch}, Training log loss: {train_loss:.3f}\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "NoLiAg7fYft7"
+ },
+ "source": [
+ "### Performance evaluation\n",
+ "\n",
+ "Observe the changes in your model's loss and accuracy over time. "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "mv3oCQPvWhr0"
+ },
+ "outputs": [],
+ "source": [
+ "plt.plot(range(epochs), train_losses, label = \"Training loss\")\n",
+ "plt.plot(range(epochs), test_losses, label = \"Testing loss\")\n",
+ "plt.xlabel(\"Epoch\")\n",
+ "plt.ylabel(\"Log loss\")\n",
+ "plt.legend()\n",
+ "plt.title(\"Log loss vs training iterations\");"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "D2HDVGLPODIE"
+ },
+ "outputs": [],
+ "source": [
+ "plt.plot(range(epochs), train_accs, label = \"Training accuracy\")\n",
+ "plt.plot(range(epochs), test_accs, label = \"Testing accuracy\")\n",
+ "plt.xlabel(\"Epoch\")\n",
+ "plt.ylabel(\"Accuracy (%)\")\n",
+ "plt.legend()\n",
+ "plt.title(\"Accuracy vs training iterations\");"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "jonKhUzuPyfa"
+ },
+ "outputs": [],
+ "source": [
+ "print(f\"Final training log loss: {train_losses[-1]:.3f}\")\n",
+ "print(f\"Final testing log Loss: {test_losses[-1]:.3f}\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "d3DF4qyrPyke"
+ },
+ "outputs": [],
+ "source": [
+ "print(f\"Final training accuracy: {train_accs[-1]:.3f}\")\n",
+ "print(f\"Final testing accuracy: {test_accs[-1]:.3f}\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "yrj1TbOJasjA"
+ },
+ "source": [
+ "The model demonstrates a high accuracy and a low loss when it comes to classifying tumors in the training dataset and also generalizes well to the unseen test data. To go one step further, you can explore error rates that give more insight beyond the overall accuracy score. The two most popular error rates for binary classification problems are the false positive rate (FPR) and the false negative rate (FNR).\n",
+ "\n",
+ "For this problem, the FPR is the proportion of malignant tumor predictions amongst tumors that are actually benign. Conversely, the FNR is the proportion of benign tumor predictions among tumors that are actually malignant.\n",
+ "\n",
+ "Compute a confusion matrix using [`sklearn.metrics.confusion_matrix`](https://scikit-learn.org/stable/modules/generated/sklearn.metrics.confusion_matrix.html#sklearn.metrics.confusion_matrix), which evaluates the accuracy of the classification, and use matplotlib to display the matrix:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "OJO7YkA8ZDMU"
+ },
+ "outputs": [],
+ "source": [
+ "def show_confusion_matrix(y, y_classes, typ):\n",
+ " # Compute the confusion matrix and normalize it\n",
+ " plt.figure(figsize=(10,10))\n",
+ " confusion = sk_metrics.confusion_matrix(y.numpy(), y_classes.numpy())\n",
+ " confusion_normalized = confusion / confusion.sum(axis=1, keepdims=True)\n",
+ " axis_labels = range(2)\n",
+ " ax = sns.heatmap(\n",
+ " confusion_normalized, xticklabels=axis_labels, yticklabels=axis_labels,\n",
+ " cmap='Blues', annot=True, fmt='.4f', square=True)\n",
+ " plt.title(f\"Confusion matrix: {typ}\")\n",
+ " plt.ylabel(\"True label\")\n",
+ " plt.xlabel(\"Predicted label\")\n",
+ "\n",
+ "y_pred_train, y_pred_test = log_reg(x_train_norm, train=False), log_reg(x_test_norm, train=False)\n",
+ "train_classes, test_classes = predict_class(y_pred_train), predict_class(y_pred_test)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "OQ5DFcleiDFm"
+ },
+ "outputs": [],
+ "source": [
+ "show_confusion_matrix(y_train, train_classes, 'Training')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "gtfcsAp_iCNR"
+ },
+ "outputs": [],
+ "source": [
+ "show_confusion_matrix(y_test, test_classes, 'Testing')"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "DlivxaDmTnGq"
+ },
+ "source": [
+ "Observe the error rate measurements and interpret their significance in the context of this example. In many medical testing studies such as cancer detection, having a high false positive rate to ensure a low false negative rate is perfectly acceptable and in fact encouraged since the risk of missing a malignant tumor diagnosis (false negative) is a lot worse than misclassifying a benign tumor as malignant (false positive).\n",
+ "\n",
+ "In order to control for the FPR and FNR, try changing the threshold hyperparameter before classifying the probability predictions. A lower threshold increases the model's overall chances of making a malignant tumor classification. This inevitably increases the number of false positives and the FPR but it also helps to decrease the number of false negatives and the FNR."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "7ADEN2rb4Nhj"
+ },
+ "source": [
+ "## Save the model\n",
+ "\n",
+ "Start by making an export module that takes in raw data and performs the following operations:\n",
+ "- Normalization\n",
+ "- Probability prediction\n",
+ "- Class prediction\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "6KPRHCzg4ZxH"
+ },
+ "outputs": [],
+ "source": [
+ "class ExportModule(tf.Module):\n",
+ " def __init__(self, model, norm_x, class_pred):\n",
+ " # Initialize pre- and post-processing functions\n",
+ " self.model = model\n",
+ " self.norm_x = norm_x\n",
+ " self.class_pred = class_pred\n",
+ "\n",
+ " @tf.function(input_signature=[tf.TensorSpec(shape=[None, None], dtype=tf.float32)])\n",
+ " def __call__(self, x):\n",
+ " # Run the `ExportModule` for new data points\n",
+ " x = self.norm_x.norm(x)\n",
+ " y = self.model(x, train=False)\n",
+ " y = self.class_pred(y)\n",
+ " return y "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "2YzRclo5-yjO"
+ },
+ "outputs": [],
+ "source": [
+ "log_reg_export = ExportModule(model=log_reg,\n",
+ " norm_x=norm_x,\n",
+ " class_pred=predict_class)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "gtofGIBN_qFd"
+ },
+ "source": [
+ "If you want to save the model at its current state, you can do so with the `tf.saved_model.save` function. To load a saved model and make predictions, use the `tf.saved_model.load` function."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "a4Qum1Ts_pmF"
+ },
+ "outputs": [],
+ "source": [
+ "models = tempfile.mkdtemp()\n",
+ "save_path = os.path.join(models, 'log_reg_export')\n",
+ "tf.saved_model.save(log_reg_export, save_path)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "3KPILr1i_M_c"
+ },
+ "outputs": [],
+ "source": [
+ "log_reg_loaded = tf.saved_model.load(save_path)\n",
+ "test_preds = log_reg_loaded(x_test)\n",
+ "test_preds[:10].numpy()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "vgGQuV-yqYZH"
+ },
+ "source": [
+ "## Conclusion\n",
+ "\n",
+ "This notebook introduced a few techniques to handle a logistic regression problem. Here are a few more tips that may help:\n",
+ "\n",
+ "- The [TensorFlow Core APIs](https://www.tensorflow.org/guide/core) can be used to build machine learning workflows with high levels of configurability\n",
+ "- Analyzing error rates is a great way to gain more insight about a classification model's performance beyond its overall accuracy score.\n",
+ "- Overfitting is another common problem for logistic regression models, though it wasn't a problem for this tutorial. Visit the [Overfit and underfit](../../tutorials/keras/overfit_and_underfit.ipynb) tutorial for more help with this.\n",
+ "\n",
+ "For more examples of using the TensorFlow Core APIs, check out the [guide](https://www.tensorflow.org/guide/core). If you want to learn more about loading and preparing data, see the tutorials on [image data loading](../../tutorials/load_data/images.ipynb) or [CSV data loading](../../tutorials/load_data/csv.ipynb)."
+ ]
+ }
+ ],
+ "metadata": {
+ "colab": {
+ "collapsed_sections": [],
+ "name": "logistic_regression_core.ipynb",
+ "toc_visible": true
+ },
+ "kernelspec": {
+ "display_name": "Python 3",
+ "name": "python3"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}
diff --git a/site/en/guide/core/matrix_core.ipynb b/site/en/guide/core/matrix_core.ipynb
new file mode 100644
index 00000000000..1d7d35ed047
--- /dev/null
+++ b/site/en/guide/core/matrix_core.ipynb
@@ -0,0 +1,731 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "FhGuhbZ6M5tl"
+ },
+ "source": [
+ "##### Copyright 2022 The TensorFlow Authors."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "cellView": "form",
+ "id": "AwOEIRJC6Une"
+ },
+ "outputs": [],
+ "source": [
+ "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+ "# you may not use this file except in compliance with the License.\n",
+ "# You may obtain a copy of the License at\n",
+ "#\n",
+ "# https://www.apache.org/licenses/LICENSE-2.0\n",
+ "#\n",
+ "# Unless required by applicable law or agreed to in writing, software\n",
+ "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+ "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+ "# See the License for the specific language governing permissions and\n",
+ "# limitations under the License."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "EIdT9iu_Z4Rb"
+ },
+ "source": [
+ "# Matrix approximation with Core APIs"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "bBIlTPscrIT9"
+ },
+ "source": [
+ "
"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "qGw8TF2vtzru"
+ },
+ "source": [
+ "## Introduction \n",
+ "\n",
+ "This notebook uses the [TensorFlow Core low-level APIs](https://www.tensorflow.org/guide/core) to showcase TensorFlow's capabilities as a high-performance scientific computing platform. Visit the [Core APIs overview](https://www.tensorflow.org/guide/core) to learn more about TensorFlow Core and its intended use cases.\n",
+ "\n",
+ "This tutorial explores the technique of [singular value decomposition](https://developers.google.com/machine-learning/recommendation/collaborative/matrix) (SVD) and its applications for low-rank approximation problems. The SVD is used to factorize real or complex matrices and has a variety of use cases in data science such as image compression. The images for this tutorial come from Google Brain's [Imagen](https://imagen.research.google/) project. "
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "5_FdwaovEkCC"
+ },
+ "source": [
+ ">"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "nchsZfwEVtVs"
+ },
+ "source": [
+ "## Setup"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "1rRo8oNqZ-Rj"
+ },
+ "outputs": [],
+ "source": [
+ "import matplotlib\n",
+ "from matplotlib.image import imread\n",
+ "from matplotlib import pyplot as plt\n",
+ "import requests\n",
+ "# Preset Matplotlib figure sizes.\n",
+ "matplotlib.rcParams['figure.figsize'] = [16, 9]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "9xQKvCJ85kCQ"
+ },
+ "outputs": [],
+ "source": [
+ "import tensorflow as tf\n",
+ "print(tf.__version__)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "so_ewq3gAoEI"
+ },
+ "source": [
+ "## SVD fundamentals\n",
+ "\n",
+ "The singular value decomposition of a matrix, ${\\mathrm{A}}$, is determined by the following factorization:\n",
+ "\n",
+ "$${\\mathrm{A}} = {\\mathrm{U}} \\Sigma {\\mathrm{V}}^T$$\n",
+ "\n",
+ "where\n",
+ "\n",
+ "* $\\underset{m \\times n}{\\mathrm{A}}$: input matrix where $m \\geq n$\n",
+ "* $\\underset{m \\times n}{\\mathrm{U}}$: orthogonal matrix, ${\\mathrm{U}}^T{\\mathrm{U}} = {\\mathrm{I}}$, with each column, $u_i$, denoting a left singular vector of ${\\mathrm{A}}$\n",
+ "* $\\underset{n \\times n}{\\Sigma}$: diagonal matrix with each diagonal entry, $\\sigma_i$, denoting a singular value of ${\\mathrm{A}}$\n",
+ "* $\\underset{n \\times n}{{\\mathrm{V}}^T}$: orthogonal matrix, ${\\mathrm{V}}^T{\\mathrm{V}} = {\\mathrm{I}}$, with each row, $v_i$, denoting a right singular vector of ${\\mathrm{A}}$\n",
+ "\n",
+ "When $m < n$, ${\\mathrm{U}}$ and $\\Sigma$ both have dimension $(m \\times m)$, and ${\\mathrm{V}}^T$ has dimension $(m \\times n)$."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "enGGGXCQKNv8"
+ },
+ "source": [
+ ">"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "NlP-cBdSKLtc"
+ },
+ "source": [
+ "TensorFlow's linear algebra package has a function, `tf.linalg.svd`, which can be used to compute the singular value decomposition of one or more matrices. Start by defining a simple matrix and computing its SVD factorization.\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "C3QAcgyoeIpv"
+ },
+ "outputs": [],
+ "source": [
+ "A = tf.random.uniform(shape=[40,30])\n",
+ "# Compute the SVD factorization\n",
+ "s, U, V = tf.linalg.svd(A)\n",
+ "# Define Sigma and V Transpose\n",
+ "S = tf.linalg.diag(s)\n",
+ "V_T = tf.transpose(V)\n",
+ "# Reconstruct the original matrix\n",
+ "A_svd = U@S@V_T\n",
+ "# Visualize \n",
+ "plt.bar(range(len(s)), s);\n",
+ "plt.xlabel(\"Singular value rank\")\n",
+ "plt.ylabel(\"Singular value\")\n",
+ "plt.title(\"Bar graph of singular values\");"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "6H_C9WhFACm4"
+ },
+ "source": [
+ "The `tf.einsum` function can be used to directly compute the matrix reconstruction from the outputs of `tf.linalg.svd`."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "TPE6QeMtADUn"
+ },
+ "outputs": [],
+ "source": [
+ "A_svd = tf.einsum('s,us,vs -> uv',s,U,V)\n",
+ "print('\\nReconstructed Matrix, A_svd', A_svd)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "x1m6JIsM9DLP"
+ },
+ "source": [
+ "## Low rank approximation with the SVD\n",
+ "\n",
+ "The rank of a matrix, ${\\mathrm{A}}$, is determined by the dimension of the vector space spanned by its columns. \n",
+ "The SVD can be used to approximate a matrix with a lower rank, which ultimately decreases the dimensionality of data required to store the information represented by the matrix.\n",
+ "\n",
+ "The rank-r approximation of ${\\mathrm{A}}$ in terms of the SVD is defined by the formula:\n",
+ "\n",
+ "$${\\mathrm{A_r}} = {\\mathrm{U_r}} \\Sigma_r {\\mathrm{V_r}}^T$$\n",
+ "\n",
+ "where\n",
+ "\n",
+ "* $\\underset{m \\times r}{\\mathrm{U_r}}$: matrix consisting of the first $r$ columns of ${\\mathrm{U}}$\n",
+ "* $\\underset{r \\times r}{\\Sigma_r}$: diagonal matrix consisting of the first $r$ singular values in $\\Sigma$\n",
+ "* $\\underset{r \\times n}{\\mathrm{V_r}}^T$: matrix consisting of the first $r$ rows of ${\\mathrm{V}}^T$"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "nJWMJu36QyUV"
+ },
+ "source": [
+ ">"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "TkiVUxeaQybq"
+ },
+ "source": [
+ "Start by writing a function to compute the rank-r approximation of a given matrix. This low-rank approximation procedure is used for image compression; therefore, it is also helpful to compute the physical data sizes for each approximation. For simplicity, assume that data size for an rank-r approximated matrix is equal to the total number of elements required to compute the approximation. Next, write a function to visualize the original matrix, $\\mathrm{A}$ its rank-r approximation, $\\mathrm{A}_r$ and the error matrix, $|\\mathrm{A} - \\mathrm{A}_r|$."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "2oY3pMPagJrO"
+ },
+ "outputs": [],
+ "source": [
+ "def rank_r_approx(s, U, V, r, verbose=False):\n",
+ " # Compute the matrices necessary for a rank-r approximation\n",
+ " s_r, U_r, V_r = s[..., :r], U[..., :, :r], V[..., :, :r] # ... implies any number of extra batch axes\n",
+ " # Compute the low-rank approximation and its size\n",
+ " A_r = tf.einsum('...s,...us,...vs->...uv',s_r,U_r,V_r)\n",
+ " A_r_size = tf.size(U_r) + tf.size(s_r) + tf.size(V_r)\n",
+ " if verbose:\n",
+ " print(f\"Approximation Size: {A_r_size}\")\n",
+ " return A_r, A_r_size\n",
+ "\n",
+ "def viz_approx(A, A_r):\n",
+ " # Plot A, A_r, and A - A_r\n",
+ " vmin, vmax = 0, tf.reduce_max(A)\n",
+ " fig, ax = plt.subplots(1,3)\n",
+ " mats = [A, A_r, abs(A - A_r)]\n",
+ " titles = ['Original A', 'Approximated A_r', 'Error |A - A_r|']\n",
+ " for i, (mat, title) in enumerate(zip(mats, titles)):\n",
+ " ax[i].pcolormesh(mat, vmin=vmin, vmax=vmax)\n",
+ " ax[i].set_title(title)\n",
+ " ax[i].axis('off')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "O3ZRkYCkX2FQ"
+ },
+ "outputs": [],
+ "source": [
+ "print(f\"Original Size of A: {tf.size(A)}\")\n",
+ "s, U, V = tf.linalg.svd(A)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "S1DR83VMX4cM"
+ },
+ "outputs": [],
+ "source": [
+ "# Rank-15 approximation\n",
+ "A_15, A_15_size = rank_r_approx(s, U, V, 15, verbose = True)\n",
+ "viz_approx(A, A_15)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "KgFT70XFX57E"
+ },
+ "outputs": [],
+ "source": [
+ "# Rank-3 approximation\n",
+ "A_3, A_3_size = rank_r_approx(s, U, V, 3, verbose = True)\n",
+ "viz_approx(A, A_3)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "DS4XoSlTJgX0"
+ },
+ "source": [
+ "As expected, using lower ranks results in less-accurate approximations. However, the quality of these low-rank approximations are often good enough in real world scenarios. Also note that the main goal of low-rank approximation with SVD \n",
+ "is to reduce the dimensionality of the data but not to reduce the disk space of the data itself. However, as the input matrices become higher-dimensional, many low-rank approximations also end up benefiting from reduced data size. This reduction benefit is why the process is applicable for image compression problems."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "IhsaiOnnZs6M"
+ },
+ "source": [
+ "## Image loading\n",
+ "\n",
+ "The following image is available on the [Imagen](https://imagen.research.google/) home page. Imagen is a text-to-image diffusion model developed by Google Research's Brain team. An AI created this image based on the prompt: \"A photo of a Corgi dog riding a bike in Times Square. It is wearing sunglasses and a beach hat.\" How cool is that! You can also change the url below to any .jpg link to load in a custom image of choice. \n",
+ "\n",
+ "Start by reading in and visualizing the image. After reading a JPEG file, Matplotlib outputs a matrix, ${\\mathrm{I}}$, of shape $(m \\times n \\times 3)$ which represents a 2-dimensional image with 3 color channels for red, green and blue respectively."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "OVsZOQUAZ2C7"
+ },
+ "outputs": [],
+ "source": [
+ "img_link = \"https://imagen.research.google/main_gallery_images/a-photo-of-a-corgi-dog-riding-a-bike-in-times-square.jpg\"\n",
+ "img_path = requests.get(img_link, stream=True).raw\n",
+ "I = imread(img_path, 0)\n",
+ "print(\"Input Image Shape:\", I.shape)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "Qvs7uftcZ54x"
+ },
+ "outputs": [],
+ "source": [
+ "def show_img(I):\n",
+ " # Display the image in matplotlib\n",
+ " img = plt.imshow(I)\n",
+ " plt.axis('off')\n",
+ " return"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "ZbesXO3HZ6Qs"
+ },
+ "outputs": [],
+ "source": [
+ "show_img(I)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "tdnUBVg_JoOa"
+ },
+ "source": [
+ "## The image compression algorithm\n",
+ "\n",
+ "Now, use the SVD to compute low-rank approximations of the sample image. Recall that the image is of shape $(1024 \\times 1024 \\times 3)$ and that the theory SVD only applies for 2-dimensional matrices. This means that the sample image has to be batched into 3 equal-size matrices that correspond to each of the 3 color channels. This can be done so by transposing the matrix to be of shape $(3 \\times 1024 \\times 1024)$. In order to clearly visualize the approximation error, rescale the RGB values of the image from $[0,255]$ to $[0,1]$. Remember to clip the approximated values to fall within this interval before visualizing them. The `tf.clip_by_value` function is useful for this."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "i7DDp0h7oSIk"
+ },
+ "outputs": [],
+ "source": [
+ "def compress_image(I, r, verbose=False):\n",
+ " # Compress an image with the SVD given a rank \n",
+ " I_size = tf.size(I)\n",
+ " print(f\"Original size of image: {I_size}\")\n",
+ " # Compute SVD of image\n",
+ " I = tf.convert_to_tensor(I)/255\n",
+ " I_batched = tf.transpose(I, [2, 0, 1]) # einops.rearrange(I, 'h w c -> c h w')\n",
+ " s, U, V = tf.linalg.svd(I_batched)\n",
+ " # Compute low-rank approximation of image across each RGB channel\n",
+ " I_r, I_r_size = rank_r_approx(s, U, V, r)\n",
+ " I_r = tf.transpose(I_r, [1, 2, 0]) # einops.rearrange(I_r, 'c h w -> h w c')\n",
+ " I_r_prop = (I_r_size / I_size)\n",
+ " if verbose:\n",
+ " # Display compressed image and attributes\n",
+ " print(f\"Number of singular values used in compression: {r}\")\n",
+ " print(f\"Compressed image size: {I_r_size}\")\n",
+ " print(f\"Proportion of original size: {I_r_prop:.3f}\")\n",
+ " ax_1 = plt.subplot(1,2,1)\n",
+ " show_img(tf.clip_by_value(I_r,0.,1.))\n",
+ " ax_1.set_title(\"Approximated image\")\n",
+ " ax_2 = plt.subplot(1,2,2)\n",
+ " show_img(tf.clip_by_value(0.5+abs(I-I_r),0.,1.))\n",
+ " ax_2.set_title(\"Error\")\n",
+ " return I_r, I_r_prop"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "RGQ_rTyKDX9F"
+ },
+ "source": [
+ "Now, compute rank-r approximations for the following ranks : 100, 50, 10"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "7GlKkVLGDjre"
+ },
+ "outputs": [],
+ "source": [
+ "I_100, I_100_prop = compress_image(I, 100, verbose=True)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "XdvUkF5_E75D"
+ },
+ "outputs": [],
+ "source": [
+ "I_50, I_50_prop = compress_image(I, 50, verbose=True)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "MsCNZ8416Sbk"
+ },
+ "outputs": [],
+ "source": [
+ "I_10, I_10_prop = compress_image(I, 10, verbose=True)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "RfYYBhcuNkvH"
+ },
+ "source": [
+ "## Evaluating approximations\n",
+ "\n",
+ "There are a variety of interesting methods to measure the effectiveness and have more control over matrix approximations."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "D2Lotde9Zg7v"
+ },
+ "source": [
+ "### Compression factor vs rank\n",
+ "\n",
+ "For each of the above approximations, observe how the data sizes change with the rank."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "O1ariNQe6Wbl"
+ },
+ "outputs": [],
+ "source": [
+ "plt.figure(figsize=(11,6))\n",
+ "plt.plot([100, 50, 10], [I_100_prop, I_50_prop, I_10_prop])\n",
+ "plt.xlabel(\"Rank\")\n",
+ "plt.ylabel(\"Proportion of original image size\")\n",
+ "plt.title(\"Compression factor vs rank\");"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "dvHcLRj2QoDg"
+ },
+ "source": [
+ "Based on this plot, there is a linear relationship between an approximated image's compression factor and its rank. To explore this further, recall that the data size of an approximated matrix, ${\\mathrm{A}}_r$, is defined as the total number of elements required for its computation. The following equations can be used to find the relationship between compression factor and rank:\n",
+ "\n",
+ "$$x = (m \\times r) + r + (r \\times n) = r \\times (m + n + 1)$$\n",
+ "\n",
+ "$$c = \\large \\frac{x}{y} = \\frac{r \\times (m + n + 1)}{m \\times n}$$\n",
+ "\n",
+ "where\n",
+ "\n",
+ "* $x$: size of ${\\mathrm{A_r}}$\n",
+ "* $y$: size of ${\\mathrm{A}}$\n",
+ "* $c = \\frac{x}{y}$: compression factor\n",
+ "* $r$: rank of the approximation\n",
+ "* $m$ and $n$: row and column dimensions of ${\\mathrm{A}}$\n",
+ "\n",
+ "In order to find the rank, $r$, that is necessary to compress an image to a desired factor, $c$, the above equation can be rearranged to solve for $r$:\n",
+ "\n",
+ "$$r = ⌊{\\large\\frac{c \\times m \\times n}{m + n + 1}}⌋$$\n",
+ "\n",
+ "Note that this formula is independent of the color channel dimension since each of the RGB approximations do not affect each other. Now, write a function to compress an input image given a desired compression factor."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "viVO-I60QynI"
+ },
+ "outputs": [],
+ "source": [
+ "def compress_image_with_factor(I, compression_factor, verbose=False):\n",
+ " # Returns a compressed image based on a desired compression factor\n",
+ " m,n,o = I.shape\n",
+ " r = int((compression_factor * m * n)/(m + n + 1))\n",
+ " I_r, I_r_prop = compress_image(I, r, verbose=verbose)\n",
+ " return I_r"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "gWSv58J6LSRQ"
+ },
+ "source": [
+ "Compress an image to 15% of its original size."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "HVeeloIwQ1b6"
+ },
+ "outputs": [],
+ "source": [
+ "compression_factor = 0.15\n",
+ "I_r_img = compress_image_with_factor(I, compression_factor, verbose=True)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "LkeRyms7jZMd"
+ },
+ "source": [
+ "### Cumulative sum of singular values\n",
+ "\n",
+ "The cumulative sum of singular values can be a useful indicator for the amount of energy captured by a rank-r approximation. Visualize the RGB-averaged cumulative proportion of singular values in the sample image. The `tf.cumsum` function can be useful for this."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "CteJ6VbKlndu"
+ },
+ "outputs": [],
+ "source": [
+ "def viz_energy(I):\n",
+ " # Visualize the energy captured based on rank\n",
+ " # Computing SVD\n",
+ " I = tf.convert_to_tensor(I)/255\n",
+ " I_batched = tf.transpose(I, [2, 0, 1]) \n",
+ " s, U, V = tf.linalg.svd(I_batched)\n",
+ " # Plotting average proportion across RGB channels \n",
+ " props_rgb = tf.map_fn(lambda x: tf.cumsum(x)/tf.reduce_sum(x), s)\n",
+ " props_rgb_mean = tf.reduce_mean(props_rgb, axis=0)\n",
+ " plt.figure(figsize=(11,6))\n",
+ " plt.plot(range(len(I)), props_rgb_mean, color='k')\n",
+ " plt.xlabel(\"Rank / singular value number\")\n",
+ " plt.ylabel(\"Cumulative proportion of singular values\")\n",
+ " plt.title(\"RGB-averaged proportion of energy captured by the first 'r' singular values\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "Vl9PKow-GgCp"
+ },
+ "outputs": [],
+ "source": [
+ "viz_energy(I)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "vQtwimKuQP19"
+ },
+ "source": [
+ "It looks like over 90% of the energy in this image is captured within the first 100 singular values. Now, write a function to compress an input image given a desired energy retention factor."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "fum5Cvm7R5vH"
+ },
+ "outputs": [],
+ "source": [
+ "def compress_image_with_energy(I, energy_factor, verbose=False):\n",
+ " # Returns a compressed image based on a desired energy factor\n",
+ " # Computing SVD\n",
+ " I_rescaled = tf.convert_to_tensor(I)/255\n",
+ " I_batched = tf.transpose(I_rescaled, [2, 0, 1]) \n",
+ " s, U, V = tf.linalg.svd(I_batched)\n",
+ " # Extracting singular values\n",
+ " props_rgb = tf.map_fn(lambda x: tf.cumsum(x)/tf.reduce_sum(x), s)\n",
+ " props_rgb_mean = tf.reduce_mean(props_rgb, axis=0)\n",
+ " # Find closest r that corresponds to the energy factor\n",
+ " r = tf.argmin(tf.abs(props_rgb_mean - energy_factor)) + 1\n",
+ " actual_ef = props_rgb_mean[r]\n",
+ " I_r, I_r_prop = compress_image(I, r, verbose=verbose)\n",
+ " print(f\"Proportion of energy captured by the first {r} singular values: {actual_ef:.3f}\")\n",
+ " return I_r"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "Y_rChG0OLby1"
+ },
+ "source": [
+ "Compress an image to retain 75% of its energy."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "xDXBaZQ4c5jF"
+ },
+ "outputs": [],
+ "source": [
+ "energy_factor = 0.75\n",
+ "I_r_img = compress_image_with_energy(I, energy_factor, verbose=True)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "2tmqTW0CYX-v"
+ },
+ "source": [
+ "### Error and singular values\n",
+ "\n",
+ "There is also an interesting relationship between the approximation error and the singular values. It turns out that the squared Frobenius norm of the approximation is equal to the sum of the squares of its singular values that were left out:\n",
+ "\n",
+ "$${||A - A_r||}^2 = \\sum_{i=r+1}^{R}σ_i^2$$\n",
+ "\n",
+ "Test out this relationship with a rank-10 approximation of the example matrix in the beginning of this tutorial."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "hctOvN8BckiS"
+ },
+ "outputs": [],
+ "source": [
+ "s, U, V = tf.linalg.svd(A)\n",
+ "A_10, A_10_size = rank_r_approx(s, U, V, 10)\n",
+ "squared_norm = tf.norm(A - A_10)**2\n",
+ "s_squared_sum = tf.reduce_sum(s[10:]**2)\n",
+ "print(f\"Squared Frobenius norm: {squared_norm:.3f}\")\n",
+ "print(f\"Sum of squared singular values left out: {s_squared_sum:.3f}\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "vgGQuV-yqYZH"
+ },
+ "source": [
+ "## Conclusion\n",
+ "\n",
+ "This notebook introduced the process of implementing the singular value decomposition with TensorFlow and applying it to write an image compression algorithm. Here are a few more tips that may help:\n",
+ "\n",
+ "* The [TensorFlow Core APIs](https://www.tensorflow.org/guide/core) can be utilized for a variety of high-performance scientific computing use cases.\n",
+ "* To learn more about TensorFlow's linear algebra functionalities, visit the docs for the [linalg module](https://www.tensorflow.org/api_docs/python/tf/linalg).\n",
+ "* The SVD can also be applied to build [recommendation systems](https://developers.google.com/machine-learning/recommendation/labs/movie-rec-programming-exercise).\n",
+ "\n",
+ "\n",
+ "For more examples of using the TensorFlow Core APIs, check out the [guide](https://www.tensorflow.org/guide/core). If you want learn more about loading and preparing data, see the tutorials on [image data loading](https://www.tensorflow.org/tutorials/load_data/images) or [CSV data loading](https://www.tensorflow.org/tutorials/load_data/csv)."
+ ]
+ }
+ ],
+ "metadata": {
+ "colab": {
+ "collapsed_sections": [],
+ "name": "matrix_core.ipynb",
+ "toc_visible": true
+ },
+ "kernelspec": {
+ "display_name": "Python 3",
+ "name": "python3"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}
diff --git a/site/en/guide/core/mlp_core.ipynb b/site/en/guide/core/mlp_core.ipynb
new file mode 100644
index 00000000000..a5975c20c6e
--- /dev/null
+++ b/site/en/guide/core/mlp_core.ipynb
@@ -0,0 +1,964 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "FhGuhbZ6M5tl"
+ },
+ "source": [
+ "##### Copyright 2022 The TensorFlow Authors."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "cellView": "form",
+ "id": "AwOEIRJC6Une"
+ },
+ "outputs": [],
+ "source": [
+ "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+ "# you may not use this file except in compliance with the License.\n",
+ "# You may obtain a copy of the License at\n",
+ "#\n",
+ "# https://www.apache.org/licenses/LICENSE-2.0\n",
+ "#\n",
+ "# Unless required by applicable law or agreed to in writing, software\n",
+ "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+ "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+ "# See the License for the specific language governing permissions and\n",
+ "# limitations under the License."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "EIdT9iu_Z4Rb"
+ },
+ "source": [
+ "# Multilayer perceptrons for digit recognition with Core APIs"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "bBIlTPscrIT9"
+ },
+ "source": [
+ "
"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "SjAxxRpBzVYg"
+ },
+ "source": [
+ "## Introduction\n",
+ "\n",
+ "This notebook introduces the process of creating custom optimizers with the [TensorFlow Core low-level APIs](https://www.tensorflow.org/guide/core). Visit the [Core APIs overview](https://www.tensorflow.org/guide/core) to learn more about TensorFlow Core and its intended use cases. \n",
+ "\n",
+ "The [Keras optimizers](https://www.tensorflow.org/api_docs/python/tf/keras/optimizers) module is the recommended optimization toolkit for many general training purposes. It includes a variety of prebuilt optimiziers as well as subclassing functionality for customization. The Keras optimizers are also compatible with custom layers, models, and training loops built with the Core APIs. These prebuilt and customizable optimizers are suitable for most cases, but the Core APIs allow for complete control over the optimization process. For example, techniques such as Sharpness-Aware Minimization (SAM) require the model and optimizer to be coupled, which does not fit the traditional definition of ML optimizers. This guide walks through the process of building custom optimizers from scratch with the Core APIs, giving you the power to have full control over the structure, implementation, and behavior of your optimizers."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "nBmqYyodNRd_"
+ },
+ "source": [
+ "## Optimizers overview\n",
+ "\n",
+ "An optimizer is an algorithm used to minimize a loss function with respect to a model's trainable parameters. The most straightforward optimization technique is gradient descent, which iteratively updates a model's parameters by taking a step in the direction of its loss function's steepest descent. Its step size is directly proportional to the size of the gradient, which can be problematic when the gradient is either too large or too small. There are many other gradient-based optimizers such as Adam, Adagrad, and RMSprop that leverage various mathematical properties of gradients for memory efficiency and fast convergence."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "nchsZfwEVtVs"
+ },
+ "source": [
+ "## Setup"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "d9idwpXCltUl"
+ },
+ "outputs": [],
+ "source": [
+ "import matplotlib\n",
+ "from matplotlib import pyplot as plt\n",
+ "# Preset Matplotlib figure sizes.\n",
+ "matplotlib.rcParams['figure.figsize'] = [9, 6]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "9xQKvCJ85kCQ"
+ },
+ "outputs": [],
+ "source": [
+ "import tensorflow as tf\n",
+ "print(tf.__version__)\n",
+ "# set random seed for reproducible results \n",
+ "tf.random.set_seed(22)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "0UmF5aU3MnwX"
+ },
+ "source": [
+ "## Gradient descent\n",
+ "\n",
+ "The basic optimizer class should have an initialization method and a function to update a list of variables given a list of gradients. Start by implementing the basic gradient descent optimizer which updates each variable by subtracting its gradient scaled by a learning rate."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "MWjmUmeOQFFN"
+ },
+ "outputs": [],
+ "source": [
+ "class GradientDescent(tf.Module):\n",
+ "\n",
+ " def __init__(self, learning_rate=1e-3):\n",
+ " # Initialize parameters\n",
+ " self.learning_rate = learning_rate\n",
+ " self.title = f\"Gradient descent optimizer: learning rate={self.learning_rate}\"\n",
+ "\n",
+ " def apply_gradients(self, grads, vars):\n",
+ " # Update variables\n",
+ " for grad, var in zip(grads, vars):\n",
+ " var.assign_sub(self.learning_rate*grad)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "ZSekgBHDRzmp"
+ },
+ "source": [
+ "To test this optimizer, create a sample loss function to minimize with respect to a single variable, $x$. Compute its gradient function and solve for its minimizing parameter value:\n",
+ "\n",
+ "$$L = 2x^4 + 3x^3 + 2$$\n",
+ "\n",
+ "$$\\frac{dL}{dx} = 8x^3 + 9x^2$$\n",
+ "\n",
+ "$\\frac{dL}{dx}$ is 0 at $x = 0$, which is a saddle point and at $x = - \\frac{9}{8}$, which is the global minimum. Therefore, the loss function is optimized at $x^\\star = - \\frac{9}{8}$."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "VCtJaUo6Ry8V"
+ },
+ "outputs": [],
+ "source": [
+ "x_vals = tf.linspace(-2, 2, 201)\n",
+ "x_vals = tf.cast(x_vals, tf.float32)\n",
+ "\n",
+ "def loss(x):\n",
+ " return 2*(x**4) + 3*(x**3) + 2\n",
+ "\n",
+ "def grad(f, x):\n",
+ " with tf.GradientTape() as tape:\n",
+ " tape.watch(x)\n",
+ " result = f(x)\n",
+ " return tape.gradient(result, x)\n",
+ "\n",
+ "plt.plot(x_vals, loss(x_vals), c='k', label = \"Loss function\")\n",
+ "plt.plot(x_vals, grad(loss, x_vals), c='tab:blue', label = \"Gradient function\")\n",
+ "plt.plot(0, loss(0), marker=\"o\", c='g', label = \"Inflection point\")\n",
+ "plt.plot(-9/8, loss(-9/8), marker=\"o\", c='r', label = \"Global minimum\")\n",
+ "plt.legend()\n",
+ "plt.ylim(0,5)\n",
+ "plt.xlabel(\"x\")\n",
+ "plt.ylabel(\"loss\")\n",
+ "plt.title(\"Sample loss function and gradient\");"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "fLlIBJ9yuwhE"
+ },
+ "source": [
+ "Write a function to test the convergence of an optimizer with a single variable loss function. Assume that convergence has been achieved when the updated parameter's value at timestep $t$ is the same as its value held at timestep $t-1$. Terminate the test after a set number of iterations and also keep track of any exploding gradients during the process. In order to truly challenge the optimization algorithm, initialize the parameter poorly. In the above example, $x = 2$ is a good choice since it involves an steep gradient and also leads into an inflection point."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "SLQTc41ouv0F"
+ },
+ "outputs": [],
+ "source": [
+ "def convergence_test(optimizer, loss_fn, grad_fn=grad, init_val=2., max_iters=2000):\n",
+ " # Function for optimizer convergence test\n",
+ " print(optimizer.title)\n",
+ " print(\"-------------------------------\")\n",
+ " # Initializing variables and structures\n",
+ " x_star = tf.Variable(init_val)\n",
+ " param_path = []\n",
+ " converged = False\n",
+ "\n",
+ " for iter in range(1, max_iters + 1):\n",
+ " x_grad = grad_fn(loss_fn, x_star)\n",
+ "\n",
+ " # Case for exploding gradient\n",
+ " if tf.math.is_nan(x_grad):\n",
+ " print(f\"Gradient exploded at iteration {iter}\\n\")\n",
+ " return []\n",
+ "\n",
+ " # Updating the variable and storing its old-version\n",
+ " x_old = x_star.numpy()\n",
+ " optimizer.apply_gradients([x_grad], [x_star])\n",
+ " param_path.append(x_star.numpy())\n",
+ "\n",
+ " # Checking for convergence\n",
+ " if x_star == x_old:\n",
+ " print(f\"Converged in {iter} iterations\\n\")\n",
+ " converged = True\n",
+ " break\n",
+ " \n",
+ " # Print early termination message\n",
+ " if not converged:\n",
+ " print(f\"Exceeded maximum of {max_iters} iterations. Test terminated.\\n\")\n",
+ " return param_path"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "vK-7_TsmyAgI"
+ },
+ "source": [
+ "Test the convergence of the gradient descent optimizer for the following learning rates: 1e-3, 1e-2, 1e-1"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "lWRn8c91mqB0"
+ },
+ "outputs": [],
+ "source": [
+ "param_map_gd = {}\n",
+ "learning_rates = [1e-3, 1e-2, 1e-1]\n",
+ "for learning_rate in learning_rates:\n",
+ " param_map_gd[learning_rate] = (convergence_test(\n",
+ " GradientDescent(learning_rate=learning_rate), loss_fn=loss))"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "TydrGHF5y6iI"
+ },
+ "source": [
+ "Visualize the path of the parameters over a contour plot of the loss function."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "piffzGHI_u5G"
+ },
+ "outputs": [],
+ "source": [
+ "def viz_paths(param_map, x_vals, loss_fn, title, max_iters=2000):\n",
+ " # Creating a controur plot of the loss function\n",
+ " t_vals = tf.range(1., max_iters + 100.)\n",
+ " t_grid, x_grid = tf.meshgrid(t_vals, x_vals)\n",
+ " loss_grid = tf.math.log(loss_fn(x_grid))\n",
+ " plt.pcolormesh(t_vals, x_vals, loss_grid, vmin=0, shading='nearest')\n",
+ " colors = ['r', 'w', 'c']\n",
+ " # Plotting the parameter paths over the contour plot\n",
+ " for i, learning_rate in enumerate(param_map):\n",
+ " param_path = param_map[learning_rate]\n",
+ " if len(param_path) > 0:\n",
+ " x_star = param_path[-1]\n",
+ " plt.plot(t_vals[:len(param_path)], param_path, c=colors[i])\n",
+ " plt.plot(len(param_path), x_star, marker='o', c=colors[i], \n",
+ " label = f\"x*: learning rate={learning_rate}\")\n",
+ " plt.xlabel(\"Iterations\")\n",
+ " plt.ylabel(\"Parameter value\")\n",
+ " plt.legend()\n",
+ " plt.title(f\"{title} parameter paths\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "Ssyj2sO4BcNY"
+ },
+ "outputs": [],
+ "source": [
+ "viz_paths(param_map_gd, x_vals, loss, \"Gradient descent\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "MmM-5eDLFnmC"
+ },
+ "source": [
+ "Gradient descent seems to get stuck at the inflection point when using smaller learning rates. Increasing the learning rate can encourage faster movement around the plateau region due to a larger step size; however, this comes at the risk of having exploding gradients in early iterations when the loss function is extremely steep."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "m5CDeXN8S1SF"
+ },
+ "source": [
+ "## Gradient descent with momentum\n",
+ "\n",
+ "Gradient descent with momentum not only uses the gradient to update a variable but also involves the change in position of a variable based on its previous update. The momentum parameter determines the level of influence the update at timestep $t-1$ has on the update at timestep $t$. Accumulating momentum helps to move variables past plataeu regions faster than basic gradient descent. The momentum update rule is as follows:\n",
+ "\n",
+ "$$\\Delta_x^{[t]} = lr \\cdot L^\\prime(x^{[t-1]}) + p \\cdot \\Delta_x^{[t-1]}$$\n",
+ "\n",
+ "$$x^{[t]} = x^{[t-1]} - \\Delta_x^{[t]}$$\n",
+ "\n",
+ "where\n",
+ "\n",
+ "* $x$: the variable being optimized\n",
+ "* $\\Delta_x$: change in $x$ \n",
+ "* $lr$: learning rate\n",
+ "* $L^\\prime(x)$: gradient of the loss function with respect to x\n",
+ "* $p$: momentum parameter"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "rOBY8Tz4S0dX"
+ },
+ "outputs": [],
+ "source": [
+ "class Momentum(tf.Module):\n",
+ "\n",
+ " def __init__(self, learning_rate=1e-3, momentum=0.7):\n",
+ " # Initialize parameters\n",
+ " self.learning_rate = learning_rate\n",
+ " self.momentum = momentum\n",
+ " self.change = 0.\n",
+ " self.title = f\"Gradient descent optimizer: learning rate={self.learning_rate}\"\n",
+ "\n",
+ " def apply_gradients(self, grads, vars):\n",
+ " # Update variables \n",
+ " for grad, var in zip(grads, vars):\n",
+ " curr_change = self.learning_rate*grad + self.momentum*self.change\n",
+ " var.assign_sub(curr_change)\n",
+ " self.change = curr_change"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "t_nDu38gW6Fu"
+ },
+ "source": [
+ "Test the convergence of the momentum optimizer for the following learning rates: 1e-3, 1e-2, 1e-1"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "tA6oQL-sW2xg"
+ },
+ "outputs": [],
+ "source": [
+ "param_map_mtm = {}\n",
+ "learning_rates = [1e-3, 1e-2, 1e-1]\n",
+ "for learning_rate in learning_rates:\n",
+ " param_map_mtm[learning_rate] = (convergence_test(\n",
+ " Momentum(learning_rate=learning_rate),\n",
+ " loss_fn=loss, grad_fn=grad))"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "wz_LV0EPYE6k"
+ },
+ "source": [
+ "Visualize the path of the parameters over a contour plot of the loss function."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "qbW1eEKaX3T9"
+ },
+ "outputs": [],
+ "source": [
+ "viz_paths(param_map_mtm, x_vals, loss, \"Momentum\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "4bEFnhPRTBXh"
+ },
+ "source": [
+ "## Adaptive moment estimation (Adam)\n",
+ "\n",
+ "The Adaptive Moment Estimation (Adam) algorithm is an efficient and highly generalizable optimization technique that leverages two key gradient descent methedologies: momentum, and root mean square propogation (RMSP). Momentum helps accelerate gradient descent by using the first moment (sum of gradients) along with a decay parameter. RMSP is similar; however, it leverages the second moment (sum of gradients squared). \n",
+ "\n",
+ "The Adam algorithm combines both the first and second moment to provide a more generalizable update rule. The sign of a variable, $x$, can be determined by computing $\\frac{x}{\\sqrt{x^2}}$. The Adam optimizer uses this fact to calculate an update step which is effectively a smoothed sign. Instead of calculating $\\frac{x}{\\sqrt{x^2}}$, the optimizer calculates a smoothed version of $x$ (first moment) and $x^2$ (second moment) for each variable update. \n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "WjgyqRiZ7XhA"
+ },
+ "source": [
+ "**Adam algorithm**\n",
+ "\n",
+ "$\\beta_1 \\gets 0.9 \\; \\triangleright \\text{literature value}$\n",
+ "\n",
+ "$\\beta_2 \\gets 0.999 \\; \\triangleright \\text{literature value}$\n",
+ "\n",
+ "$lr \\gets \\text{1e-3} \\; \\triangleright \\text{configurable learning rate}$\n",
+ "\n",
+ "$\\epsilon \\gets \\text{1e-7} \\; \\triangleright \\text{prevents divide by 0 error}$\n",
+ "\n",
+ "$V_{dv} \\gets \\vec {\\underset{n\\times1}{0}} \\;\\triangleright \\text{stores momentum updates for each variable}$\n",
+ "\n",
+ "$S_{dv} \\gets \\vec {\\underset{n\\times1}{0}} \\; \\triangleright \\text{stores RMSP updates for each variable}$\n",
+ "\n",
+ "$t \\gets 1$\n",
+ "\n",
+ "$\\text{On iteration } t:$\n",
+ "\n",
+ "$\\;\\;\\;\\; \\text{For} (\\frac{dL}{dv}, v) \\text{ in gradient variable pairs}:$\n",
+ "\n",
+ "$\\;\\;\\;\\;\\;\\;\\;\\; V_{dv\\_i} = \\beta_1V_{dv\\_i} + (1 - \\beta_1)\\frac{dL}{dv} \\; \\triangleright \\text{momentum update}$\n",
+ "\n",
+ "$\\;\\;\\;\\;\\;\\;\\;\\; S_{dv\\_i} = \\beta_2V_{dv\\_i} + (1 - \\beta_2)(\\frac{dL}{dv})^2 \\; \\triangleright \\text{RMSP update}$\n",
+ "\n",
+ "$\\;\\;\\;\\;\\;\\;\\;\\; v_{dv}^{bc} = \\frac{V_{dv\\_i}}{(1-\\beta_1)^t} \\; \\triangleright \\text{momentum bias correction}$\n",
+ "\n",
+ "$\\;\\;\\;\\;\\;\\;\\;\\; s_{dv}^{bc} = \\frac{S_{dv\\_i}}{(1-\\beta_2)^t} \\; \\triangleright \\text{RMSP bias correction}$\n",
+ "\n",
+ "$\\;\\;\\;\\;\\;\\;\\;\\; v = v - lr\\frac{v_{dv}^{bc}}{\\sqrt{s_{dv}^{bc}} + \\epsilon} \\; \\triangleright \\text{parameter update}$\n",
+ "\n",
+ "$\\;\\;\\;\\;\\;\\;\\;\\; t = t + 1$\n",
+ "\n",
+ "**End of algorithm**\n",
+ "\n",
+ "Given that $V_{dv}$ and $S_{dv}$ are initialized to 0 and that $\\beta_1$ and $\\beta_2$ are close to 1, the momentum and RMSP updates are naturally biased towards 0; therefore, the variables can benefit from bias correction. Bias correction also helps to control the osccilation of weights as they approach the global minimum."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "hm5vffRJRsEc"
+ },
+ "outputs": [],
+ "source": [
+ "class Adam(tf.Module):\n",
+ " \n",
+ " def __init__(self, learning_rate=1e-3, beta_1=0.9, beta_2=0.999, ep=1e-7):\n",
+ " # Initialize the Adam parameters\n",
+ " self.beta_1 = beta_1\n",
+ " self.beta_2 = beta_2\n",
+ " self.learning_rate = learning_rate\n",
+ " self.ep = ep\n",
+ " self.t = 1.\n",
+ " self.v_dvar, self.s_dvar = [], []\n",
+ " self.title = f\"Adam: learning rate={self.learning_rate}\"\n",
+ " self.built = False\n",
+ "\n",
+ " def apply_gradients(self, grads, vars):\n",
+ " # Set up moment and RMSprop slots for each variable on the first call\n",
+ " if not self.built:\n",
+ " for var in vars:\n",
+ " v = tf.Variable(tf.zeros(shape=var.shape))\n",
+ " s = tf.Variable(tf.zeros(shape=var.shape))\n",
+ " self.v_dvar.append(v)\n",
+ " self.s_dvar.append(s)\n",
+ " self.built = True\n",
+ " # Perform Adam updates\n",
+ " for i, (d_var, var) in enumerate(zip(grads, vars)):\n",
+ " # Moment calculation\n",
+ " self.v_dvar[i] = self.beta_1*self.v_dvar[i] + (1-self.beta_1)*d_var\n",
+ " # RMSprop calculation\n",
+ " self.s_dvar[i] = self.beta_2*self.s_dvar[i] + (1-self.beta_2)*tf.square(d_var)\n",
+ " # Bias correction\n",
+ " v_dvar_bc = self.v_dvar[i]/(1-(self.beta_1**self.t))\n",
+ " s_dvar_bc = self.s_dvar[i]/(1-(self.beta_2**self.t))\n",
+ " # Update model variables\n",
+ " var.assign_sub(self.learning_rate*(v_dvar_bc/(tf.sqrt(s_dvar_bc) + self.ep)))\n",
+ " # Increment the iteration counter\n",
+ " self.t += 1."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "UWN4Qus7flUO"
+ },
+ "source": [
+ "Test the performance of the Adam optimizer with the same learning rates used with the gradient descent examples. "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "GXHCxtemFBpR"
+ },
+ "outputs": [],
+ "source": [
+ "param_map_adam = {}\n",
+ "learning_rates = [1e-3, 1e-2, 1e-1]\n",
+ "for learning_rate in learning_rates:\n",
+ " param_map_adam[learning_rate] = (convergence_test(\n",
+ " Adam(learning_rate=learning_rate), loss_fn=loss))"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "jgpUcs_xXEjX"
+ },
+ "source": [
+ "Visualize the path of the parameters over a contour plot of the loss function."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "ctvOUmlzFK8s"
+ },
+ "outputs": [],
+ "source": [
+ "viz_paths(param_map_adam, x_vals, loss, \"Adam\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "_oGScF8zJcY4"
+ },
+ "source": [
+ "In this particular example, the Adam optimizer has slower convergence compared to traditional gradient descent when using small learning rates. However, the algorithm successfully moves past the plataeu region and converges to the global minimum when a larger learning rate. Exploding gradients are no longer an issue due to Adam's dynamic scaling of learning rates when encountering large gradients."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "VFLfEH4ManbW"
+ },
+ "source": [
+ "## Conclusion\n",
+ "\n",
+ "This notebook introduced the basics of writing and comparing optimizers with the [TensorFlow Core APIs](https://www.tensorflow.org/guide/core). Although prebuilt optimizers like Adam are generalizable, they may not always be the best choice for every model or dataset. Having fine-grained control over the optimization process can help streamline ML training workflows and improve overall performance. Refer to the following documentation for more examples of custom optimizers:\n",
+ "\n",
+ "* This Adam optimizer is used in the [Multilayer perceptrons](https://www.tensorflow.org/guide/core/mlp_core) tutorial and the [Distributed training]()\n",
+ "* [Model Garden](https://blog.tensorflow.org/2020/03/introducing-model-garden-for-tensorflow-2.html) has a variety of [custom optimizers](https://github.com/tensorflow/models/tree/master/official/modeling/optimization) written with the Core APIs.\n"
+ ]
+ }
+ ],
+ "metadata": {
+ "colab": {
+ "collapsed_sections": [],
+ "name": "optimizers_core.ipynb",
+ "toc_visible": true
+ },
+ "kernelspec": {
+ "display_name": "Python 3",
+ "name": "python3"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}
diff --git a/site/en/guide/core/quickstart_core.ipynb b/site/en/guide/core/quickstart_core.ipynb
new file mode 100644
index 00000000000..70586fd3f0c
--- /dev/null
+++ b/site/en/guide/core/quickstart_core.ipynb
@@ -0,0 +1,591 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "rX8mhOLljYeM"
+ },
+ "source": [
+ "##### Copyright 2022 The TensorFlow Authors."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "cellView": "form",
+ "id": "BZSlp3DAjdYf"
+ },
+ "outputs": [],
+ "source": [
+ "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+ "# you may not use this file except in compliance with the License.\n",
+ "# You may obtain a copy of the License at\n",
+ "#\n",
+ "# https://www.apache.org/licenses/LICENSE-2.0\n",
+ "#\n",
+ "# Unless required by applicable law or agreed to in writing, software\n",
+ "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+ "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+ "# See the License for the specific language governing permissions and\n",
+ "# limitations under the License."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "3wF5wszaj97Y"
+ },
+ "source": [
+ "# Quickstart for the TensorFlow Core APIs"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "DUNzJc4jTj6G"
+ },
+ "source": [
+ "
"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "04QgGZc9bF5D"
+ },
+ "source": [
+ "This quickstart tutorial demonstrates how you can use the [TensorFlow Core low-level APIs](https://www.tensorflow.org/guide/core) to build and train a multiple linear regression model that predicts fuel efficiency. It uses the [Auto MPG](https://archive.ics.uci.edu/ml/datasets/auto+mpg) dataset which contains fuel efficiency data for late-1970s and early 1980s automobiles.\n",
+ "\n",
+ "You will follow the typical stages of a machine learning process:\n",
+ "\n",
+ "1. Load the dataset.\n",
+ "2. Build an [input pipeline](../data.ipynb).\n",
+ "3. Build a multiple [linear regression](https://developers.google.com/machine-learning/glossary#linear-regression) model.\n",
+ "4. Evaluate the performance of the model."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "nnrWf3PCEzXL"
+ },
+ "source": [
+ "## Setup\n",
+ "\n",
+ "Import TensorFlow and other necessary libraries to get started:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "0trJmd6DjqBZ"
+ },
+ "outputs": [],
+ "source": [
+ "import tensorflow as tf\n",
+ "import pandas as pd\n",
+ "import matplotlib\n",
+ "from matplotlib import pyplot as plt\n",
+ "print(\"TensorFlow version:\", tf.__version__)\n",
+ "# Set a random seed for reproducible results \n",
+ "tf.random.set_seed(22)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "7NAbSZiaoJ4z"
+ },
+ "source": [
+ "## Load and preprocess the dataset\n",
+ "\n",
+ "Next, you need to load and preprocess the [Auto MPG dataset](https://archive.ics.uci.edu/ml/datasets/auto+mpg) from the [UCI Machine Learning Repository](https://archive.ics.uci.edu/ml/). This dataset uses a variety of quantitative and categorical features such as cylinders, displacement, horsepower and weight to predict the fuel efficiencies of automobiles in the late-1970s and early 1980s.\n",
+ "\n",
+ "The dataset contains a few unknown values. Make sure to drop any missing values with `pandas.DataFrame.dropna`, and convert the dataset to a `tf.float32` tensor type with the `tf.convert_to_tensor` and `tf.cast` functions."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "HglhDsUfrJ98"
+ },
+ "outputs": [],
+ "source": [
+ "url = 'http://archive.ics.uci.edu/ml/machine-learning-databases/auto-mpg/auto-mpg.data'\n",
+ "column_names = ['MPG', 'Cylinders', 'Displacement', 'Horsepower', 'Weight',\n",
+ " 'Acceleration', 'Model Year', 'Origin']\n",
+ "\n",
+ "dataset = pd.read_csv(url, names=column_names, na_values='?', comment='\\t',\n",
+ " sep=' ', skipinitialspace=True)\n",
+ "\n",
+ "dataset = dataset.dropna()\n",
+ "dataset_tf = tf.convert_to_tensor(dataset, dtype=tf.float32)\n",
+ "dataset.tail()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "0vgoDL3hYesB"
+ },
+ "source": [
+ "Next, split the dataset into training and test sets. Make sure to shuffle the dataset with `tf.random.shuffle` to avoid biased splits."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "0mJU4kt6YiAp"
+ },
+ "outputs": [],
+ "source": [
+ "dataset_shuffled = tf.random.shuffle(dataset_tf, seed=22)\n",
+ "train_data, test_data = dataset_shuffled[100:], dataset_shuffled[:100]\n",
+ "x_train, y_train = train_data[:, 1:], train_data[:, 0]\n",
+ "x_test, y_test = test_data[:, 1:], test_data[:, 0]"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "Bscb2Vsbi3TE"
+ },
+ "source": [
+ "Perform basic feature engineering by one-hot-encoding the `\"Origin\"` feature. The `tf.one_hot` function is useful for transforming this categorical column into 3 separate binary columns."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "_B8N9IV1i6IV"
+ },
+ "outputs": [],
+ "source": [
+ "def onehot_origin(x):\n",
+ " origin = tf.cast(x[:, -1], tf.int32)\n",
+ " # Use `origin - 1` to account for 1-indexed feature\n",
+ " origin_oh = tf.one_hot(origin - 1, 3)\n",
+ " x_ohe = tf.concat([x[:, :-1], origin_oh], axis = 1)\n",
+ " return x_ohe\n",
+ "\n",
+ "x_train_ohe, x_test_ohe = onehot_origin(x_train), onehot_origin(x_test)\n",
+ "x_train_ohe.numpy()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "qnoCDzzedite"
+ },
+ "source": [
+ "This example shows a multiple regression problem with predictors or features on vastly different scales. Therefore, it is beneficial to standardize the data so that each feature has zero mean and unit variance. Use the `tf.reduce_mean` and `tf.math.reduce_std` functions for standardization. The regression model's prediction can then be unstandardized to obtain its value in terms of the original units."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "dJJFdvqydhyp"
+ },
+ "outputs": [],
+ "source": [
+ "class Normalize(tf.Module):\n",
+ " def __init__(self, x):\n",
+ " # Initialize the mean and standard deviation for normalization\n",
+ " self.mean = tf.math.reduce_mean(x, axis=0)\n",
+ " self.std = tf.math.reduce_std(x, axis=0)\n",
+ "\n",
+ " def norm(self, x):\n",
+ " # Normalize the input\n",
+ " return (x - self.mean)/self.std\n",
+ "\n",
+ " def unnorm(self, x):\n",
+ " # Unnormalize the input\n",
+ " return (x * self.std) + self.mean"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "5BONV6fYYwZb"
+ },
+ "outputs": [],
+ "source": [
+ "norm_x = Normalize(x_train_ohe)\n",
+ "norm_y = Normalize(y_train)\n",
+ "x_train_norm, y_train_norm = norm_x.norm(x_train_ohe), norm_y.norm(y_train)\n",
+ "x_test_norm, y_test_norm = norm_x.norm(x_test_ohe), norm_y.norm(y_test)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "BPZ68wASog_I"
+ },
+ "source": [
+ "## Build a machine learning model\n",
+ "\n",
+ "Build a linear regression model with the TensorFlow Core APIs. The equation for multiple linear regression is as follows:\n",
+ "\n",
+ "$${\\mathrm{Y}} = {\\mathrm{X}}w + b$$\n",
+ "\n",
+ "where\n",
+ "\n",
+ "* $\\underset{m\\times 1}{\\mathrm{Y}}$: target vector\n",
+ "* $\\underset{m\\times n}{\\mathrm{X}}$: feature matrix\n",
+ "* $\\underset{n\\times 1}w$: weight vector\n",
+ "* $b$: bias\n",
+ "\n",
+ "By using the `@tf.function` decorator, the corresponding Python code is traced to generate a callable TensorFlow graph. This approach is beneficial for saving and loading the model after training. It can also provide a performance boost for models with many layers and complex operations. "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "h3IKyzTCDNGo"
+ },
+ "outputs": [],
+ "source": [
+ "class LinearRegression(tf.Module):\n",
+ "\n",
+ " def __init__(self):\n",
+ " self.built = False\n",
+ "\n",
+ " @tf.function\n",
+ " def __call__(self, x):\n",
+ " # Initialize the model parameters on the first call\n",
+ " if not self.built:\n",
+ " # Randomly generate the weight vector and bias term\n",
+ " rand_w = tf.random.uniform(shape=[x.shape[-1], 1])\n",
+ " rand_b = tf.random.uniform(shape=[])\n",
+ " self.w = tf.Variable(rand_w)\n",
+ " self.b = tf.Variable(rand_b)\n",
+ " self.built = True\n",
+ " y = tf.add(tf.matmul(x, self.w), self.b)\n",
+ " return tf.squeeze(y, axis=1)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "l2hiez2eIUz8"
+ },
+ "source": [
+ "For each example, the model returns a prediction for the input automobile's MPG by computing the weighted sum of its features plus a bias term. This prediction can then be unstandardized to obtain its value in terms of the original units."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "OeOrNdnkEEcR"
+ },
+ "outputs": [],
+ "source": [
+ "lin_reg = LinearRegression()\n",
+ "prediction = lin_reg(x_train_norm[:1])\n",
+ "prediction_unnorm = norm_y.unnorm(prediction)\n",
+ "prediction_unnorm.numpy()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "FIHANxNSvWr9"
+ },
+ "source": [
+ "## Define a loss function\n",
+ "\n",
+ "Now, define a loss function to evaluate the model's performance during the training process.\n",
+ "\n",
+ "Since regression problems deal with continuous outputs, the mean squared error (MSE) is an ideal choice for the loss function. The MSE is defined by the following equation:\n",
+ "\n",
+ "$$MSE = \\frac{1}{m}\\sum_{i=1}^{m}(\\hat{y}_i -y_i)^2$$\n",
+ "\n",
+ "where\n",
+ "\n",
+ "* $\\hat{y}$: vector of predictions\n",
+ "* $y$: vector of true targets\n",
+ "\n",
+ "The goal of this regression problem is to find the optimal weight vector, $w$, and bias, $b$, that minimizes the MSE loss function. "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "8tYNVUkmw35s"
+ },
+ "outputs": [],
+ "source": [
+ "def mse_loss(y_pred, y):\n",
+ " return tf.reduce_mean(tf.square(y_pred - y))"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "htI-7aJPqclK"
+ },
+ "source": [
+ "## Train and evaluate your model\n",
+ "\n",
+ "Using mini-batches for training provides both memory efficiency and faster convergence. The `tf.data.Dataset` API has useful functions for batching and shuffling. The API enables you to build complex input pipelines from simple, reusable pieces. Learn more about building TensorFlow input pipelines in [this guide](https://www.tensorflow.org/guide/data)."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "kxST2w_Nq0C5"
+ },
+ "outputs": [],
+ "source": [
+ "batch_size = 64\n",
+ "train_dataset = tf.data.Dataset.from_tensor_slices((x_train_norm, y_train_norm))\n",
+ "train_dataset = train_dataset.shuffle(buffer_size=x_train.shape[0]).batch(batch_size)\n",
+ "test_dataset = tf.data.Dataset.from_tensor_slices((x_test_norm, y_test_norm))\n",
+ "test_dataset = test_dataset.shuffle(buffer_size=x_test.shape[0]).batch(batch_size)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "C9haUW8Yq3xD"
+ },
+ "source": [
+ "Next, write a training loop to iteratively update your model's parameters by making use of the MSE loss function and its gradients with respect to the input parameters.\n",
+ "\n",
+ "This iterative method is referred to as [gradient descent](https://developers.google.com/machine-learning/glossary#gradient-descent). At each iteration, the model's parameters are updated by taking a step in the opposite direction of their computed gradients. The size of this step is determined by the learning rate, which is a configurable hyperparameter. Recall that the gradient of a function indicates the direction of its steepest ascent; therefore, taking a step in the opposite direction indicates the direction of steepest descent, which ultimately helps to minimize the MSE loss function."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "y7suUbJXVLqP"
+ },
+ "outputs": [],
+ "source": [
+ "# Set training parameters\n",
+ "epochs = 100\n",
+ "learning_rate = 0.01\n",
+ "train_losses, test_losses = [], []\n",
+ "\n",
+ "# Format training loop\n",
+ "for epoch in range(epochs):\n",
+ " batch_losses_train, batch_losses_test = [], []\n",
+ "\n",
+ " # Iterate through the training data\n",
+ " for x_batch, y_batch in train_dataset:\n",
+ " with tf.GradientTape() as tape:\n",
+ " y_pred_batch = lin_reg(x_batch)\n",
+ " batch_loss = mse_loss(y_pred_batch, y_batch)\n",
+ " # Update parameters with respect to the gradient calculations\n",
+ " grads = tape.gradient(batch_loss, lin_reg.variables)\n",
+ " for g,v in zip(grads, lin_reg.variables):\n",
+ " v.assign_sub(learning_rate * g)\n",
+ " # Keep track of batch-level training performance \n",
+ " batch_losses_train.append(batch_loss)\n",
+ " \n",
+ " # Iterate through the testing data\n",
+ " for x_batch, y_batch in test_dataset:\n",
+ " y_pred_batch = lin_reg(x_batch)\n",
+ " batch_loss = mse_loss(y_pred_batch, y_batch)\n",
+ " # Keep track of batch-level testing performance \n",
+ " batch_losses_test.append(batch_loss)\n",
+ "\n",
+ " # Keep track of epoch-level model performance\n",
+ " train_loss = tf.reduce_mean(batch_losses_train)\n",
+ " test_loss = tf.reduce_mean(batch_losses_test)\n",
+ " train_losses.append(train_loss)\n",
+ " test_losses.append(test_loss)\n",
+ " if epoch % 10 == 0:\n",
+ " print(f'Mean squared error for step {epoch}: {train_loss.numpy():0.3f}')\n",
+ "\n",
+ "# Output final losses\n",
+ "print(f\"\\nFinal train loss: {train_loss:0.3f}\")\n",
+ "print(f\"Final test loss: {test_loss:0.3f}\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "4mDAAPFqVVgn"
+ },
+ "source": [
+ "Plot the changes in MSE loss over time. Calculating performance metrics on a designated [validation set](https://developers.google.com/machine-learning/glossary#validation-set) or [test set](https://developers.google.com/machine-learning/glossary#test-set) ensures the model does not overfit to the training dataset and can generalize well to unseen data."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "F7dTAzgHDUh7"
+ },
+ "outputs": [],
+ "source": [
+ "matplotlib.rcParams['figure.figsize'] = [9, 6]\n",
+ "\n",
+ "plt.plot(range(epochs), train_losses, label = \"Training loss\")\n",
+ "plt.plot(range(epochs), test_losses, label = \"Testing loss\")\n",
+ "plt.xlabel(\"Epoch\")\n",
+ "plt.ylabel(\"Mean squared error loss\")\n",
+ "plt.legend()\n",
+ "plt.title(\"MSE loss vs training iterations\");"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "Aj8NrlzlJqDG"
+ },
+ "source": [
+ "It seems like the model does a good job of fitting the training data while also generalizing well to the unseen test data."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "AUNIPubuPYDR"
+ },
+ "source": [
+ "## Save and load the model\n",
+ "\n",
+ "Start by making an export module that takes in raw data and performs the following operations:\n",
+ "- Feature extraction \n",
+ "- Normalization \n",
+ "- Prediction\n",
+ "- Unnormalization"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "g-uOrGa9ZehG"
+ },
+ "outputs": [],
+ "source": [
+ "class ExportModule(tf.Module):\n",
+ " def __init__(self, model, extract_features, norm_x, norm_y):\n",
+ " # Initialize pre and postprocessing functions\n",
+ " self.model = model\n",
+ " self.extract_features = extract_features\n",
+ " self.norm_x = norm_x\n",
+ " self.norm_y = norm_y\n",
+ "\n",
+ " @tf.function(input_signature=[tf.TensorSpec(shape=[None, None], dtype=tf.float32)]) \n",
+ " def __call__(self, x):\n",
+ " # Run the ExportModule for new data points\n",
+ " x = self.extract_features(x)\n",
+ " x = self.norm_x.norm(x)\n",
+ " y = self.model(x)\n",
+ " y = self.norm_y.unnorm(y)\n",
+ " return y "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "YPYYLQ8EZiU8"
+ },
+ "outputs": [],
+ "source": [
+ "lin_reg_export = ExportModule(model=lin_reg,\n",
+ " extract_features=onehot_origin,\n",
+ " norm_x=norm_x,\n",
+ " norm_y=norm_y)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "6v8xi06XZWiC"
+ },
+ "source": [
+ "If you want to save the model at its current state, use the `tf.saved_model.save` function. To load a saved model for making predictions, use the `tf.saved_model.load` function."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "K1IvMoHbptht"
+ },
+ "outputs": [],
+ "source": [
+ "import tempfile\n",
+ "import os\n",
+ "\n",
+ "models = tempfile.mkdtemp()\n",
+ "save_path = os.path.join(models, 'lin_reg_export')\n",
+ "tf.saved_model.save(lin_reg_export, save_path)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "rYb6DrEH0GMv"
+ },
+ "outputs": [],
+ "source": [
+ "lin_reg_loaded = tf.saved_model.load(save_path)\n",
+ "test_preds = lin_reg_loaded(x_test)\n",
+ "test_preds[:10].numpy()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "-47O6_GLdRuT"
+ },
+ "source": [
+ "## Conclusion\n",
+ "\n",
+ "Congratulations! You have trained a regression model using the TensorFlow Core low-level APIs.\n",
+ "\n",
+ "For more examples of using TensorFlow Core APIs, check out the following guides:\n",
+ "* [Logistic regression](./logistic_regression_core.ipynb) for binary classification\n",
+ "* [Multi-layer perceptrons](./mlp_core.ipynb) for hand-written digit recognition\n"
+ ]
+ }
+ ],
+ "metadata": {
+ "colab": {
+ "collapsed_sections": [
+ "rX8mhOLljYeM"
+ ],
+ "name": "quickstart_core.ipynb",
+ "toc_visible": true
+ },
+ "kernelspec": {
+ "display_name": "Python 3",
+ "name": "python3"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}
diff --git a/site/en/guide/create_op.md b/site/en/guide/create_op.md
index 5f35812c272..fa4f573fa32 100644
--- a/site/en/guide/create_op.md
+++ b/site/en/guide/create_op.md
@@ -47,7 +47,7 @@ To incorporate your custom op you'll need to:
test the op in C++. If you define gradients, you can verify them with the
Python `tf.test.compute_gradient_error`.
See
- [`relu_op_test.py`](https://www.tensorflow.org/code/tensorflow/python/kernel_tests/relu_op_test.py) as
+ [`relu_op_test.py`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/kernel_tests/nn_ops/relu_op_test.py) as
an example that tests the forward functions of Relu-like operators and
their gradients.
@@ -55,8 +55,8 @@ To incorporate your custom op you'll need to:
* Some familiarity with C++.
* Must have installed the
- [TensorFlow binary](../../install), or must have
- [downloaded TensorFlow source](../../install/source.md),
+ [TensorFlow binary](https://www.tensorflow.org/install), or must have
+ [downloaded TensorFlow source](https://www.tensorflow.org/install/source),
and be able to build it.
## Define the op interface
@@ -152,17 +152,17 @@ REGISTER_KERNEL_BUILDER(Name("ZeroOut").Device(DEVICE_CPU), ZeroOutOp);
> Important: Instances of your OpKernel may be accessed concurrently.
> Your `Compute` method must be thread-safe. Guard any access to class
> members with a mutex. Or better yet, don't share state via class members!
-> Consider using a [`ResourceMgr`](https://www.tensorflow.org/code/tensorflow/core/framework/resource_mgr.h)
+> Consider using a [`ResourceMgr`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/resource_mgr.h)
> to keep track of op state.
### Multi-threaded CPU kernels
To write a multi-threaded CPU kernel, the Shard function in
-[`work_sharder.h`](https://www.tensorflow.org/code/tensorflow/core/util/work_sharder.h)
+[`work_sharder.h`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/util/work_sharder.h)
can be used. This function shards a computation function across the
threads configured to be used for intra-op threading (see
intra_op_parallelism_threads in
-[`config.proto`](https://www.tensorflow.org/code/tensorflow/core/protobuf/config.proto)).
+[`config.proto`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/protobuf/config.proto)).
### GPU kernels
@@ -360,12 +360,13 @@ g++ -std=c++14 -shared zero_out.cc -o zero_out.so -fPIC ${TF_CFLAGS[@]} ${TF_LFL
On macOS, the additional flag "-undefined dynamic_lookup" is required when
building the `.so` file.
-> Note on `gcc` version `>=5`: gcc uses the new C++
-> [ABI](https://gcc.gnu.org/gcc-5/changes.html#libstdcxx) since version `5`. The binary pip
-> packages available on the TensorFlow website are built with `gcc4` that uses
-> the older ABI. If you compile your op library with `gcc>=5`, add
-> `-D_GLIBCXX_USE_CXX11_ABI=0` to the command line to make the library
-> compatible with the older abi.
+> Note on `gcc` version `>=5`: gcc uses the new C++
+> [ABI](https://gcc.gnu.org/gcc-5/changes.html#libstdcxx) since version `5`.
+> TensorFlow 2.8 and earlier were built with `gcc4` that uses the older ABI. If
+> you are using these versions of TensorFlow and are trying to compile your op
+> library with `gcc>=5`, add `-D_GLIBCXX_USE_CXX11_ABI=0` to the command line to
+> make the library compatible with the older ABI. TensorFlow 2.9+ packages are
+> compatible with the newer ABI by default.
### Compile the op using bazel (TensorFlow source installation)
@@ -518,13 +519,13 @@ This asserts that the input is a vector, and returns having set the
* The `context`, which can either be an `OpKernelContext` or
`OpKernelConstruction` pointer (see
- [`tensorflow/core/framework/op_kernel.h`](https://www.tensorflow.org/code/tensorflow/core/framework/op_kernel.h)),
+ [`tensorflow/core/framework/op_kernel.h`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/op_kernel.h)),
for its `SetStatus()` method.
* The condition. For example, there are functions for validating the shape
of a tensor in
- [`tensorflow/core/framework/tensor_shape.h`](https://www.tensorflow.org/code/tensorflow/core/framework/tensor_shape.h)
+ [`tensorflow/core/framework/tensor_shape.h`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/tensor_shape.h)
* The error itself, which is represented by a `Status` object, see
- [`tensorflow/core/platform/status.h`](https://www.tensorflow.org/code/tensorflow/core/platform/status.h). A
+ [`tensorflow/core/platform/status.h`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/platform/status.h). A
`Status` has both a type (frequently `InvalidArgument`, but see the list of
types) and a message. Functions for constructing an error may be found in
[`tensorflow/core/platform/errors.h`][validation-macros].
@@ -667,7 +668,7 @@ There are shortcuts for common type constraints:
The specific lists of types allowed by these are defined by the functions (like
`NumberTypes()`) in
-[`tensorflow/core/framework/types.h`](https://www.tensorflow.org/code/tensorflow/core/framework/types.h).
+[`tensorflow/core/framework/types.h`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/types.h).
In this example the attr `t` must be one of the numeric types:
```c++
@@ -1225,7 +1226,7 @@ There are several ways to preserve backwards-compatibility.
type into a list of varying types).
The full list of safe and unsafe changes can be found in
-[`tensorflow/core/framework/op_compatibility_test.cc`](https://www.tensorflow.org/code/tensorflow/core/framework/op_compatibility_test.cc).
+[`tensorflow/core/framework/op_compatibility_test.cc`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/op_compatibility_test.cc).
If you cannot make your change to an operation backwards compatible, then create
a new operation with a new name with the new semantics.
@@ -1242,16 +1243,16 @@ made when TensorFlow changes major versions, and must conform to the
You can implement different OpKernels and register one for CPU and another for
GPU, just like you can [register kernels for different types](#polymorphism).
There are several examples of kernels with GPU support in
-[`tensorflow/core/kernels/`](https://www.tensorflow.org/code/tensorflow/core/kernels/).
+[`tensorflow/core/kernels/`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/kernels/).
Notice some kernels have a CPU version in a `.cc` file, a GPU version in a file
ending in `_gpu.cu.cc`, and some code shared in common in a `.h` file.
For example, the `tf.pad` has
everything but the GPU kernel in [`tensorflow/core/kernels/pad_op.cc`][pad_op].
The GPU kernel is in
-[`tensorflow/core/kernels/pad_op_gpu.cu.cc`](https://www.tensorflow.org/code/tensorflow/core/kernels/pad_op_gpu.cu.cc),
+[`tensorflow/core/kernels/pad_op_gpu.cu.cc`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/kernels/pad_op_gpu.cu.cc),
and the shared code is a templated class defined in
-[`tensorflow/core/kernels/pad_op.h`](https://www.tensorflow.org/code/tensorflow/core/kernels/pad_op.h).
+[`tensorflow/core/kernels/pad_op.h`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/kernels/pad_op.h).
We organize the code this way for two reasons: it allows you to share common
code among the CPU and GPU implementations, and it puts the GPU implementation
into a separate file so that it can be compiled only by the GPU compiler.
@@ -1272,16 +1273,16 @@ kept on the CPU, add a `HostMemory()` call to the kernel registration, e.g.:
#### Compiling the kernel for the GPU device
Look at
-[cuda_op_kernel.cu.cc](https://www.tensorflow.org/code/tensorflow/examples/adding_an_op/cuda_op_kernel.cu.cc)
+[cuda_op_kernel.cu.cc](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/examples/adding_an_op/cuda_op_kernel.cu.cc)
for an example that uses a CUDA kernel to implement an op. The
`tf_custom_op_library` accepts a `gpu_srcs` argument in which the list of source
files containing the CUDA kernels (`*.cu.cc` files) can be specified. For use
with a binary installation of TensorFlow, the CUDA kernels have to be compiled
with NVIDIA's `nvcc` compiler. Here is the sequence of commands you can use to
compile the
-[cuda_op_kernel.cu.cc](https://www.tensorflow.org/code/tensorflow/examples/adding_an_op/cuda_op_kernel.cu.cc)
+[cuda_op_kernel.cu.cc](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/examples/adding_an_op/cuda_op_kernel.cu.cc)
and
-[cuda_op_kernel.cc](https://www.tensorflow.org/code/tensorflow/examples/adding_an_op/cuda_op_kernel.cc)
+[cuda_op_kernel.cc](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/examples/adding_an_op/cuda_op_kernel.cc)
into a single dynamically loadable library:
```bash
@@ -1379,6 +1380,13 @@ Note that at the time the gradient function is called, only the data flow graph
of ops is available, not the tensor data itself. Thus, all computation must be
performed using other tensorflow ops, to be run at graph execution time.
+Add type hints when registering the custom gradient for an op type to make the
+code more readable, debuggable, easier to maintain, and more robust through data
+validation. For example, when taking an `op` as a parameter in a function,
+specify that the gradient function will take an
+tf.Operation
+as its parameter type.
+
### Shape functions in C++
The TensorFlow API has a feature called "shape inference" that provides
@@ -1404,7 +1412,7 @@ be set to the first input's shape. If the output is selected by its index as in
There are a number of common shape functions
that apply to many ops, such as `shape_inference::UnchangedShape` which can be
-found in [common_shape_fns.h](https://www.tensorflow.org/code/tensorflow/core/framework/common_shape_fns.h) and used as follows:
+found in [common_shape_fns.h](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/common_shape_fns.h) and used as follows:
```c++
REGISTER_OP("ZeroOut")
@@ -1451,7 +1459,7 @@ provides access to the attributes of the op).
Since shape inference is an optional feature, and the shapes of tensors may vary
dynamically, shape functions must be robust to incomplete shape information for
-any of the inputs. The `Merge` method in [`InferenceContext`](https://www.tensorflow.org/code/tensorflow/core/framework/shape_inference.h)
+any of the inputs. The `Merge` method in [`InferenceContext`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/shape_inference.h)
allows the caller to assert that two shapes are the same, even if either
or both of them do not have complete information. Shape functions are defined
for all of the core TensorFlow ops and provide many different usage examples.
@@ -1476,7 +1484,7 @@ If you have a complicated shape function, you should consider adding a test for
validating that various input shape combinations produce the expected output
shape combinations. You can see examples of how to write these tests in some
our
-[core ops tests](https://www.tensorflow.org/code/tensorflow/core/ops/array_ops_test.cc).
+[core ops tests](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/ops/array_ops_test.cc).
(The syntax of `INFER_OK` and `INFER_ERROR` are a little cryptic, but try to be
compact in representing input and output shape specifications in tests. For
now, see the surrounding comments in those tests to get a sense of the shape
@@ -1489,20 +1497,20 @@ To build a `pip` package for your op, see the
guide shows how to build custom ops from the TensorFlow pip package instead
of building TensorFlow from source.
-[core-array_ops]:https://www.tensorflow.org/code/tensorflow/core/ops/array_ops.cc
-[python-user_ops]:https://www.tensorflow.org/code/tensorflow/python/user_ops/user_ops.py
-[tf-kernels]:https://www.tensorflow.org/code/tensorflow/core/kernels/
-[user_ops]:https://www.tensorflow.org/code/tensorflow/core/user_ops/
-[pad_op]:https://www.tensorflow.org/code/tensorflow/core/kernels/pad_op.cc
-[standard_ops-py]:https://www.tensorflow.org/code/tensorflow/python/ops/standard_ops.py
-[standard_ops-cc]:https://www.tensorflow.org/code/tensorflow/cc/ops/standard_ops.h
-[python-BUILD]:https://www.tensorflow.org/code/tensorflow/python/BUILD
-[validation-macros]:https://www.tensorflow.org/code/tensorflow/core/platform/errors.h
-[op_def_builder]:https://www.tensorflow.org/code/tensorflow/core/framework/op_def_builder.h
-[register_types]:https://www.tensorflow.org/code/tensorflow/core/framework/register_types.h
-[FinalizeAttr]:https://www.tensorflow.org/code/tensorflow/core/framework/op_def_builder.cc
-[DataTypeString]:https://www.tensorflow.org/code/tensorflow/core/framework/types.cc
-[python-BUILD]:https://www.tensorflow.org/code/tensorflow/python/BUILD
-[types-proto]:https://www.tensorflow.org/code/tensorflow/core/framework/types.proto
-[TensorShapeProto]:https://www.tensorflow.org/code/tensorflow/core/framework/tensor_shape.proto
-[TensorProto]:https://www.tensorflow.org/code/tensorflow/core/framework/tensor.proto
+[core-array_ops]:https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/ops/array_ops.cc
+[python-user_ops]:https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/user_ops/user_ops.py
+[tf-kernels]:https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/kernels/
+[user_ops]:https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/user_ops/
+[pad_op]:https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/kernels/pad_op.cc
+[standard_ops-py]:https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/ops/standard_ops.py
+[standard_ops-cc]:https://github.com/tensorflow/tensorflow/blob/master/tensorflow/cc/ops/standard_ops.h
+[python-BUILD]:https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/BUILD
+[validation-macros]:https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/platform/errors.h
+[op_def_builder]:https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/op_def_builder.h
+[register_types]:https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/register_types.h
+[FinalizeAttr]:https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/op_def_builder.cc
+[DataTypeString]:https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/types.cc
+[python-BUILD]:https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/BUILD
+[types-proto]:https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/types.proto
+[TensorShapeProto]:https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/tensor_shape.proto
+[TensorProto]:https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/tensor.proto
diff --git a/site/en/guide/data.ipynb b/site/en/guide/data.ipynb
index d3dde22fd43..739ef131005 100644
--- a/site/en/guide/data.ipynb
+++ b/site/en/guide/data.ipynb
@@ -15,7 +15,6 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
- "cellView": "form",
"id": "llMNufAK7nfK"
},
"outputs": [],
@@ -589,7 +588,7 @@
"source": [
"The first output is an `int32` the second is a `float32`.\n",
"\n",
- "The first item is a scalar, shape `()`, and the second is a vector of unknown length, shape `(None,)` "
+ "The first item is a scalar, shape `()`, and the second is a vector of unknown length, shape `(None,)`"
]
},
{
@@ -601,8 +600,8 @@
"outputs": [],
"source": [
"ds_series = tf.data.Dataset.from_generator(\n",
- " gen_series, \n",
- " output_types=(tf.int32, tf.float32), \n",
+ " gen_series,\n",
+ " output_types=(tf.int32, tf.float32),\n",
" output_shapes=((), (None,)))\n",
"\n",
"ds_series"
@@ -710,8 +709,8 @@
"outputs": [],
"source": [
"ds = tf.data.Dataset.from_generator(\n",
- " lambda: img_gen.flow_from_directory(flowers), \n",
- " output_types=(tf.float32, tf.float32), \n",
+ " lambda: img_gen.flow_from_directory(flowers),\n",
+ " output_types=(tf.float32, tf.float32),\n",
" output_shapes=([32,256,256,3], [32,5])\n",
")\n",
"\n",
@@ -1386,7 +1385,7 @@
"The simplest form of batching stacks `n` consecutive elements of a dataset into\n",
"a single element. The `Dataset.batch()` transformation does exactly this, with\n",
"the same constraints as the `tf.stack()` operator, applied to each component\n",
- "of the elements: i.e. for each component *i*, all elements must have a tensor\n",
+ "of the elements: i.e., for each component *i*, all elements must have a tensor\n",
"of the exact same shape."
]
},
@@ -1932,6 +1931,7 @@
"source": [
"import scipy.ndimage as ndimage\n",
"\n",
+ "@tf.py_function(Tout=tf.float32)\n",
"def random_rotate_image(image):\n",
" image = ndimage.rotate(image, np.random.uniform(-30, 30), reshape=False)\n",
" return image"
@@ -1969,7 +1969,7 @@
"source": [
"def tf_random_rotate_image(image, label):\n",
" im_shape = image.shape\n",
- " [image,] = tf.py_function(random_rotate_image, [image], [tf.float32])\n",
+ " image = random_rotate_image(image)\n",
" image.set_shape(im_shape)\n",
" return image, label"
]
@@ -2627,7 +2627,7 @@
"it needs a separate `tf.data.Dataset` per class. You could use `Dataset.filter`\n",
"to create those two datasets, but that results in all the data being loaded twice.\n",
"\n",
- "The `tf.data.Dataset.rejection_resample` method can be applied to a dataset to rebalance it, while only loading it once. Elements will be dropped from the dataset to achieve balance.\n",
+ "The `tf.data.Dataset.rejection_resample` method can be applied to a dataset to rebalance it, while only loading it once. Elements will be dropped or repeated to achieve balance.\n",
"\n",
"The `rejection_resample` method takes a `class_func` argument. This `class_func` is applied to each dataset element, and is used to determine which class an example belongs to for the purposes of balancing.\n",
"\n",
@@ -2819,7 +2819,7 @@
"])\n",
"\n",
"model.compile(optimizer='adam',\n",
- " loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True), \n",
+ " loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n",
" metrics=['accuracy'])"
]
},
@@ -2953,8 +2953,8 @@
],
"metadata": {
"colab": {
- "collapsed_sections": [],
"name": "data.ipynb",
+ "provenance": [],
"toc_visible": true
},
"kernelspec": {
diff --git a/site/en/guide/data_performance.ipynb b/site/en/guide/data_performance.ipynb
index 78427505020..81d8b3fd5b3 100644
--- a/site/en/guide/data_performance.ipynb
+++ b/site/en/guide/data_performance.ipynb
@@ -274,6 +274,8 @@
"source": [
"### Prefetching\n",
"\n",
+ "\n",
+ "\n",
"Prefetching overlaps the preprocessing and model execution of a training step.\n",
"While the model is executing training step `s`, the input pipeline is reading the data for step `s+1`.\n",
"Doing so reduces the step time to the maximum (as opposed to the sum) of the training and the time it takes to extract the data.\n",
@@ -321,6 +323,8 @@
"source": [
"### Parallelizing data extraction\n",
"\n",
+ "\n",
+ "\n",
"In a real-world setting, the input data may be stored remotely (for example, on Google Cloud Storage or HDFS).\n",
"A dataset pipeline that works well when reading data locally might become bottlenecked on I/O when reading data remotely because of the following differences between local and remote storage:\n",
"\n",
@@ -420,6 +424,8 @@
"source": [
"### Parallelizing data transformation\n",
"\n",
+ "\n",
+ "\n",
"When preparing data, input elements may need to be pre-processed.\n",
"To this end, the `tf.data` API offers the `tf.data.Dataset.map` transformation, which applies a user-defined function to each element of the input dataset.\n",
"Because input elements are independent of one another, the pre-processing can be parallelized across multiple CPU cores.\n",
@@ -527,6 +533,8 @@
"source": [
"### Caching\n",
"\n",
+ "\n",
+ "\n",
"The `tf.data.Dataset.cache` transformation can cache a dataset, either in memory or on local storage.\n",
"This will save some operations (like file opening and data reading) from being executed during each epoch."
]
@@ -572,6 +580,8 @@
"source": [
"### Vectorizing mapping\n",
"\n",
+ "\n",
+ "\n",
"Invoking a user-defined function passed into the `map` transformation has overhead related to scheduling and executing the user-defined function.\n",
"Vectorize the user-defined function (that is, have it operate over a batch of inputs at once) and apply the `batch` transformation _before_ the `map` transformation.\n",
"\n",
@@ -687,6 +697,8 @@
"source": [
"### Reducing memory footprint\n",
"\n",
+ "\n",
+ "\n",
"A number of transformations, including `interleave`, `prefetch`, and `shuffle`, maintain an internal buffer of elements. If the user-defined function passed into the `map` transformation changes the size of the elements, then the ordering of the map transformation and the transformations that buffer elements affects the memory usage. In general, choose the order that results in lower memory footprint, unless different ordering is desirable for performance.\n",
"\n",
"#### Caching partial computations\n",
@@ -713,12 +725,12 @@
"Here is a summary of the best practices for designing performant TensorFlow\n",
"input pipelines:\n",
"\n",
- "* [Use the `prefetch` transformation](#Pipelining) to overlap the work of a producer and consumer\n",
- "* [Parallelize the data reading transformation](#Parallelizing-data-extraction) using the `interleave` transformation\n",
- "* [Parallelize the `map` transformation](#Parallelizing-data-transformation) by setting the `num_parallel_calls` argument\n",
- "* [Use the `cache` transformation](#Caching) to cache data in memory during the first epoch\n",
- "* [Vectorize user-defined functions](#Map-and-batch) passed in to the `map` transformation\n",
- "* [Reduce memory usage](#Reducing-memory-footprint) when applying the `interleave`, `prefetch`, and `shuffle` transformations"
+ "* [Use the `prefetch` transformation](#prefetching) to overlap the work of a producer and consumer\n",
+ "* [Parallelize the data reading transformation](#parallelizing_data_extraction) using the `interleave` transformation\n",
+ "* [Parallelize the `map` transformation](#parallelizing_data_transformation) by setting the `num_parallel_calls` argument\n",
+ "* [Use the `cache` transformation](#caching) to cache data in memory during the first epoch\n",
+ "* [Vectorize user-defined functions](#vectorizing_mapping) passed in to the `map` transformation\n",
+ "* [Reduce memory usage](#reducing_memory_footprint) when applying the `interleave`, `prefetch`, and `shuffle` transformations"
]
},
{
@@ -1153,7 +1165,6 @@
"colab": {
"collapsed_sections": [],
"name": "data_performance.ipynb",
- "provenance": [],
"toc_visible": true
},
"kernelspec": {
diff --git a/site/en/guide/distributed_training.ipynb b/site/en/guide/distributed_training.ipynb
index a15be67cfec..04b7118b1f2 100644
--- a/site/en/guide/distributed_training.ipynb
+++ b/site/en/guide/distributed_training.ipynb
@@ -130,7 +130,7 @@
"| **Custom training loop** | Supported | Supported | Supported | Experimental support | Experimental support |\n",
"| **Estimator API** | Limited Support | Not supported | Limited Support | Limited Support | Limited Support |\n",
"\n",
- "Note: [Experimental support](https://www.tensorflow.org/guide/versions#what_is_not_covered) means the APIs are not covered by any compatibilities guarantees.\n",
+ "Note: [Experimental support](https://www.tensorflow.org/guide/versions#what_is_not_covered) means the APIs are not covered by any compatibility guarantees.\n",
"\n",
"Warning: Estimator support is limited. Basic training and evaluation are experimental, and advanced features—such as scaffold—are not implemented. You should be using Keras or custom training loops if a use case is not covered. Estimators are not recommended for new code. Estimators run `v1.Session`-style code which is more difficult to write correctly, and can behave unexpectedly, especially when combined with TF 2 code. Estimators do fall under our [compatibility guarantees](https://tensorflow.org/guide/versions), but will receive no fixes other than security vulnerabilities. Go to the [migration guide](https://tensorflow.org/guide/migrate) for details."
]
@@ -503,14 +503,14 @@
"source": [
"## Use tf.distribute.Strategy with Keras Model.fit\n",
"\n",
- "`tf.distribute.Strategy` is integrated into `tf.keras`, which is TensorFlow's implementation of the [Keras API specification](https://keras.io). `tf.keras` is a high-level API to build and train models. By integrating into the `tf.keras` backend, it's seamless for you to distribute your training written in the Keras training framework [using Model.fit](/keras/customizing_what_happens_in_fit.ipynb).\n",
+ "`tf.distribute.Strategy` is integrated into `tf.keras`, which is TensorFlow's implementation of the [Keras API specification](https://keras.io/api/). `tf.keras` is a high-level API to build and train models. By integrating into the `tf.keras` backend, it's seamless for you to distribute your training written in the Keras training framework [using Model.fit](https://www.tensorflow.org/guide/keras/customizing_what_happens_in_fit).\n",
"\n",
"Here's what you need to change in your code:\n",
"\n",
"1. Create an instance of the appropriate `tf.distribute.Strategy`.\n",
"2. Move the creation of Keras model, optimizer and metrics inside `strategy.scope`. Thus the code in the model's `call()`, `train_step()`, and `test_step()` methods will all be distributed and executed on the accelerator(s).\n",
"\n",
- "TensorFlow distribution strategies support all types of Keras models—[Sequential](/keras/sequential_model.ipynb), [Functional](/keras/functional.ipynb), and [subclassed](/keras/custom_layers_and_models.ipynb).\n",
+ "TensorFlow distribution strategies support all types of Keras models—[Sequential](https://www.tensorflow.org/guide/keras/sequential_model), [Functional](https://www.tensorflow.org/guide/keras/functional), and [subclassed](https://www.tensorflow.org/guide/keras/custom_layers_and_models)\n",
"\n",
"Here is a snippet of code to do this for a very simple Keras model with one `Dense` layer:"
]
@@ -526,9 +526,10 @@
"mirrored_strategy = tf.distribute.MirroredStrategy()\n",
"\n",
"with mirrored_strategy.scope():\n",
- " model = tf.keras.Sequential([tf.keras.layers.Dense(1, input_shape=(1,))])\n",
- "\n",
- "model.compile(loss='mse', optimizer='sgd')"
+ " model = tf.keras.Sequential([\n",
+ " tf.keras.layers.Dense(1, input_shape=(1,),\n",
+ " kernel_regularizer=tf.keras.regularizers.L2(1e-4))])\n",
+ " model.compile(loss='mse', optimizer='sgd')"
]
},
{
@@ -647,7 +648,7 @@
"source": [
"## Use tf.distribute.Strategy with custom training loops\n",
"\n",
- "As demonstrated above, using `tf.distribute.Strategy` with Keras `Model.fit` requires changing only a couple lines of your code. With a little more effort, you can also use `tf.distribute.Strategy` [with custom training loops](/keras/writing_a_training_loop_from_scratch.ipynb).\n",
+ "As demonstrated above, using `tf.distribute.Strategy` with Keras `Model.fit` requires changing only a couple lines of your code. With a little more effort, you can also use `tf.distribute.Strategy` [with custom training loops](https://www.tensorflow.org/guide/keras/writing_a_training_loop_from_scratch).\n",
"\n",
"If you need more flexibility and control over your training loops than is possible with Estimator or Keras, you can write custom training loops. For instance, when using a GAN, you may want to take a different number of generator or discriminator steps each round. Similarly, the high level frameworks are not very suitable for Reinforcement Learning training.\n",
"\n",
@@ -674,7 +675,9 @@
"outputs": [],
"source": [
"with mirrored_strategy.scope():\n",
- " model = tf.keras.Sequential([tf.keras.layers.Dense(1, input_shape=(1,))])\n",
+ " model = tf.keras.Sequential([\n",
+ " tf.keras.layers.Dense(1, input_shape=(1,),\n",
+ " kernel_regularizer=tf.keras.regularizers.L2(1e-4))])\n",
" optimizer = tf.keras.optimizers.SGD()"
]
},
@@ -717,20 +720,21 @@
},
"outputs": [],
"source": [
+ "# Sets `reduction=NONE` to leave it to tf.nn.compute_average_loss() below.\n",
"loss_object = tf.keras.losses.BinaryCrossentropy(\n",
" from_logits=True,\n",
" reduction=tf.keras.losses.Reduction.NONE)\n",
"\n",
- "def compute_loss(labels, predictions):\n",
- " per_example_loss = loss_object(labels, predictions)\n",
- " return tf.nn.compute_average_loss(per_example_loss, global_batch_size=global_batch_size)\n",
- "\n",
"def train_step(inputs):\n",
" features, labels = inputs\n",
"\n",
" with tf.GradientTape() as tape:\n",
" predictions = model(features, training=True)\n",
- " loss = compute_loss(labels, predictions)\n",
+ " per_example_loss = loss_object(labels, predictions)\n",
+ " loss = tf.nn.compute_average_loss(per_example_loss)\n",
+ " model_losses = model.losses\n",
+ " if model_losses:\n",
+ " loss += tf.nn.scale_regularization_loss(tf.add_n(model_losses))\n",
"\n",
" gradients = tape.gradient(loss, model.trainable_variables)\n",
" optimizer.apply_gradients(zip(gradients, model.trainable_variables))\n",
@@ -751,9 +755,16 @@
"source": [
"A few other things to note in the code above:\n",
"\n",
- "1. You used `tf.nn.compute_average_loss` to compute the loss. `tf.nn.compute_average_loss` sums the per example loss and divides the sum by the `global_batch_size`. This is important because later after the gradients are calculated on each replica, they are aggregated across the replicas by **summing** them.\n",
- "2. You also used the `tf.distribute.Strategy.reduce` API to aggregate the results returned by `tf.distribute.Strategy.run`. `tf.distribute.Strategy.run` returns results from each local replica in the strategy, and there are multiple ways to consume this result. You can `reduce` them to get an aggregated value. You can also do `tf.distribute.Strategy.experimental_local_results` to get the list of values contained in the result, one per local replica.\n",
- "3. When you call `apply_gradients` within a distribution strategy scope, its behavior is modified. Specifically, before applying gradients on each parallel instance during synchronous training, it performs a sum-over-all-replicas of the gradients.\n"
+ " 1. You used `tf.nn.compute_average_loss` to reduce the per-example prediction losses to a scalar. `tf.nn.compute_average_loss` sums the per example loss and divides the sum by the global batch size. This is important because later after the gradients are calculated on each replica, they are aggregated across the replicas by **summing** them.\n",
+ "\n",
+ " By default, the global batch size is taken to be `tf.get_strategy().num_replicas_in_sync * tf.shape(per_example_loss)[0]`. It can also be specified explicitly as a keyword argument `global_batch_size=`. Without short batches, the default is equivalent to `tf.nn.compute_average_loss(..., global_batch_size=global_batch_size)` with the `global_batch_size` defined above. (For more on short batches and how to avoid or handle them, see the [Custom Training tutorial](../tutorials/distribute/custom_training.ipynb).)\n",
+ "\n",
+ " 2. You used `tf.nn.scale_regularization_loss` to scale regularization losses registered with the `Model` object, if any, by `1/num_replicas_in_sync` as well. For those regularization losses that are input-dependent, it falls on the modeling code, not the custom training loop, to perform the averaging over the per-replica(!) batch size; that way the modeling code can remain agnostic of replication while the training loop remains agnostic of how regularization losses are computed.\n",
+ "\n",
+ " 3. When you call `apply_gradients` within a distribution strategy scope, its behavior is modified. Specifically, before applying gradients on each parallel instance during synchronous training, it performs a sum-over-all-replicas of the gradients.\n",
+ "\n",
+ " 4. You also used the `tf.distribute.Strategy.reduce` API to aggregate the results returned by `tf.distribute.Strategy.run` for reporting. `tf.distribute.Strategy.run` returns results from each local replica in the strategy, and there are multiple ways to consume this result. You can `reduce` them to get an aggregated value. You can also do `tf.distribute.Strategy.experimental_local_results` to get the list of values contained in the result, one per local replica.\n",
+ "\n"
]
},
{
diff --git a/site/en/guide/dtensor_overview.ipynb b/site/en/guide/dtensor_overview.ipynb
index 3dcbe06698a..1b55ee0283f 100644
--- a/site/en/guide/dtensor_overview.ipynb
+++ b/site/en/guide/dtensor_overview.ipynb
@@ -37,7 +37,7 @@
"id": "VcQIa1uG86Wh"
},
"source": [
- "# DTensor Concepts"
+ "# DTensor concepts"
]
},
{
@@ -76,7 +76,7 @@
"\n",
"By decoupling the application from sharding directives, DTensor enables running the same application on a single device, multiple devices, or even multiple clients, while preserving its global semantics.\n",
"\n",
- "This guide introduces DTensor concepts for distributed computing, and how DTensor integrates with TensorFlow. To see a demo of using DTensor in model training, see [Distributed training with DTensor](https://www.tensorflow.org/tutorials/distribute/dtensor_ml_tutorial) tutorial."
+ "This guide introduces DTensor concepts for distributed computing, and how DTensor integrates with TensorFlow. For a demo of using DTensor in model training, refer to the [Distributed training with DTensor](../tutorials/distribute/dtensor_ml_tutorial.ipynb) tutorial."
]
},
{
@@ -87,29 +87,9 @@
"source": [
"## Setup\n",
"\n",
- "DTensor is part of TensorFlow 2.9.0 release, and also included in the TensorFlow nightly builds since 04/09/2022."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "id": "OKaPw8vwwZAC"
- },
- "outputs": [],
- "source": [
- "!pip install --quiet --upgrade --pre tensorflow"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {
- "id": "O3pG29uZIWYO"
- },
- "source": [
- "Once installed, import `tensorflow` and `tf.experimental.dtensor`. Then configure TensorFlow to use 6 virtual CPUs.\n",
+ "DTensor (`tf.experimental.dtensor`) has been part of TensorFlow since the 2.9.0 release.\n",
"\n",
- "Even though this example uses vCPUs, DTensor works the same way on CPU, GPU or TPU devices."
+ "Begin by importing TensorFlow, `dtensor`, and configure TensorFlow to use 6 virtual CPUs. Even though this example uses virtual CPUs, DTensor works the same way on CPU, GPU or TPU devices."
]
},
{
@@ -292,7 +272,7 @@
"id": "DgciDNmK76l9"
},
"source": [
- "Given a 2-dimensional 3x2 mesh such as `[(\"x\", 3), (\"y\", 2)]`, (`mesh_2d` from the previous section), `Layout([\"y\", \"x\"], mesh_2d)` is a layout for a rank-2 `Tensor` whose first axis is sharded across across mesh dimension `\"y\"`, and whose second axis is sharded across mesh dimension `\"x\"`."
+ "Given a 2-dimensional 3x2 mesh such as `[(\"x\", 3), (\"y\", 2)]`, (`mesh_2d` from the previous section), `Layout([\"y\", \"x\"], mesh_2d)` is a layout for a rank-2 `Tensor` whose first axis is sharded across mesh dimension `\"y\"`, and whose second axis is sharded across mesh dimension `\"x\"`."
]
},
{
@@ -301,7 +281,7 @@
"id": "Eyp_qOSyvieo"
},
"source": [
- "\n"
+ "\n"
]
},
{
@@ -323,7 +303,7 @@
"source": [
"For the same `mesh_2d`, the layout `Layout([\"x\", dtensor.UNSHARDED], mesh_2d)` is a layout for a rank-2 `Tensor` that is replicated across `\"y\"`, and whose first axis is sharded on mesh dimension `x`.\n",
"\n",
- "\n"
+ "\n"
]
},
{
@@ -343,13 +323,13 @@
"id": "TTalu6M-ISYb"
},
"source": [
- "### Single-Client and Multi-Client Applications\n",
+ "### Single-client and multi-client applications\n",
"\n",
"DTensor supports both single-client and multi-client applications. The colab Python kernel is an example of a single client DTensor application, where there is a single Python process.\n",
"\n",
"In a multi-client DTensor application, multiple Python processes collectively perform as a coherent application. The Cartisian grid of a `Mesh` in a multi-client DTensor application can span across devices regardless of whether they are attached locally to the current client or attached remotely to another client. The set of all devices used by a `Mesh` are called the *global device list*.\n",
"\n",
- "The creation of a `Mesh` in a multi-client DTensor application is a collective operation where the *global device list* is identicial for all of the participating clients, and the creation of the `Mesh` serves as a global barrier.\n",
+ "The creation of a `Mesh` in a multi-client DTensor application is a collective operation where the *global device list* is identical for all of the participating clients, and the creation of the `Mesh` serves as a global barrier.\n",
"\n",
"During `Mesh` creation, each client provides its *local device list* together with the expected *global device list*. DTensor validates that both lists are consistent. Please refer to the API documentation for `dtensor.create_mesh` and `dtensor.create_distributed_mesh`\n",
" for more information on multi-client mesh creation and the *global device list*.\n",
@@ -365,7 +345,8 @@
"source": [
"## DTensor as a sharded tensor\n",
"\n",
- "Now let's start coding with `DTensor`. The helper function, `dtensor_from_array`, demonstrates creating DTensors from something that looks like a `tf.Tensor`. The function performs 2 steps:\n",
+ "Now, start coding with `DTensor`. The helper function, `dtensor_from_array`, demonstrates creating DTensors from something that looks like a `tf.Tensor`. The function performs two steps:\n",
+ "\n",
" - Replicates the tensor to every device on the mesh.\n",
" - Shards the copy according to the layout requested in its arguments."
]
@@ -410,7 +391,7 @@
" - A `Layout`, which defines the `Mesh` the `Tensor` belongs to, and how the `Tensor` is sharded onto the `Mesh`.\n",
" - A list of **component tensors**, one item per local device in the `Mesh`.\n",
"\n",
- "With `dtensor_from_array`, you can create your first DTensor, `my_first_dtensor`, and examine its contents."
+ "With `dtensor_from_array`, you can create your first DTensor, `my_first_dtensor`, and examine its contents:"
]
},
{
@@ -426,7 +407,7 @@
"\n",
"my_first_dtensor = dtensor_from_array([0, 1], layout)\n",
"\n",
- "# Examine the dtensor content\n",
+ "# Examine the DTensor content\n",
"print(my_first_dtensor)\n",
"print(\"global shape:\", my_first_dtensor.shape)\n",
"print(\"dtype:\", my_first_dtensor.dtype)"
@@ -440,7 +421,7 @@
"source": [
"#### Layout and `fetch_layout`\n",
"\n",
- "The layout of a DTensor is not a regular attribute of `tf.Tensor`. Instead, DTensor provides a function, `dtensor.fetch_layout` to access the layout of a DTensor."
+ "The layout of a DTensor is not a regular attribute of `tf.Tensor`. Instead, DTensor provides a function, `dtensor.fetch_layout` to access the layout of a DTensor:"
]
},
{
@@ -499,7 +480,7 @@
"source": [
"The inverse operation of `dtensor.unpack` is `dtensor.pack`. Component tensors can be packed back into a DTensor.\n",
"\n",
- "The components must have the same rank and dtype, which will be the rank and dtype of the returned DTensor. However there is no strict requirement on the device placement of component tensors as inputs of `dtensor.unpack`: the function will automatically copy the component tensors to their respective corresponding devices.\n"
+ "The components must have the same rank and dtype, which will be the rank and dtype of the returned DTensor. However, there is no strict requirement on the device placement of component tensors as inputs of `dtensor.unpack`: the function will automatically copy the component tensors to their respective corresponding devices.\n"
]
},
{
@@ -528,7 +509,7 @@
"\n",
"So far you've worked with the `my_first_dtensor`, which is a rank-1 DTensor fully replicated across a dim-1 `Mesh`.\n",
"\n",
- "Next create and inspect DTensors that are sharded across a dim-2 `Mesh`. The next example does this with a 3x2 `Mesh` on 6 CPU devices, where size of mesh dimension `'x'` is 3 devices, and size of mesh dimension`'y'` is 2 devices."
+ "Next, create and inspect DTensors that are sharded across a dim-2 `Mesh`. The following example does this with a 3x2 `Mesh` on 6 CPU devices, where size of mesh dimension `'x'` is 3 devices, and size of mesh dimension`'y'` is 2 devices:"
]
},
{
@@ -620,7 +601,7 @@
" - 1st axis sharded along the `'x'` mesh dimension.\n",
" - 2nd axis replicated along the `'y'` mesh dimension.\n",
"\n",
- "To achieve this sharding scheme, you just need to replace the sharding spec of the 2nd axis from `'y'` to `dtensor.UNSHARDED`, to indicate your intention of replicating along the 2nd axis. The layout object will look like `Layout(['x', dtensor.UNSHARDED], mesh)`."
+ "To achieve this sharding scheme, you just need to replace the sharding spec of the 2nd axis from `'y'` to `dtensor.UNSHARDED`, to indicate your intention of replicating along the 2nd axis. The layout object will look like `Layout(['x', dtensor.UNSHARDED], mesh)`:"
]
},
{
@@ -659,7 +640,7 @@
"source": [
"#### Tensor.numpy() and sharded DTensor\n",
"\n",
- "Be aware that calling the `.numpy()` method on a sharded DTensor raises an error. The rationale for erroring is to protect against unintended gathering of data from multiple computing devices to the host CPU device backing the returned numpy array."
+ "Be aware that calling the `.numpy()` method on a sharded DTensor raises an error. The rationale for erroring is to protect against unintended gathering of data from multiple computing devices to the host CPU device backing the returned NumPy array:"
]
},
{
@@ -704,8 +685,9 @@
"Note: DTensor is still an experimental API which means you will be exploring and pushing the boundaries and limits of the DTensor programming model.\n",
"\n",
"There are 2 ways of triggering DTensor execution:\n",
- " - DTensor as operands of a Python function, e.g. `tf.matmul(a, b)` will run through DTensor if `a`, `b`, or both are DTensors.\n",
- " - Requesting the result of a Python function to be a DTensor, e.g. `dtensor.call_with_layout(tf.ones, layout, shape=(3, 2))` will run through DTensor because we requested the output of tf.ones to be sharded according to a `layout`."
+ "\n",
+ " - DTensor as operands of a Python function, such as `tf.matmul(a, b)`, will run through DTensor if `a`, `b`, or both are DTensors.\n",
+ " - Requesting the result of a Python function to be a DTensor, such as `dtensor.call_with_layout(tf.ones, layout, shape=(3, 2))`, will run through DTensor because we requested the output of `tf.ones` to be sharded according to a `layout`."
]
},
{
@@ -714,7 +696,7 @@
"id": "urKzmqAoPssT"
},
"source": [
- "### DTensor as Operands\n",
+ "### DTensor as operands\n",
"\n",
"Many TensorFlow API functions take `tf.Tensor` as their operands, and returns `tf.Tensor` as their results. For these functions, you can express intention to run a function through DTensor by passing in DTensor as operands. This section uses `tf.matmul(a, b)` as an example."
]
@@ -755,7 +737,7 @@
"print('Sharding spec:', dtensor.fetch_layout(c).sharding_specs)\n",
"print(\"components:\")\n",
"for component_tensor in dtensor.unpack(c):\n",
- " print(component_tensor.device, component_tensor.numpy())\n"
+ " print(component_tensor.device, component_tensor.numpy())"
]
},
{
@@ -770,7 +752,7 @@
"\n",
"The global matrix product sharded under this scheme can be performed efficiently, by local matmuls that runs concurrently, followed by a collective reduction to aggregate the local results. This is also the [canonical way](https://github.com/open-mpi/ompi/blob/ee87ec391f48512d3718fc7c8b13596403a09056/docs/man-openmpi/man3/MPI_Reduce.3.rst?plain=1#L265) of implementing a distributed matrix dot product.\n",
"\n",
- "Total number of floating point mul operations is `6 devices * 4 result * 1 = 24`, a factor of 3 reduction compared to the fully replicated case (72) above. The factor of 3 is due to the sharing along `x` mesh dimension with a size of `3` devices.\n",
+ "Total number of floating point mul operations is `6 devices * 4 result * 1 = 24`, a factor of 3 reduction compared to the fully replicated case (72) above. The factor of 3 is due to the sharding along `x` mesh dimension with a size of `3` devices.\n",
"\n",
"The reduction of the number of operations run sequentially is the main mechansism with which synchronuous model parallelism accelerates training."
]
@@ -800,12 +782,11 @@
"id": "IhD8yYgJiCEh"
},
"source": [
- "#### Additional Sharding\n",
+ "#### Additional sharding\n",
"\n",
"You can perform additional sharding on the inputs, and they are appropriately carried over to the results. For example, you can apply additional sharding of operand `a` along its first axis to the `'y'` mesh dimension. The additional sharding will be carried over to the first axis of the result `c`.\n",
"\n",
- "\n",
- "Total number of floating point mul operations is `6 devices * 2 result * 1 = 12`, an additional factor of 2 reduction compared to the case (24) above. The factor of 2 is due to the sharing along `y` mesh dimension with a size of `2` devices."
+ "Total number of floating point mul operations is `6 devices * 2 result * 1 = 12`, an additional factor of 2 reduction compared to the case (24) above. The factor of 2 is due to the sharding along `y` mesh dimension with a size of `2` devices."
]
},
{
@@ -837,13 +818,13 @@
"id": "c-1NazCVmLWZ"
},
"source": [
- "### DTensor as Output\n",
+ "### DTensor as output\n",
"\n",
- "What about Python functions that do not take operands, but returns a Tensor result that can be sharded? Examples of such functions are\n",
+ "What about Python functions that do not take operands, but returns a Tensor result that can be sharded? Examples of such functions are:\n",
"\n",
- " - `tf.ones`, `tf.zeros`, `tf.random.stateless_normal`,\n",
+ " - `tf.ones`, `tf.zeros`, `tf.random.stateless_normal`\n",
"\n",
- "For these Python functions, DTensor provides `dtensor.call_with_layout` which eagelry executes a Python function with DTensor, and ensures that the returned Tensor is a DTensor with the requested `Layout`."
+ "For these Python functions, DTensor provides `dtensor.call_with_layout` which eagerly executes a Python function with DTensor, and ensures that the returned Tensor is a DTensor with the requested `Layout`."
]
},
{
@@ -876,7 +857,7 @@
"source": [
"#### APIs that emit a single TensorFlow Op\n",
"\n",
- "If a function emits a single TensorFlow Op, you can directly apply `dtensor.call_with_layout` to the function."
+ "If a function emits a single TensorFlow Op, you can directly apply `dtensor.call_with_layout` to the function:"
]
},
{
@@ -911,7 +892,7 @@
"source": [
"#### APIs that emit multiple TensorFlow Ops\n",
"\n",
- "If the API emits multiple TensorFlow Ops, convert the function into a single Op through `tf.function`. For example `tf.random.stateleess_normal`"
+ "If the API emits multiple TensorFlow Ops, convert the function into a single Op through `tf.function`. For example, `tf.random.stateleess_normal`:"
]
},
{
@@ -1030,7 +1011,7 @@
"id": "QxBdNHWSu-kV"
},
"source": [
- "You can also assign a DTensor to a DVariable.\n"
+ "You can also assign a DTensor to a DVariable:\n"
]
},
{
@@ -1051,7 +1032,7 @@
"id": "4fvSk_VUvGnj"
},
"source": [
- "Attempting to mutate the layout of a `DVariable`, by assigning a DTensor with an incompatible layout produces an error."
+ "Attempting to mutate the layout of a `DVariable`, by assigning a DTensor with an incompatible layout produces an error:"
]
},
{
@@ -1081,7 +1062,7 @@
"source": [
"## What's next?\n",
"\n",
- "In this colab, you learned about DTensor, an extension to TensorFlow for distributed computing. To try out these concepts in a tutorial, see [Distributed training with DTensor](https://www.tensorflow.org/tutorials/distribute/dtensor_ml_tutorial)."
+ "In this colab, you learned about DTensor, an extension to TensorFlow for distributed computing. To try out these concepts in a tutorial, check out [Distributed training with DTensor](../tutorials/distribute/dtensor_ml_tutorial.ipynb)."
]
}
],
diff --git a/site/en/guide/estimator.ipynb b/site/en/guide/estimator.ipynb
index e0ae0a3792f..05e8fb4012a 100644
--- a/site/en/guide/estimator.ipynb
+++ b/site/en/guide/estimator.ipynb
@@ -68,7 +68,7 @@
"id": "rILQuAiiRlI7"
},
"source": [
- "> Warning: Estimators are not recommended for new code. Estimators run `v1.Session`-style code which is more difficult to write correctly, and can behave unexpectedly, especially when combined with TF 2 code. Estimators do fall under our [compatibility guarantees](https://tensorflow.org/guide/versions), but will receive no fixes other than security vulnerabilities. See the [migration guide](https://tensorflow.org/guide/migrate) for details."
+ "> Warning: TensorFlow 2.15 included the final release of the `tf-estimator` package. Estimators will not be available in TensorFlow 2.16 or after. See the [migration guide](https://www.tensorflow.org/guide/migrate/migrating_estimator) for more information about how to convert off of Estimators."
]
},
{
@@ -869,7 +869,6 @@
"A_lvUsSLZzVg"
],
"name": "estimator.ipynb",
- "provenance": [],
"toc_visible": true
},
"kernelspec": {
diff --git a/site/en/guide/extension_type.ipynb b/site/en/guide/extension_type.ipynb
index 76e20e8d283..7e8edeea7c9 100644
--- a/site/en/guide/extension_type.ipynb
+++ b/site/en/guide/extension_type.ipynb
@@ -1822,13 +1822,17 @@
" transpose_a=False, transpose_b=False,\n",
" adjoint_a=False, adjoint_b=False,\n",
" a_is_sparse=False, b_is_sparse=False,\n",
- " output_type=None):\n",
+ " output_type=None,\n",
+ " grad_a=False, grad_b=False,\n",
+ " name=None,\n",
+ " ):\n",
" if isinstance(a, MaskedTensor):\n",
" a = a.with_default(0)\n",
" if isinstance(b, MaskedTensor):\n",
" b = b.with_default(0)\n",
" return tf.matmul(a, b, transpose_a, transpose_b, adjoint_a,\n",
- " adjoint_b, a_is_sparse, b_is_sparse, output_type)"
+ " adjoint_b, a_is_sparse, b_is_sparse,\n",
+ " output_type)"
]
},
{
diff --git a/site/en/guide/function.ipynb b/site/en/guide/function.ipynb
index 280cec1fce5..f4677f21eb8 100644
--- a/site/en/guide/function.ipynb
+++ b/site/en/guide/function.ipynb
@@ -61,7 +61,7 @@
"id": "J122XQYG7W6w"
},
"source": [
- "In TensorFlow 2, [eager execution](eager.ipynb) is turned on by default. The user interface is intuitive and flexible (running one-off operations is much easier and faster), but this can come at the expense of performance and deployability.\n",
+ "In TensorFlow 2, [eager execution](basics.ipynb) is turned on by default. The user interface is intuitive and flexible (running one-off operations is much easier and faster), but this can come at the expense of performance and deployability.\n",
"\n",
"You can use `tf.function` to make graphs out of your programs. It is a transformation tool that creates Python-independent dataflow graphs out of your Python code. This will help you create performant and portable models, and it is required to use `SavedModel`.\n",
"\n",
@@ -91,8 +91,6 @@
},
"outputs": [],
"source": [
- "# Update TensorFlow, as this notebook requires version 2.9 or later\n",
- "!pip install -q -U tensorflow>=2.9.0\n",
"import tensorflow as tf"
]
},
@@ -148,7 +146,7 @@
"source": [
"### Usage\n",
"\n",
- "A `Function` you define (for example by applying the `@tf.function` decorator) is just like a core TensorFlow operation: You can execute it eagerly; you can compute gradients; and so on."
+ "A `tf.function` that you define (for example by applying the `@tf.function` decorator) is just like a core TensorFlow operation: You can execute it eagerly; you can compute gradients; and so on."
]
},
{
@@ -159,7 +157,7 @@
},
"outputs": [],
"source": [
- "@tf.function # The decorator converts `add` into a `Function`.\n",
+ "@tf.function # The decorator converts `add` into a `PolymorphicFunction`.\n",
"def add(a, b):\n",
" return a + b\n",
"\n",
@@ -186,7 +184,7 @@
"id": "ocWZvqrmHnmX"
},
"source": [
- "You can use `Function`s inside other `Function`s."
+ "You can use `tf.function`s inside other `tf.function`s."
]
},
{
@@ -210,7 +208,7 @@
"id": "piBhz7gYsHqU"
},
"source": [
- "`Function`s can be faster than eager code, especially for graphs with many small ops. But for graphs with a few expensive ops (like convolutions), you may not see much speedup.\n"
+ "`tf.function`s can be faster than eager code, especially for graphs with many small ops. But for graphs with a few expensive ops (like convolutions), you may not see much speedup.\n"
]
},
{
@@ -244,7 +242,7 @@
"source": [
"### Tracing\n",
"\n",
- "This section exposes how `Function` works under the hood, including implementation details *which may change in the future*. However, once you understand why and when tracing happens, it's much easier to use `tf.function` effectively!"
+ "This section exposes how `tf.function` works under the hood, including implementation details *which may change in the future*. However, once you understand why and when tracing happens, it's much easier to use `tf.function` effectively!"
]
},
{
@@ -255,17 +253,17 @@
"source": [
"#### What is \"tracing\"?\n",
"\n",
- "A `Function` runs your program in a [TensorFlow Graph](https://www.tensorflow.org/guide/intro_to_graphs#what_are_graphs). However, a `tf.Graph` cannot represent all the things that you'd write in an eager TensorFlow program. For instance, Python supports polymorphism, but `tf.Graph` requires its inputs to have a specified data type and dimension. Or you may perform side tasks like reading command-line arguments, raising an error, or working with a more complex Python object; none of these things can run in a `tf.Graph`.\n",
+ "A `tf.function` runs your program in a [TensorFlow Graph](https://www.tensorflow.org/guide/intro_to_graphs#what_are_graphs). However, a `tf.Graph` cannot represent all the things that you'd write in an eager TensorFlow program. For instance, Python supports polymorphism, but `tf.Graph` requires its inputs to have a specified data type and dimension. Or you may perform side tasks like reading command-line arguments, raising an error, or working with a more complex Python object; none of these things can run in a `tf.Graph`.\n",
"\n",
- "`Function` bridges this gap by separating your code in two stages:\n",
+ "`tf.function` bridges this gap by separating your code in two stages:\n",
"\n",
- " 1) In the first stage, referred to as \"**tracing**\", `Function` creates a new `tf.Graph`. Python code runs normally, but all TensorFlow operations (like adding two Tensors) are *deferred*: they are captured by the `tf.Graph` and not run.\n",
+ " 1) In the first stage, referred to as \"**tracing**\", `tf.function` creates a new `tf.Graph`. Python code runs normally, but all TensorFlow operations (like adding two Tensors) are *deferred*: they are captured by the `tf.Graph` and not run.\n",
"\n",
" 2) In the second stage, a `tf.Graph` which contains everything that was deferred in the first stage is run. This stage is much faster than the tracing stage.\n",
"\n",
- "Depending on its inputs, `Function` will not always run the first stage when it is called. See [\"Rules of tracing\"](#rules_of_tracing) below to get a better sense of how it makes that determination. Skipping the first stage and only executing the second stage is what gives you TensorFlow's high performance.\n",
+ "Depending on its inputs, `tf.function` will not always run the first stage when it is called. See [\"Rules of tracing\"](#rules_of_tracing) below to get a better sense of how it makes that determination. Skipping the first stage and only executing the second stage is what gives you TensorFlow's high performance.\n",
"\n",
- "When `Function` does decide to trace, the tracing stage is immediately followed by the second stage, so calling the `Function` both creates and runs the `tf.Graph`. Later you will see how you can run only the tracing stage with [`get_concrete_function`](#obtaining_concrete_functions)."
+ "When `tf.function` does decide to trace, the tracing stage is immediately followed by the second stage, so calling the `tf.function` both creates and runs the `tf.Graph`. Later you will see how you can run only the tracing stage with [`get_concrete_function`](#obtaining_concrete_functions)."
]
},
{
@@ -274,7 +272,7 @@
"id": "K7scSzLx662f"
},
"source": [
- "When you pass arguments of different types into a `Function`, both stages are run:\n"
+ "When you pass arguments of different types into a `tf.function`, both stages are run:\n"
]
},
{
@@ -304,7 +302,7 @@
"id": "QPfouGUQrcNb"
},
"source": [
- "Note that if you repeatedly call a `Function` with the same argument type, TensorFlow will skip the tracing stage and reuse a previously traced graph, as the generated graph would be identical."
+ "Note that if you repeatedly call a `tf.function` with the same argument type, TensorFlow will skip the tracing stage and reuse a previously traced graph, as the generated graph would be identical."
]
},
{
@@ -348,10 +346,11 @@
"So far, you've seen that `tf.function` creates a cached, dynamic dispatch layer over TensorFlow's graph tracing logic. To be more specific about the terminology:\n",
"\n",
"- A `tf.Graph` is the raw, language-agnostic, portable representation of a TensorFlow computation.\n",
- "- A `ConcreteFunction` wraps a `tf.Graph`.\n",
- "- A `Function` manages a cache of `ConcreteFunction`s and picks the right one for your inputs.\n",
- "- `tf.function` wraps a Python function, returning a `Function` object.\n",
- "- **Tracing** creates a `tf.Graph` and wraps it in a `ConcreteFunction`, also known as a **trace.**\n"
+ "- Tracing is the process through which new `tf.Graph`s are generated from Python code.\n",
+ "- An instance of `tf.Graph` is specialized to the specific input types it was traced with. Differing types require retracing.\n",
+ "- Each traced `tf.Graph` has a corresponding `ConcreteFunction`.\n",
+ "- A `tf.function` manages a cache of `ConcreteFunction`s and picks the right one for your inputs.\n",
+ "- `tf.function` wraps the Python function that will be traced, returning a `tf.types.experimental.PolymorphicFunction` object.\n"
]
},
{
@@ -362,18 +361,22 @@
"source": [
"#### Rules of tracing\n",
"\n",
- "When called, a `Function` matches the call arguments to existing `ConcreteFunction`s using `tf.types.experimental.TraceType` of each argument. If a matching `ConcreteFunction` is found, the call is dispatched to it. If no match is found, a new `ConcreteFunction` is traced. \n",
+ "When called, a `tf.function` first evaluates the type of each input argument using the `tf.types.experimental.TraceType` of each argument. This is used to construct a `tf.types.experimental.FunctionType` describing the signature of the desired `ConcreteFunction`. We compare this `FunctionType` to the `FunctionType`s of existing `ConcreteFunction`s. If a matching `ConcreteFunction` is found, the call is dispatched to it. If no match is found, a new `ConcreteFunction` is traced for the desired `FunctionType`.\n",
"\n",
- "If multiple matches are found, the most specific signature is chosen. Matching is done by [subtyping](https://en.wikipedia.org/wiki/Subtyping), much like normal function calls in C++ or Java, for instance. For example, `TensorShape([1, 2])` is a subtype of `TensorShape([None, None])` and so a call to the tf.function with `TensorShape([1, 2])` can be dispatched to the `ConcreteFunction` produced with `TensorShape([None, None])` but if a `ConcreteFunction` with `TensorShape([1, None])` also exists then it will prioritized since it is more specific.\n",
+ "If multiple matches are found, the most specific signature is chosen. Matching is done by [subtyping](https://en.wikipedia.org/wiki/Subtyping), much like normal function calls in C++ or Java, for instance. For example, `TensorShape([1, 2])` is a subtype of `TensorShape([None, None])` and so a call to the tf.function with `TensorShape([1, 2])` can be dispatched to the `ConcreteFunction` produced with `TensorShape([None, None])` but if a `ConcreteFunction` with `TensorShape([1, None])` also exists then it will be prioritized since it is more specific.\n",
"\n",
"The `TraceType` is determined from input arguments as follows:\n",
"* For `Tensor`, the type is parameterized by the `Tensor`'s `dtype` and `shape`; ranked shapes are a subtype of unranked shapes; fixed dimensions are a subtype of unknown dimensions\n",
"* For `Variable`, the type is similar to `Tensor`, but also includes a unique resource ID of the variable, necessary to correctly wire control dependencies\n",
"* For Python primitive values, the type corresponds to the **value** itself. For example, the `TraceType` of the value `3` is `LiteralTraceType<3>`, not `int`.\n",
"* For Python ordered containers such as `list` and `tuple`, etc., the type is parameterized by the types of their elements; for example, the type of `[1, 2]` is `ListTraceType, LiteralTraceType<2>>` and the type for `[2, 1]` is `ListTraceType, LiteralTraceType<1>>` which is different.\n",
- "* For Python mappings such as `dict`, the type is also a mapping from the same keys but to the types of values instead the actual values. For example, the type of `{1: 2, 3: 4}`, is `MappingTraceType<>>, >>>`. However, unlike ordered containers, `{1: 2, 3: 4}` and `{3: 4, 1: 2}` have equivalent types.\n",
- "* For Python objects which implement the `__tf_tracing_type__` method, the type is whatever that method returns\n",
- "* For any other Python objects, the type is a generic `TraceType` which uses the object's Python equality and hashing for matching. (Note: It relies on [weakref](https://docs.python.org/3/library/weakref.html) to the object and hence only works as long as the object is in scope/not deleted.)\n"
+ "* For Python mappings such as `dict`, the type is also a mapping from the same keys but to the types of values instead of the actual values. For example, the type of `{1: 2, 3: 4}`, is `MappingTraceType<>>, >>>`. However, unlike ordered containers, `{1: 2, 3: 4}` and `{3: 4, 1: 2}` have equivalent types.\n",
+ "* For Python objects which implement the `__tf_tracing_type__` method, the type is whatever that method returns.\n",
+ "* For any other Python objects, the type is a generic `TraceType`, and the matching precedure is:\n",
+ " * First it checks if the object is the same object used in the previous trace (using Python `id()` or `is`). Note that this will still match if the object has changed, so if you use Python objects as `tf.function` arguments it's best to use *immutable* ones.\n",
+ " * Next it checks if the object is equal to the object used in the previous trace (using Python `==`).\n",
+ " \n",
+ " Note that this procedure only keeps a [weakref](https://docs.python.org/3/library/weakref.html) to the object and hence only works as long as the object is in scope/not deleted.\n"
]
},
{
@@ -382,7 +385,7 @@
"id": "GNNN4lgRzpIs"
},
"source": [
- "Note: `TraceType` is based on the `Function` input parameters so changes to global and [free variables](https://docs.python.org/3/reference/executionmodel.html#binding-of-names) alone will not create a new trace. See [this section](#depending_on_python_global_and_free_variables) for recommended practices when dealing with Python global and free variables."
+ "Note: `TraceType` is based on the `tf.function` input parameters so changes to global and [free variables](https://docs.python.org/3/reference/executionmodel.html#binding-of-names) alone will not create a new trace. See [this section](#depending_on_python_global_and_free_variables) for recommended practices when dealing with Python global and free variables."
]
},
{
@@ -393,7 +396,7 @@
"source": [
"### Controlling retracing\n",
"\n",
- "Retracing, which is when your `Function` creates more than one trace, helps ensures that TensorFlow generates correct graphs for each set of inputs. However, tracing is an expensive operation! If your `Function` retraces a new graph for every call, you'll find that your code executes more slowly than if you didn't use `tf.function`.\n",
+ "Retracing, which is when your `tf.function` creates more than one trace, helps ensure that TensorFlow generates correct graphs for each set of inputs. However, tracing is an expensive operation! If your `tf.function` retraces a new graph for every call, you'll find that your code executes more slowly than if you didn't use `tf.function`.\n",
"\n",
"To control the tracing behavior, you can use the following techniques:"
]
@@ -404,7 +407,9 @@
"id": "EUtycWJa34TT"
},
"source": [
- "#### Pass a fixed `input_signature` to `tf.function`"
+ "#### Pass a fixed `input_signature` to `tf.function`\n",
+ "\n",
+ "This forces `tf.function` to constrain itself to only one `tf.types.experimental.FunctionType` composed of the types enumerated by the `input_signature`. Calls that cannot be dispatched to this `FunctionType` will throw an error."
]
},
{
@@ -422,11 +427,11 @@
"\n",
"print(next_collatz(tf.constant([1, 2])))\n",
"# You specified a 1-D tensor in the input signature, so this should fail.\n",
- "with assert_raises(ValueError):\n",
+ "with assert_raises(TypeError):\n",
" next_collatz(tf.constant([[1, 2], [3, 4]]))\n",
"\n",
"# You specified an int32 dtype in the input signature, so this should fail.\n",
- "with assert_raises(ValueError):\n",
+ "with assert_raises(TypeError):\n",
" next_collatz(tf.constant([1.0, 2.0]))\n"
]
},
@@ -438,7 +443,7 @@
"source": [
"#### Use unknown dimensions for flexibility\n",
"\n",
- " Since TensorFlow matches tensors based on their shape, using a `None` dimension as a wildcard will allow `Function`s to reuse traces for variably-sized input. Variably-sized input can occur if you have sequences of different length, or images of different sizes for each batch (See the [Transformer](../tutorials/text/transformer.ipynb) and [Deep Dream](../tutorials/generative/deepdream.ipynb) tutorials for example)."
+ " Since TensorFlow matches tensors based on their shape, using a `None` dimension as a wildcard will allow `tf.function`s to reuse traces for variably-sized input. Variably-sized input can occur if you have sequences of different length, or images of different sizes for each batch. You can check out the [Transformer](https://www.tensorflow.org/text/tutorials/transformer) and [Deep Dream](../tutorials/generative/deepdream.ipynb) tutorials for examples."
]
},
{
@@ -459,6 +464,41 @@
"print(g(tf.constant([1, 2, 3, 4, 5])))\n"
]
},
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "37cc12f93cbd"
+ },
+ "source": [
+ "#### Use `reduce_retracing` for automatic flexibility\n",
+ "\n",
+ "When `reduce_retracing` is enabled, `tf.function` automatically identifies supertypes of the input types it is observing and chooses to trace more generalized graphs automatically. It is less efficient than setting the `input_signature` directly but useful when many types need to be supported."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "0403fae03a1f"
+ },
+ "outputs": [],
+ "source": [
+ "@tf.function(reduce_retracing=True)\n",
+ "def g(x):\n",
+ " print('Tracing with', x)\n",
+ " return x\n",
+ "\n",
+ "# Traces once.\n",
+ "print(g(tf.constant([1, 2, 3])))\n",
+ "\n",
+ "# Traces again, but more generalized this time.\n",
+ "print(g(tf.constant([1, 2, 3, 4, 5])))\n",
+ "\n",
+ "# No more tracing!\n",
+ "print(g(tf.constant([1, 2, 3, 4, 5, 6, 7])))\n",
+ "print(g(tf.constant([1, 2, 3, 4, 5, 6, 7, 8, 9])))"
+ ]
+ },
{
"cell_type": "markdown",
"metadata": {
@@ -506,7 +546,7 @@
"id": "4pJqkDR_Q2wz"
},
"source": [
- "If you need to force retracing, create a new `Function`. Separate `Function` objects are guaranteed not to share traces."
+ "If you need to force retracing, create a new `tf.function`. Separate `tf.function` objects are guaranteed not to share traces."
]
},
{
@@ -535,7 +575,7 @@
"\n",
"Where possible, you should prefer converting the Python type into a `tf.experimental.ExtensionType` instead. Moreover, the `TraceType` of an `ExtensionType` is the `tf.TypeSpec` associated with it. Therefore, if needed, you can simply override the default `tf.TypeSpec` to take control of an `ExtensionType`'s `Tracing Protocol`. Refer to the _Customizing the ExtensionType's TypeSpec_ section in the [Extension types](extension_type.ipynb) guide for details.\n",
"\n",
- "Otherwise, for direct control over when `Function` should retrace in regards to a particular Python type, you can implement the `Tracing Protocol` for it yourself."
+ "Otherwise, for direct control over when `tf.function` should retrace in regards to a particular Python type, you can implement the `Tracing Protocol` for it yourself."
]
},
{
@@ -560,8 +600,8 @@
" flavor = tf.constant([3, 4])\n",
"\n",
"# As described in the above rules, a generic TraceType for `Apple` and `Mango`\n",
- "# is generated (and a corresponding ConcreteFunction is traced) but it fails to \n",
- "# match the second function call since the first pair of Apple() and Mango() \n",
+ "# is generated (and a corresponding ConcreteFunction is traced) but it fails to\n",
+ "# match the second function call since the first pair of Apple() and Mango()\n",
"# have gone out out of scope by then and deleted.\n",
"get_mixed_flavor(Apple(), Mango()) # Traces a new concrete function\n",
"get_mixed_flavor(Apple(), Mango()) # Traces a new concrete function again\n",
@@ -572,26 +612,33 @@
"# can have significant performance benefits.\n",
"\n",
"class FruitTraceType(tf.types.experimental.TraceType):\n",
- " def __init__(self, fruit_type):\n",
- " self.fruit_type = fruit_type\n",
+ " def __init__(self, fruit):\n",
+ " self.fruit_type = type(fruit)\n",
+ " self.fruit_value = fruit\n",
"\n",
" def is_subtype_of(self, other):\n",
+ " # True if self subtypes `other` and `other`'s type matches FruitTraceType.\n",
" return (type(other) is FruitTraceType and\n",
" self.fruit_type is other.fruit_type)\n",
"\n",
" def most_specific_common_supertype(self, others):\n",
+ " # `self` is the specific common supertype if all input types match it.\n",
" return self if all(self == other for other in others) else None\n",
"\n",
+ " def placeholder_value(self, placeholder_context=None):\n",
+ " # Use the fruit itself instead of the type for correct tracing.\n",
+ " return self.fruit_value\n",
+ "\n",
" def __eq__(self, other):\n",
" return type(other) is FruitTraceType and self.fruit_type == other.fruit_type\n",
- " \n",
+ "\n",
" def __hash__(self):\n",
" return hash(self.fruit_type)\n",
"\n",
"class FruitWithTraceType:\n",
"\n",
" def __tf_tracing_type__(self, context):\n",
- " return FruitTraceType(type(self))\n",
+ " return FruitTraceType(self)\n",
"\n",
"class AppleWithTraceType(FruitWithTraceType):\n",
" flavor = tf.constant([1, 2])\n",
@@ -680,8 +727,7 @@
},
"outputs": [],
"source": [
- "print(double_strings.structured_input_signature)\n",
- "print(double_strings.structured_outputs)"
+ "print(double_strings.function_type)"
]
},
{
@@ -752,7 +798,7 @@
"source": [
"### Obtaining graphs\n",
"\n",
- "Each concrete function is a callable wrapper around a `tf.Graph`. Although retrieving the actual `tf.Graph` object is not something you'll normally need to do, you can obtain it easily from any concrete function."
+ "Although retrieving the actual `tf.Graph` object is not something you'll normally need to do, you can obtain it easily from any concrete function."
]
},
{
@@ -768,6 +814,36 @@
" print(f'{node.input} -> {node.name}')\n"
]
},
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "2d49c486ccd4"
+ },
+ "source": [
+ "In reality, `tf.Graph`s are not directly callable. We actually use an `tf.types.experimental.AtomicFunction` to perform the computations described by the `tf.Graph`. You can access the `AtomicFunction` describing the traced `tf.Graph` and call it directly instead of the `ConcreteFunction`:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "4c3879aa0be0"
+ },
+ "outputs": [],
+ "source": [
+ "atomic_fn = double_strings.inference_fn\n",
+ "atomic_fn(tf.constant(\"a\"))"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "c3bd1036c18c"
+ },
+ "source": [
+ "This has the advantage of having lower Python overhead for high-performance scenarios. But it should only be used for forward inference (no gradient support), and captured tensor values (if any) would need to be explicitly supplied."
+ ]
+ },
{
"cell_type": "markdown",
"metadata": {
@@ -824,7 +900,7 @@
"id": "KxwJ8znPI0Cg"
},
"source": [
- "If you're curious you can inspect the code autograph generates."
+ "If you're curious you can inspect the code AutoGraph generates."
]
},
{
@@ -963,9 +1039,9 @@
"id": "JeD2U-yrbfVb"
},
"source": [
- "When wrapping Python/NumPy data in a Dataset, be mindful of `tf.data.Dataset.from_generator` versus ` tf.data.Dataset.from_tensors`. The former will keep the data in Python and fetch it via `tf.py_function` which can have performance implications, whereas the latter will bundle a copy of the data as one large `tf.constant()` node in the graph, which can have memory implications.\n",
+ "When wrapping Python/NumPy data in a Dataset, be mindful of `tf.data.Dataset.from_generator` versus ` tf.data.Dataset.from_tensor_slices`. The former will keep the data in Python and fetch it via `tf.py_function` which can have performance implications, whereas the latter will bundle a copy of the data as one large `tf.constant()` node in the graph, which can have memory implications.\n",
"\n",
- "Reading data from files via `TFRecordDataset`, `CsvDataset`, etc. is the most effective way to consume data, as then TensorFlow itself can manage the asynchronous loading and prefetching of data, without having to involve Python. To learn more, see the [`tf.data`: Build TensorFlow input pipelines](../../guide/data) guide."
+ "Reading data from files via `TFRecordDataset`, `CsvDataset`, etc. is the most effective way to consume data, as then TensorFlow itself can manage the asynchronous loading and prefetching of data, without having to involve Python. To learn more, see the [`tf.data`: Build TensorFlow input pipelines](data.ipynb) guide."
]
},
{
@@ -1020,7 +1096,7 @@
"source": [
"## Limitations\n",
"\n",
- "TensorFlow `Function` has a few limitations by design that you should be aware of when converting a Python function to a `Function`."
+ "`tf.function` has a few limitations by design that you should be aware of when converting a Python function to a `tf.function`."
]
},
{
@@ -1031,7 +1107,7 @@
"source": [
"### Executing Python side effects\n",
"\n",
- "Side effects, like printing, appending to lists, and mutating globals, can behave unexpectedly inside a `Function`, sometimes executing twice or not all. They only happen the first time you call a `Function` with a set of inputs. Afterwards, the traced `tf.Graph` is reexecuted, without executing the Python code.\n",
+ "Side effects, like printing, appending to lists, and mutating globals, can behave unexpectedly inside a `tf.function`, sometimes executing twice or not all. They only happen the first time you call a `tf.function` with a set of inputs. Afterwards, the traced `tf.Graph` is reexecuted, without executing the Python code.\n",
"\n",
"The general rule of thumb is to avoid relying on Python side effects in your logic and only use them to debug your traces. Otherwise, TensorFlow APIs like `tf.data`, `tf.print`, `tf.summary`, `tf.Variable.assign`, and `tf.TensorArray` are the best way to ensure your code will be executed by the TensorFlow runtime with each call."
]
@@ -1060,7 +1136,66 @@
"id": "e1I0dPiqTV8H"
},
"source": [
- "If you would like to execute Python code during each invocation of a `Function`, `tf.py_function` is an exit hatch. The drawback of `tf.py_function` is that it's not portable or particularly performant, cannot be saved with SavedModel, and does not work well in distributed (multi-GPU, TPU) setups. Also, since `tf.py_function` has to be wired into the graph, it casts all inputs/outputs to tensors."
+ "If you would like to execute Python code during each invocation of a `tf.function`, `tf. py_function` is an exit hatch. The drawbacks of `tf.py_function` are that it's not portable or particularly performant, cannot be saved with `SavedModel`, and does not work well in distributed (multi-GPU, TPU) setups. Also, since `tf.py_function` has to be wired into the graph, it casts all inputs/outputs to tensors."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "ZbI7XA_e6yA2"
+ },
+ "outputs": [],
+ "source": [
+ "@tf.py_function(Tout=tf.float32)\n",
+ "def py_plus(x, y):\n",
+ " print('Executing eagerly.')\n",
+ " return x + y\n",
+ "\n",
+ "@tf.function\n",
+ "def tf_wrapper(x, y):\n",
+ " print('Tracing.')\n",
+ " return py_plus(x, y)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "h5ttN_sI7TdQ"
+ },
+ "source": [
+ "The `tf.function` will trace the first time:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "mAK4XINl7Ldy"
+ },
+ "outputs": [],
+ "source": [
+ "tf_wrapper(tf.constant(1.0), tf.constant(2.0)).numpy()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "Atxvrd_o7dSy"
+ },
+ "source": [
+ "But the `tf.py_function` inside executes eagerly every time:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "vv7qTiTU7bjy"
+ },
+ "outputs": [],
+ "source": [
+ "tf_wrapper(tf.constant(1.0), tf.constant(2.0)).numpy()"
]
},
{
@@ -1102,7 +1237,7 @@
"id": "5eZTFRv_k_nR"
},
"source": [
- "Sometimes unexpected behaviors are very hard to notice. In the example below, the `counter` is intended to safeguard the increment of a variable. However because it is a python integer and not a TensorFlow object, it's value is captured during the first trace. When the `tf.function` is used, the `assign_add` will be recorded unconditionally in the underlying graph. Therefore `v` will increase by 1, every time the `tf.function` is called. This issue is common among users that try to migrate their Grpah-mode Tensorflow code to Tensorflow 2 using `tf.function` decorators, when python side-effects (the `counter` in the example) are used to determine what ops to run (`assign_add` in the example). Usually, users realize this only after seeing suspicious numerical results, or significantly lower performance than expected (e.g. if the guarded operation is very costly)."
+ "Sometimes unexpected behaviors are very hard to notice. In the example below, the `counter` is intended to safeguard the increment of a variable. However because it is a python integer and not a TensorFlow object, it's value is captured during the first trace. When the `tf.function` is used, the `assign_add` will be recorded unconditionally in the underlying graph. Therefore `v` will increase by 1, every time the `tf.function` is called. This issue is common among users that try to migrate their Graph-mode Tensorflow code to Tensorflow 2 using `tf.function` decorators, when python side-effects (the `counter` in the example) are used to determine what ops to run (`assign_add` in the example). Usually, users realize this only after seeing suspicious numerical results, or significantly lower performance than expected (e.g. if the guarded operation is very costly)."
]
},
{
@@ -1175,7 +1310,7 @@
"id": "pbFG5CX4LwQA"
},
"source": [
- "In summary, as a rule of thumb, you should avoid mutating python objects such as integers or containers like lists that live outside the `Function`. Instead, use arguments and TF objects. For example, the section [\"Accumulating values in a loop\"](#accumulating_values_in_a_loop) has one example of how list-like operations can be implemented.\n",
+ "In summary, as a rule of thumb, you should avoid mutating python objects such as integers or containers like lists that live outside the `tf.function`. Instead, use arguments and TF objects. For example, the section [\"Accumulating values in a loop\"](#accumulating_values_in_a_loop) has one example of how list-like operations can be implemented.\n",
"\n",
"You can, in some cases, capture and manipulate state if it is a [`tf.Variable`](https://www.tensorflow.org/guide/variable). This is how the weights of Keras models are updated with repeated calls to the same `ConcreteFunction`."
]
@@ -1369,7 +1504,7 @@
"source": [
"### Recursive tf.functions are not supported\n",
"\n",
- "Recursive `Function`s are not supported and could cause infinite loops. For example,"
+ "Recursive `tf.function`s are not supported and could cause infinite loops. For example,"
]
},
{
@@ -1397,7 +1532,7 @@
"id": "LyRyooKGUxNV"
},
"source": [
- "Even if a recursive `Function` seems to work, the python function will be traced multiple times and could have performance implication. For example,"
+ "Even if a recursive `tf.function` seems to work, the Python function will be traced multiple times and could have performance implications. For example,"
]
},
{
@@ -1427,7 +1562,7 @@
"source": [
"## Known Issues\n",
"\n",
- "If your `Function` is not evaluating correctly, the error may be explained by these known issues which are planned to be fixed in the future."
+ "If your `tf.function` is not evaluating correctly, the error may be explained by these known issues which are planned to be fixed in the future."
]
},
{
@@ -1438,7 +1573,7 @@
"source": [
"### Depending on Python global and free variables\n",
"\n",
- "`Function` creates a new `ConcreteFunction` when called with a new value of a Python argument. However, it does not do that for the Python closure, globals, or nonlocals of that `Function`. If their value changes in between calls to the `Function`, the `Function` will still use the values they had when it was traced. This is different from how regular Python functions work.\n",
+ "`tf.function` creates a new `ConcreteFunction` when called with a new value of a Python argument. However, it does not do that for the Python closure, globals, or nonlocals of that `tf.function`. If their value changes in between calls to the `tf.function`, the `tf.function` will still use the values they had when it was traced. This is different from how regular Python functions work.\n",
"\n",
"For that reason, you should follow a functional programming style that uses arguments instead of closing over outer names."
]
@@ -1484,7 +1619,7 @@
"id": "ZoPg5w1Pjqnb"
},
"source": [
- "Another way to update a global value, is to make it a `tf.Variable` and use the `Variable.assign` method instead.\n"
+ "Another way to update a global value is to make it a `tf.Variable` and use the `Variable.assign` method instead.\n"
]
},
{
@@ -1522,7 +1657,7 @@
"id": "hvwe9gTIWfx6"
},
"source": [
- "#### Depending on Python objects"
+ "### Depending on Python objects"
]
},
{
@@ -1531,7 +1666,11 @@
"id": "BJkZS-SwPvOQ"
},
"source": [
- "The recommendation to pass Python objects as arguments into `tf.function` has a number of known issues, that are expected to be fixed in the future. In general, you can rely on consistent tracing if you use a Python primitive or `tf.nest`-compatible structure as an argument or pass in a *different* instance of an object into a `Function`. However, `Function` will *not* create a new trace when you pass **the same object and only change its attributes**."
+ "Passing custom Python objects as arguments to `tf.function` is supported but has certain limitations.\n",
+ "\n",
+ "For maximum feature coverage, consider transforming the objects into [Extension types](extension_type.ipynb) before passing them to `tf.function`. You can also use Python primitives and `tf.nest`-compatible structures.\n",
+ "\n",
+ "However, as covered in the [rules of tracing](#rules_of_tracing), when a custom `TraceType` is not provided by the custom Python class, `tf.function` is forced to use instance-based equality which means it will **not create a new trace** when you pass the **same object with modified attributes**."
]
},
{
@@ -1576,11 +1715,11 @@
"id": "Ytcgg2qFWaBF"
},
"source": [
- "Using the same `Function` to evaluate the updated instance of the model will be buggy since the updated model has the [same cache key](#rules_of_tracing) as the original model.\n",
+ "Using the same `tf.function` to evaluate the modified instance of the model will be buggy since it still has the [same instance-based TraceType](#rules_of_tracing) as the original model.\n",
"\n",
- "For that reason, you're recommended to write your `Function` to avoid depending on mutable object attributes or create new objects.\n",
+ "For that reason, you're recommended to write your `tf.function` to avoid depending on mutable object attributes or implement the [Tracing Protocol](#use_the_tracing_protocol) for the objects to inform `tf.function` about such attributes.\n",
"\n",
- "If that is not possible, one workaround is to make new `Function`s each time you modify your object to force retracing:"
+ "If that is not possible, one workaround is to make new `tf.function`s each time you modify your object to force retracing:"
]
},
{
@@ -1596,8 +1735,8 @@
"\n",
"new_model = SimpleModel()\n",
"evaluate_no_bias = tf.function(evaluate).get_concrete_function(new_model, x)\n",
- "# Don't pass in `new_model`, `Function` already captured its state during tracing.\n",
- "print(evaluate_no_bias(x)) "
+ "# Don't pass in `new_model`. `tf.function` already captured its state during tracing.\n",
+ "print(evaluate_no_bias(x))"
]
},
{
@@ -1610,7 +1749,7 @@
"source": [
"print(\"Adding bias!\")\n",
"new_model.bias += 5.0\n",
- "# Create new Function and ConcreteFunction since you modified new_model.\n",
+ "# Create new `tf.function` and `ConcreteFunction` since you modified `new_model`.\n",
"evaluate_with_bias = tf.function(evaluate).get_concrete_function(new_model, x)\n",
"print(evaluate_with_bias(x)) # Don't pass in `new_model`."
]
@@ -1667,7 +1806,7 @@
"source": [
"### Creating tf.Variables\n",
"\n",
- "`Function` only supports singleton `tf.Variable`s created once on the first call, and reused across subsequent function calls. The code snippet below would create a new `tf.Variable` in every function call, which results in a `ValueError` exception.\n",
+ "`tf.function` only supports singleton `tf.Variable`s created once on the first call, and reused across subsequent function calls. The code snippet below would create a new `tf.Variable` in every function call, which results in a `ValueError` exception.\n",
"\n",
"Example:"
]
@@ -1728,7 +1867,7 @@
},
"source": [
"#### Using with multiple Keras optimizers\n",
- "You may encounter `ValueError: tf.function only supports singleton tf.Variables created on the first call.` when using more than one Keras optimizer with a `tf.function`. This error occurs because optimizers internally create `tf.Variables` when they apply gradients for the first time."
+ "You may encounter `ValueError: tf.function only supports singleton tf.Variables created on the first call.` when using more than one Keras optimizer with a `tf.function`. This error occurs because optimizers internally create `tf.Variable`s when they apply gradients for the first time."
]
},
{
@@ -1741,7 +1880,7 @@
"source": [
"opt1 = tf.keras.optimizers.Adam(learning_rate = 1e-2)\n",
"opt2 = tf.keras.optimizers.Adam(learning_rate = 1e-3)\n",
- " \n",
+ "\n",
"@tf.function\n",
"def train_step(w, x, y, optimizer):\n",
" with tf.GradientTape() as tape:\n",
@@ -1765,7 +1904,46 @@
"id": "7Q8BRPCThTjB"
},
"source": [
- "If you need to change the optimizer during training, a workaround is to create a new `Function` for each optimizer, calling the [`ConcreteFunction`](#obtaining_concrete_functions) directly."
+ "If you need to change a stateful object between calls, it's simplest to define a `tf.Module` subclass, and create instances to hold those objects:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "3P59ocmIslHz"
+ },
+ "outputs": [],
+ "source": [
+ "class TrainStep(tf.Module):\n",
+ " def __init__(self, optimizer):\n",
+ " self.optimizer = optimizer\n",
+ "\n",
+ " @tf.function\n",
+ " def __call__(self, w, x, y):\n",
+ " with tf.GradientTape() as tape:\n",
+ " L = tf.reduce_sum(tf.square(w*x - y))\n",
+ " gradients = tape.gradient(L, [w])\n",
+ " self.optimizer.apply_gradients(zip(gradients, [w]))\n",
+ "\n",
+ "\n",
+ "opt1 = tf.keras.optimizers.Adam(learning_rate = 1e-2)\n",
+ "opt2 = tf.keras.optimizers.Adam(learning_rate = 1e-3)\n",
+ "\n",
+ "train_o1 = TrainStep(opt1)\n",
+ "train_o2 = TrainStep(opt2)\n",
+ "\n",
+ "train_o1(w, x, y)\n",
+ "train_o2(w, x, y)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "dUHUi881smHF"
+ },
+ "source": [
+ "You could also do this manually by creating multiple instances of the `@tf.function` wrapper, one for each optimizer:"
]
},
{
@@ -1790,14 +1968,14 @@
"x = tf.constant([-1.])\n",
"y = tf.constant([2.])\n",
"\n",
- "# Make a new Function and ConcreteFunction for each optimizer.\n",
- "train_step_1 = tf.function(train_step).get_concrete_function(w, x, y, opt1)\n",
- "train_step_2 = tf.function(train_step).get_concrete_function(w, x, y, opt2)\n",
+ "# Make a new tf.function and ConcreteFunction for each optimizer.\n",
+ "train_step_1 = tf.function(train_step)\n",
+ "train_step_2 = tf.function(train_step)\n",
"for i in range(10):\n",
" if i % 2 == 0:\n",
- " train_step_1(w, x, y) # `opt1` is not used as a parameter. \n",
+ " train_step_1(w, x, y, opt1)\n",
" else:\n",
- " train_step_2(w, x, y) # `opt2` is not used as a parameter."
+ " train_step_2(w, x, y, opt2)"
]
},
{
@@ -1808,9 +1986,9 @@
"source": [
"#### Using with multiple Keras models\n",
"\n",
- "You may also encounter `ValueError: tf.function only supports singleton tf.Variables created on the first call.` when passing different model instances to the same `Function`.\n",
+ "You may also encounter `ValueError: tf.function only supports singleton tf.Variables created on the first call.` when passing different model instances to the same `tf.function`.\n",
"\n",
- "This error occurs because Keras models (which [do not have their input shape defined](https://www.tensorflow.org/guide/keras/custom_layers_and_models#best_practice_deferring_weight_creation_until_the_shape_of_the_inputs_is_known)) and Keras layers create `tf.Variables`s when they are first called. You may be attempting to initialize those variables inside a `Function`, which has already been called. To avoid this error, try calling `model.build(input_shape)` to initialize all the weights before training the model.\n"
+ "This error occurs because Keras models (which [do not have their input shape defined](https://www.tensorflow.org/guide/keras/custom_layers_and_models#best_practice_deferring_weight_creation_until_the_shape_of_the_inputs_is_known)) and Keras layers create `tf.Variable`s when they are first called. You may be attempting to initialize those variables inside a `tf.function`, which has already been called. To avoid this error, try calling `model.build(input_shape)` to initialize all the weights before training the model.\n"
]
},
{
@@ -1821,14 +1999,15 @@
"source": [
"## Further reading\n",
"\n",
- "To learn about how to export and load a `Function`, see the [SavedModel guide](../../guide/saved_model). To learn more about graph optimizations that are performed after tracing, see the [Grappler guide](../../guide/graph_optimization). To learn how to optimize your data pipeline and profile your model, see the [Profiler guide](../../guide/profiler.md)."
+ "To learn about how to export and load a `tf.function`, see the [SavedModel guide](../../guide/saved_model). To learn more about graph optimizations that are performed after tracing, see the [Grappler guide](../../guide/graph_optimization). To learn how to optimize your data pipeline and profile your model, see the [Profiler guide](../../guide/profiler.md)."
]
}
],
"metadata": {
"colab": {
- "collapsed_sections": [],
"name": "function.ipynb",
+ "private_outputs": true,
+ "provenance": [],
"toc_visible": true
},
"kernelspec": {
diff --git a/site/en/guide/gpu_performance_analysis.md b/site/en/guide/gpu_performance_analysis.md
index ecbb03ad3ad..2832686d8f1 100644
--- a/site/en/guide/gpu_performance_analysis.md
+++ b/site/en/guide/gpu_performance_analysis.md
@@ -169,8 +169,8 @@ the trace viewer, you should look at the model code between steps and check if
disabling callbacks/metrics improves performance. Some details of these ops are
also on the trace viewer (both device and host side).The recommendation in this
scenario is to amortize the overhead of these ops by executing them after a
-fixed number of steps instead of every step. When using the `compile` method in
-the `tf.keras` API, setting the `experimental_steps_per_execution` flag does
+fixed number of steps instead of every step. When using the `Model.compile` method in
+the `tf.keras` API, setting the `steps_per_execution` flag does
this automatically. For custom training loops, use `tf.while_loop`.
#### 2. Achieve higher device utilization
diff --git a/site/en/guide/graph_optimization.ipynb b/site/en/guide/graph_optimization.ipynb
index fa573e67800..063d8817489 100644
--- a/site/en/guide/graph_optimization.ipynb
+++ b/site/en/guide/graph_optimization.ipynb
@@ -101,7 +101,7 @@
"* *Loop optimizer -* Optimizes the graph control flow by hoisting loop-invariant subgraphs out of loops and by removing redundant stack operations in loops. Also optimizes loops with statically known trip counts and removes statically known dead branches in conditionals.\n",
"* *Scoped allocator optimizer -* Introduces scoped allocators to reduce data movement and to consolidate some operations.\n",
"* *Pin to host optimizer -* Swaps small operations onto the CPU. This optimizer is turned OFF by default. \n",
- "* *Auto mixed precision optimizer -* Converts data types to float16 where applicable to improve performance. Currently applies only to GPUs.\n",
+ "* *Auto mixed precision optimizer -* Converts data types to float16 where applicable to improve performance. Currently applies to GPUs and the latest Intel Xeon CPUs.\n",
"* *Debug stripper -* Strips nodes related to debugging operations such as `tf.debugging.Assert`, `tf.debugging.check_numerics`, and `tf.print` from the graph. This optimizer is turned OFF by default."
]
},
diff --git a/site/en/guide/images/new_type_promotion/type_promotion_lattice.png b/site/en/guide/images/new_type_promotion/type_promotion_lattice.png
new file mode 100644
index 00000000000..501698965a2
Binary files /dev/null and b/site/en/guide/images/new_type_promotion/type_promotion_lattice.png differ
diff --git a/site/en/guide/images/new_type_promotion/type_promotion_table.png b/site/en/guide/images/new_type_promotion/type_promotion_table.png
new file mode 100644
index 00000000000..62bb465212a
Binary files /dev/null and b/site/en/guide/images/new_type_promotion/type_promotion_table.png differ
diff --git a/site/en/guide/intro_to_graphs.ipynb b/site/en/guide/intro_to_graphs.ipynb
index 108e9d89c9d..4fe442632ba 100644
--- a/site/en/guide/intro_to_graphs.ipynb
+++ b/site/en/guide/intro_to_graphs.ipynb
@@ -37,7 +37,7 @@
"id": "6xgB0Oz5eGSQ"
},
"source": [
- "# Introduction to graphs and `tf.function`"
+ "# Introduction to graphs and tf.function"
]
},
{
@@ -87,7 +87,7 @@
"source": [
"### What are graphs?\n",
"\n",
- "In the previous three guides, you ran TensorFlow **eagerly**. This means TensorFlow operations are executed by Python, operation by operation, and returning results back to Python.\n",
+ "In the previous three guides, you ran TensorFlow **eagerly**. This means TensorFlow operations are executed by Python, operation by operation, and return results back to Python.\n",
"\n",
"While eager execution has several unique advantages, graph execution enables portability outside Python and tends to offer better performance. **Graph execution** means that tensor computations are executed as a *TensorFlow graph*, sometimes referred to as a `tf.Graph` or simply a \"graph.\"\n",
"\n",
@@ -174,7 +174,7 @@
"source": [
"## Taking advantage of graphs\n",
"\n",
- "You create and run a graph in TensorFlow by using `tf.function`, either as a direct call or as a decorator. `tf.function` takes a regular function as input and returns a `Function`. **A `Function` is a Python callable that builds TensorFlow graphs from the Python function. You use a `Function` in the same way as its Python equivalent.**\n"
+ "You create and run a graph in TensorFlow by using `tf.function`, either as a direct call or as a decorator. `tf.function` takes a regular function as input and returns a `tf.types.experimental.PolymorphicFunction`. **A `PolymorphicFunction` is a Python callable that builds TensorFlow graphs from the Python function. You use a `tf.function` in the same way as its Python equivalent.**\n"
]
},
{
@@ -191,7 +191,8 @@
" x = x + b\n",
" return x\n",
"\n",
- "# `a_function_that_uses_a_graph` is a TensorFlow `Function`.\n",
+ "# The Python type of `a_function_that_uses_a_graph` will now be a\n",
+ "# `PolymorphicFunction`.\n",
"a_function_that_uses_a_graph = tf.function(a_regular_function)\n",
"\n",
"# Make some tensors.\n",
@@ -200,7 +201,7 @@
"b1 = tf.constant(4.0)\n",
"\n",
"orig_value = a_regular_function(x1, y1, b1).numpy()\n",
- "# Call a `Function` like a Python function.\n",
+ "# Call a `tf.function` like a Python function.\n",
"tf_function_value = a_function_that_uses_a_graph(x1, y1, b1).numpy()\n",
"assert(orig_value == tf_function_value)"
]
@@ -211,7 +212,7 @@
"id": "PNvuAYpdrTOf"
},
"source": [
- "On the outside, a `Function` looks like a regular function you write using TensorFlow operations. [Underneath](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/eager/def_function.py), however, it is *very different*. A `Function` **encapsulates several `tf.Graph`s behind one API** (learn more in the _Polymorphism_ section). That is how a `Function` is able to give you the benefits of graph execution, like speed and deployability (refer to _The benefits of graphs_ above)."
+ "On the outside, a `tf.function` looks like a regular function you write using TensorFlow operations. [Underneath](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/eager/polymorphic_function/polymorphic_function.py), however, it is *very different*. The underlying `PolymorphicFunction` **encapsulates several `tf.Graph`s behind one API** (learn more in the _Polymorphism_ section). That is how a `tf.function` is able to give you the benefits of graph execution, like speed and deployability (refer to _The benefits of graphs_ above)."
]
},
{
@@ -236,7 +237,8 @@
" x = x + b\n",
" return x\n",
"\n",
- "# Use the decorator to make `outer_function` a `Function`.\n",
+ "# Using the `tf.function` decorator makes `outer_function` into a\n",
+ "# `PolymorphicFunction`.\n",
"@tf.function\n",
"def outer_function(x):\n",
" y = tf.constant([[2.0], [3.0]])\n",
@@ -283,7 +285,8 @@
" else:\n",
" return 0\n",
"\n",
- "# `tf_simple_relu` is a TensorFlow `Function` that wraps `simple_relu`.\n",
+ "# Using `tf.function` makes `tf_simple_relu` a `PolymorphicFunction` that wraps\n",
+ "# `simple_relu`.\n",
"tf_simple_relu = tf.function(simple_relu)\n",
"\n",
"print(\"First branch, with graph:\", tf_simple_relu(tf.constant(1)).numpy())\n",
@@ -338,13 +341,13 @@
"id": "sIpc_jfjEZEg"
},
"source": [
- "### Polymorphism: one `Function`, many graphs\n",
+ "### Polymorphism: one `tf.function`, many graphs\n",
"\n",
- "A `tf.Graph` is specialized to a specific type of inputs (for example, tensors with a specific [`dtype`](https://www.tensorflow.org/api_docs/python/tf/dtypes/DType) or objects with the same [`id()`](https://docs.python.org/3/library/functions.html#id])).\n",
+ "A `tf.Graph` is specialized to a specific type of inputs (for example, tensors with a specific [`dtype`](https://www.tensorflow.org/api_docs/python/tf/dtypes/DType) or objects with the same [`id()`](https://docs.python.org/3/library/functions.html#id)).\n",
"\n",
- "Each time you invoke a `Function` with a set of arguments that can't be handled by any of its existing graphs (such as arguments with new `dtypes` or incompatible shapes), `Function` creates a new `tf.Graph` specialized to those new arguments. The type specification of a `tf.Graph`'s inputs is known as its **input signature** or just a **signature**. For more information regarding when a new `tf.Graph` is generated and how that can be controlled, go to the _Rules of tracing_ section of the [Better performance with `tf.function`](./function.ipynb) guide.\n",
+ "Each time you invoke a `tf.function` with a set of arguments that can't be handled by any of its existing graphs (such as arguments with new `dtypes` or incompatible shapes), it creates a new `tf.Graph` specialized to those new arguments. The type specification of a `tf.Graph`'s inputs is represented by `tf.types.experimental.FunctionType`, also referred to as the **signature**. For more information regarding when a new `tf.Graph` is generated, how that can be controlled, and how `FunctionType` can be useful, go to the _Rules of tracing_ section of the [Better performance with `tf.function`](./function.ipynb) guide.\n",
"\n",
- "The `Function` stores the `tf.Graph` corresponding to that signature in a `ConcreteFunction`. **A `ConcreteFunction` is a wrapper around a `tf.Graph`.**\n"
+ "The `tf.function` stores the `tf.Graph` corresponding to that signature in a `ConcreteFunction`. **A `ConcreteFunction` can be thought of as a wrapper around a `tf.Graph`.**\n"
]
},
{
@@ -359,7 +362,7 @@
"def my_relu(x):\n",
" return tf.maximum(0., x)\n",
"\n",
- "# `my_relu` creates new graphs as it observes more signatures.\n",
+ "# `my_relu` creates new graphs as it observes different input types.\n",
"print(my_relu(tf.constant(5.5)))\n",
"print(my_relu([1, -1]))\n",
"print(my_relu(tf.constant([3., -3.])))"
@@ -371,7 +374,7 @@
"id": "1qRtw7R4KL9X"
},
"source": [
- "If the `Function` has already been called with that signature, `Function` does not create a new `tf.Graph`."
+ "If the `tf.function` has already been called with the same input types, it does not create a new `tf.Graph`."
]
},
{
@@ -383,8 +386,8 @@
"outputs": [],
"source": [
"# These two calls do *not* create new graphs.\n",
- "print(my_relu(tf.constant(-2.5))) # Signature matches `tf.constant(5.5)`.\n",
- "print(my_relu(tf.constant([-1., 1.]))) # Signature matches `tf.constant([3., -3.])`."
+ "print(my_relu(tf.constant(-2.5))) # Input type matches `tf.constant(5.5)`.\n",
+ "print(my_relu(tf.constant([-1., 1.]))) # Input type matches `tf.constant([3., -3.])`."
]
},
{
@@ -393,7 +396,7 @@
"id": "UohRmexhIpvQ"
},
"source": [
- "Because it's backed by multiple graphs, a `Function` is **polymorphic**. That enables it to support more input types than a single `tf.Graph` could represent, and to optimize each `tf.Graph` for better performance."
+ "Because it's backed by multiple graphs, a `tf.function` is (as the name \"PolymorphicFunction\" suggests) **polymorphic**. That enables it to support more input types than a single `tf.Graph` could represent, and to optimize each `tf.Graph` for better performance."
]
},
{
@@ -428,7 +431,7 @@
"source": [
"### Graph execution vs. eager execution\n",
"\n",
- "The code in a `Function` can be executed both eagerly and as a graph. By default, `Function` executes its code as a graph:\n"
+ "The code in a `tf.function` can be executed both eagerly and as a graph. By default, `tf.function` executes its code as a graph:\n"
]
},
{
@@ -476,7 +479,7 @@
"id": "cyZNCRcQorGO"
},
"source": [
- "To verify that your `Function`'s graph is doing the same computation as its equivalent Python function, you can make it execute eagerly with `tf.config.run_functions_eagerly(True)`. This is a switch that **turns off `Function`'s ability to create and run graphs**, instead of executing the code normally."
+ "To verify that your `tf.function`'s graph is doing the same computation as its equivalent Python function, you can make it execute eagerly with `tf.config.run_functions_eagerly(True)`. This is a switch that **turns off `tf.function`'s ability to create and run graphs**, instead of executing the code normally."
]
},
{
@@ -519,7 +522,7 @@
"id": "DKT3YBsqy0x4"
},
"source": [
- "However, `Function` can behave differently under graph and eager execution. The Python [`print`](https://docs.python.org/3/library/functions.html#print) function is one example of how these two modes differ. Let's check out what happens when you insert a `print` statement to your function and call it repeatedly."
+ "However, `tf.function` can behave differently under graph and eager execution. The Python [`print`](https://docs.python.org/3/library/functions.html#print) function is one example of how these two modes differ. Let's check out what happens when you insert a `print` statement to your function and call it repeatedly."
]
},
{
@@ -567,7 +570,7 @@
"source": [
"Is the output surprising? **`get_MSE` only printed once even though it was called *three* times.**\n",
"\n",
- "To explain, the `print` statement is executed when `Function` runs the original code in order to create the graph in a process known as \"tracing\" (refer to the _Tracing_ section of the [`tf.function` guide](./function.ipynb). **Tracing captures the TensorFlow operations into a graph, and `print` is not captured in the graph.** That graph is then executed for all three calls **without ever running the Python code again**.\n",
+ "To explain, the `print` statement is executed when `tf.function` runs the original code in order to create the graph in a process known as \"tracing\" (refer to the _Tracing_ section of the [`tf.function` guide](./function.ipynb). **Tracing captures the TensorFlow operations into a graph, and `print` is not captured in the graph.** That graph is then executed for all three calls **without ever running the Python code again**.\n",
"\n",
"As a sanity check, let's turn off graph execution to compare:"
]
@@ -615,7 +618,7 @@
"id": "PUR7qC_bquCn"
},
"source": [
- "`print` is a *Python side effect*, and there are other differences that you should be aware of when converting a function into a `Function`. Learn more in the _Limitations_ section of the [Better performance with `tf.function`](./function.ipynb) guide."
+ "`print` is a *Python side effect*, and there are other differences that you should be aware of when converting a function into a `tf.function`. Learn more in the _Limitations_ section of the [Better performance with `tf.function`](./function.ipynb) guide."
]
},
{
@@ -637,7 +640,7 @@
"\n",
"\n",
"\n",
- "Graph execution only executes the operations necessary to produce the observable effects, which includes:\n",
+ "Graph execution only executes the operations necessary to produce the observable effects, which include:\n",
"\n",
"- The return value of the function\n",
"- Documented well-known side-effects such as:\n",
@@ -697,7 +700,7 @@
"source": [
"### `tf.function` best practices\n",
"\n",
- "It may take some time to get used to the behavior of `Function`. To get started quickly, first-time users should play around with decorating toy functions with `@tf.function` to get experience with going from eager to graph execution.\n",
+ "It may take some time to get used to the behavior of `tf.function`. To get started quickly, first-time users should play around with decorating toy functions with `@tf.function` to get experience with going from eager to graph execution.\n",
"\n",
"*Designing for `tf.function`* may be your best bet for writing graph-compatible TensorFlow programs. Here are some tips:\n",
"- Toggle between eager and graph execution early and often with `tf.config.run_functions_eagerly` to pinpoint if/ when the two modes diverge.\n",
@@ -751,7 +754,7 @@
},
"outputs": [],
"source": [
- "print(\"Eager execution:\", timeit.timeit(lambda: power(x, 100), number=1000))"
+ "print(\"Eager execution:\", timeit.timeit(lambda: power(x, 100), number=1000), \"seconds\")"
]
},
{
@@ -763,7 +766,7 @@
"outputs": [],
"source": [
"power_as_graph = tf.function(power)\n",
- "print(\"Graph execution:\", timeit.timeit(lambda: power_as_graph(x, 100), number=1000))"
+ "print(\"Graph execution:\", timeit.timeit(lambda: power_as_graph(x, 100), number=1000), \"seconds\")"
]
},
{
@@ -787,7 +790,7 @@
"\n",
"Graphs can speed up your code, but the process of creating them has some overhead. For some functions, the creation of the graph takes more time than the execution of the graph. **This investment is usually quickly paid back with the performance boost of subsequent executions, but it's important to be aware that the first few steps of any large model training can be slower due to tracing.**\n",
"\n",
- "No matter how large your model, you want to avoid tracing frequently. The [`tf.function` guide](./function.ipynb) discusses how to set input specifications and use tensor arguments to avoid retracing in the _Controlling retracing_ section. If you find you are getting unusually poor performance, it's a good idea to check if you are retracing accidentally."
+ "No matter how large your model, you want to avoid tracing frequently. In the _Controlling retracing_ section, the [`tf.function` guide](./function.ipynb) discusses how to set input specifications and use tensor arguments to avoid retracing. If you find you are getting unusually poor performance, it's a good idea to check if you are retracing accidentally."
]
},
{
@@ -796,9 +799,9 @@
"id": "F4InDaTjwmBA"
},
"source": [
- "## When is a `Function` tracing?\n",
+ "## When is a `tf.function` tracing?\n",
"\n",
- "To figure out when your `Function` is tracing, add a `print` statement to its code. As a rule of thumb, `Function` will execute the `print` statement every time it traces."
+ "To figure out when your `tf.function` is tracing, add a `print` statement to its code. As a rule of thumb, `tf.function` will execute the `print` statement every time it traces."
]
},
{
@@ -858,7 +861,6 @@
],
"metadata": {
"colab": {
- "collapsed_sections": [],
"name": "intro_to_graphs.ipynb",
"toc_visible": true
},
diff --git a/site/en/guide/intro_to_modules.ipynb b/site/en/guide/intro_to_modules.ipynb
index 7393e1544dc..79bbe89ca56 100644
--- a/site/en/guide/intro_to_modules.ipynb
+++ b/site/en/guide/intro_to_modules.ipynb
@@ -91,6 +91,7 @@
"outputs": [],
"source": [
"import tensorflow as tf\n",
+ "import keras\n",
"from datetime import datetime\n",
"\n",
"%load_ext tensorboard"
@@ -102,10 +103,12 @@
"id": "yt5HEbsYAbw1"
},
"source": [
- "## Defining models and layers in TensorFlow\n",
+ "## TensorFlow Modules\n",
"\n",
"Most models are made of layers. Layers are functions with a known mathematical structure that can be reused and have trainable variables. In TensorFlow, most high-level implementations of layers and models, such as Keras or [Sonnet](https://github.com/deepmind/sonnet), are built on the same foundational class: `tf.Module`.\n",
"\n",
+ "### Building Modules\n",
+ "\n",
"Here's an example of a very simple `tf.Module` that operates on a scalar tensor:\n"
]
},
@@ -337,7 +340,7 @@
"id": "JOLVVBT8J_dl"
},
"source": [
- "## Saving weights\n",
+ "### Saving weights\n",
"\n",
"You can save a `tf.Module` as both a [checkpoint](./checkpoint.ipynb) and a [SavedModel](./saved_model.ipynb).\n",
"\n",
@@ -403,7 +406,7 @@
"id": "4eGaNiQWcK4j"
},
"source": [
- "During distributed (multi-machine) training they can be sharded, which is why they are numbered (e.g., '00000-of-00001'). In this case, though, there is only have one shard.\n",
+ "During distributed (multi-machine) training they can be sharded, which is why they are numbered (e.g., '00000-of-00001'). In this case, though, there is only one shard.\n",
"\n",
"When you load models back in, you overwrite the values in your Python object."
]
@@ -439,7 +442,7 @@
"id": "pSZebVuWxDXu"
},
"source": [
- "## Saving functions\n",
+ "### Saving functions\n",
"\n",
"TensorFlow can run models without the original Python objects, as demonstrated by [TensorFlow Serving](https://tensorflow.org/tfx) and [TensorFlow Lite](https://tensorflow.org/lite), even when you download a trained model from [TensorFlow Hub](https://tensorflow.org/hub).\n",
"\n",
@@ -699,6 +702,25 @@
"In this section, you will examine how Keras uses `tf.Module`. A complete user guide to Keras models can be found in the [Keras guide](https://www.tensorflow.org/guide/keras/sequential_model).\n"
]
},
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "ds08u3touwe4t"
+ },
+ "source": [
+ "Keras layers and models have a lot more extra features including:\n",
+ "\n",
+ "* Optional losses\n",
+ "* Support for [metrics](https://keras.io/api/layers/base_layer/#add_metric-method)\n",
+ "* Built-in support for an optional `training` argument to differentiate between training and inference use\n",
+ "* Saving and restoring python objects instead of just black-box functions\n",
+ "* `get_config` and `from_config` methods that allow you to accurately store configurations to allow model cloning in Python\n",
+ "\n",
+ "These features allow for far more complex models through subclassing, such as a custom GAN or a Variational AutoEncoder (VAE) model. Read about them in the [full guide](./keras/custom_layers_and_models.ipynb) to custom layers and models.\n",
+ "\n",
+ "Keras models also come with extra functionality that makes them easy to train, evaluate, load, save, and even train on multiple machines."
+ ]
+ },
{
"cell_type": "markdown",
"metadata": {
@@ -874,22 +896,6 @@
" print(\"Failed:\", e)"
]
},
- {
- "cell_type": "markdown",
- "metadata": {
- "id": "YnporXiudF1I"
- },
- "source": [
- "Keras layers have a lot more extra features including:\n",
- "\n",
- "* Optional losses\n",
- "* Support for metrics\n",
- "* Built-in support for an optional `training` argument to differentiate between training and inference use\n",
- "* `get_config` and `from_config` methods that allow you to accurately store configurations to allow model cloning in Python\n",
- "\n",
- "Read about them in the [full guide](./keras/custom_layers_and_models.ipynb) to custom layers and models."
- ]
- },
{
"cell_type": "markdown",
"metadata": {
@@ -900,7 +906,7 @@
"\n",
"You can define your model as nested Keras layers.\n",
"\n",
- "However, Keras also provides a full-featured model class called `tf.keras.Model`. It inherits from `tf.keras.layers.Layer`, so a Keras model can be used, nested, and saved in the same way as Keras layers. Keras models come with extra functionality that makes them easy to train, evaluate, load, save, and even train on multiple machines.\n",
+ "However, Keras also provides a full-featured model class called `tf.keras.Model`. It inherits from `tf.keras.layers.Layer`, so a Keras model can be used and nested in the same way as Keras layers. Keras models come with extra functionality that makes them easy to train, evaluate, load, save, and even train on multiple machines.\n",
"\n",
"You can define the `SequentialModule` from above with nearly identical code, again converting `__call__` to `call()` and changing the parent:"
]
@@ -913,6 +919,7 @@
},
"outputs": [],
"source": [
+ "@keras.saving.register_keras_serializable()\n",
"class MySequentialModel(tf.keras.Model):\n",
" def __init__(self, name=None, **kwargs):\n",
" super().__init__(**kwargs)\n",
@@ -938,7 +945,7 @@
"source": [
"All the same features are available, including tracking variables and submodules.\n",
"\n",
- "Note: To emphasize the note above, a raw `tf.Module` nested inside a Keras layer or model will not get its variables collected for training or saving. Instead, nest Keras layers inside of Keras layers."
+ "Note: A raw `tf.Module` nested inside a Keras layer or model will not get its variables collected for training or saving. Instead, nest Keras layers inside of Keras layers."
]
},
{
@@ -1022,11 +1029,9 @@
"id": "qI9aXLnaHEFF"
},
"source": [
- "## Saving Keras models\n",
- "\n",
- "Keras models can be checkpointed, and that will look the same as `tf.Module`.\n",
+ "### Saving Keras models\n",
"\n",
- "Keras models can also be saved with `tf.saved_model.save()`, as they are modules. However, Keras models have convenience methods and other functionality:"
+ "Keras models have their own specialized zip archive saving format, marked by the `.keras` extension. When calling `tf.keras.Model.save`, add a `.keras` extension to the filename. For example:"
]
},
{
@@ -1037,7 +1042,7 @@
},
"outputs": [],
"source": [
- "my_sequential_model.save(\"exname_of_file\")"
+ "my_sequential_model.save(\"exname_of_file.keras\")"
]
},
{
@@ -1057,7 +1062,7 @@
},
"outputs": [],
"source": [
- "reconstructed_model = tf.keras.models.load_model(\"exname_of_file\")"
+ "reconstructed_model = tf.keras.models.load_model(\"exname_of_file.keras\")"
]
},
{
@@ -1066,7 +1071,7 @@
"id": "EA7P_MNvpviZ"
},
"source": [
- "Keras `SavedModels` also save metric, loss, and optimizer states.\n",
+ "Keras zip archives — `.keras` files — also save metric, loss, and optimizer states.\n",
"\n",
"This reconstructed model can be used and will produce the same result when called on the same data:"
]
@@ -1082,6 +1087,17 @@
"reconstructed_model(tf.constant([[2.0, 2.0, 2.0]]))"
]
},
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "seLIUG2354s"
+ },
+ "source": [
+ "### Checkpointing Keras models\n",
+ "\n",
+ "Keras models can also be checkpointed, and that will look the same as `tf.Module`."
+ ]
+ },
{
"cell_type": "markdown",
"metadata": {
diff --git a/site/en/guide/jax2tf.ipynb b/site/en/guide/jax2tf.ipynb
new file mode 100644
index 00000000000..613c622658d
--- /dev/null
+++ b/site/en/guide/jax2tf.ipynb
@@ -0,0 +1,851 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "ckM5wJMsNTYL"
+ },
+ "source": [
+ "##### Copyright 2023 The TensorFlow Authors."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "cellView": "form",
+ "id": "NKvERjPVNWxu"
+ },
+ "outputs": [],
+ "source": [
+ "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+ "# you may not use this file except in compliance with the License.\n",
+ "# You may obtain a copy of the License at\n",
+ "#\n",
+ "# https://www.apache.org/licenses/LICENSE-2.0\n",
+ "#\n",
+ "# Unless required by applicable law or agreed to in writing, software\n",
+ "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+ "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+ "# See the License for the specific language governing permissions and\n",
+ "# limitations under the License."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "bqePLdDjNhNk"
+ },
+ "source": [
+ "# Import a JAX model using JAX2TF"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "gw3w46yhNiK_"
+ },
+ "source": [
+ "
"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "IyrsY3uTOmPY"
+ },
+ "source": [
+ "This notebook provides a complete, runnable example of creating a model using [JAX](https://jax.readthedocs.io/en/latest/) and bringing it into TensorFlow to continue training. This is made possible by [JAX2TF](https://github.com/google/jax/tree/main/jax/experimental/jax2tf), a lightweight API that provides a pathway from the JAX ecosystem to the TensorFlow ecosystem. \n",
+ "\n",
+ "JAX is a high-performance array computing library. To create the model, this notebook uses [Flax](https://flax.readthedocs.io/en/latest/), a neural network library for JAX. To train it, it uses [Optax](https://optax.readthedocs.io), an optimization library for JAX.\n",
+ "\n",
+ "If you're a researcher using JAX, JAX2TF gives you a path to production using TensorFlow's proven tools.\n",
+ "\n",
+ "There are many ways this can be useful, here are just a few:\n",
+ "\n",
+ "* Inference: Taking a model written for JAX and deploying it either on a server using TF Serving, on-device using TFLite, or on the web using TensorFlow.js. \n",
+ "\n",
+ "* Fine-tuning: Taking a model that was trained using JAX, you can bring its components to TF using JAX2TF, and continue training it in TensorFlow with your existing training data and setup.\n",
+ "\n",
+ "* Fusion: Combining parts of models that were trained using JAX with those trained using TensorFlow, for maximum flexibility.\n",
+ "\n",
+ "The key to enabling this kind of interoperation between JAX and TensorFlow is `jax2tf.convert`, which takes in model components created on top of JAX (your loss function, prediction function, etc) and creates equivalent representations of them as TensorFlow functions, which can then be exported as a TensorFlow SavedModel."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "G6rtu96yOepm"
+ },
+ "source": [
+ "## Setup\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "9yqxfHzr0LPF"
+ },
+ "outputs": [],
+ "source": [
+ "import tensorflow as tf\n",
+ "import numpy as np\n",
+ "import jax\n",
+ "import jax.numpy as jnp\n",
+ "import flax\n",
+ "import optax\n",
+ "import os\n",
+ "from matplotlib import pyplot as plt\n",
+ "from jax.experimental import jax2tf\n",
+ "from threading import Lock # Only used in the visualization utility.\n",
+ "from functools import partial"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "SDnTaZO0r872"
+ },
+ "outputs": [],
+ "source": [
+ "# Needed for TensorFlow and JAX to coexist in GPU memory.\n",
+ "os.environ['XLA_PYTHON_CLIENT_PREALLOCATE'] = \"false\"\n",
+ "gpus = tf.config.list_physical_devices('GPU')\n",
+ "if gpus:\n",
+ " try:\n",
+ " for gpu in gpus:\n",
+ " tf.config.experimental.set_memory_growth(gpu, True)\n",
+ " except RuntimeError as e:\n",
+ " # Memory growth must be set before GPUs have been initialized.\n",
+ " print(e)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "cellView": "form",
+ "id": "BXOjCNJxDLil"
+ },
+ "outputs": [],
+ "source": [
+ "#@title Visualization utilities\n",
+ "\n",
+ "plt.rcParams[\"figure.figsize\"] = (20,8)\n",
+ "\n",
+ "# The utility for displaying training and validation curves.\n",
+ "def display_train_curves(loss, avg_loss, eval_loss, eval_accuracy, epochs, steps_per_epochs, ignore_first_n=10):\n",
+ "\n",
+ " ignore_first_n_epochs = int(ignore_first_n/steps_per_epochs)\n",
+ "\n",
+ " # The losses.\n",
+ " ax = plt.subplot(121)\n",
+ " if loss is not None:\n",
+ " x = np.arange(len(loss)) / steps_per_epochs #* epochs\n",
+ " ax.plot(x, loss)\n",
+ " ax.plot(range(1, epochs+1), avg_loss, \"-o\", linewidth=3)\n",
+ " ax.plot(range(1, epochs+1), eval_loss, \"-o\", linewidth=3)\n",
+ " ax.set_title('Loss')\n",
+ " ax.set_ylabel('loss')\n",
+ " ax.set_xlabel('epoch')\n",
+ " if loss is not None:\n",
+ " ax.set_ylim(0, np.max(loss[ignore_first_n:]))\n",
+ " ax.legend(['train', 'avg train', 'eval'])\n",
+ " else:\n",
+ " ymin = np.min(avg_loss[ignore_first_n_epochs:])\n",
+ " ymax = np.max(avg_loss[ignore_first_n_epochs:])\n",
+ " ax.set_ylim(ymin-(ymax-ymin)/10, ymax+(ymax-ymin)/10)\n",
+ " ax.legend(['avg train', 'eval'])\n",
+ "\n",
+ " # The accuracy.\n",
+ " ax = plt.subplot(122)\n",
+ " ax.set_title('Eval Accuracy')\n",
+ " ax.set_ylabel('accuracy')\n",
+ " ax.set_xlabel('epoch')\n",
+ " ymin = np.min(eval_accuracy[ignore_first_n_epochs:])\n",
+ " ymax = np.max(eval_accuracy[ignore_first_n_epochs:])\n",
+ " ax.set_ylim(ymin-(ymax-ymin)/10, ymax+(ymax-ymin)/10)\n",
+ " ax.plot(range(1, epochs+1), eval_accuracy, \"-o\", linewidth=3)\n",
+ "\n",
+ "class Progress:\n",
+ " \"\"\"Text mode progress bar.\n",
+ " Usage:\n",
+ " p = Progress(30)\n",
+ " p.step()\n",
+ " p.step()\n",
+ " p.step(reset=True) # to restart form 0%\n",
+ " The progress bar displays a new header at each restart.\"\"\"\n",
+ " def __init__(self, maxi, size=100, msg=\"\"):\n",
+ " \"\"\"\n",
+ " :param maxi: the number of steps required to reach 100%\n",
+ " :param size: the number of characters taken on the screen by the progress bar\n",
+ " :param msg: the message displayed in the header of the progress bar\n",
+ " \"\"\"\n",
+ " self.maxi = maxi\n",
+ " self.p = self.__start_progress(maxi)() # `()`: to get the iterator from the generator.\n",
+ " self.header_printed = False\n",
+ " self.msg = msg\n",
+ " self.size = size\n",
+ " self.lock = Lock()\n",
+ "\n",
+ " def step(self, reset=False):\n",
+ " with self.lock:\n",
+ " if reset:\n",
+ " self.__init__(self.maxi, self.size, self.msg)\n",
+ " if not self.header_printed:\n",
+ " self.__print_header()\n",
+ " next(self.p)\n",
+ "\n",
+ " def __print_header(self):\n",
+ " print()\n",
+ " format_string = \"0%{: ^\" + str(self.size - 6) + \"}100%\"\n",
+ " print(format_string.format(self.msg))\n",
+ " self.header_printed = True\n",
+ "\n",
+ " def __start_progress(self, maxi):\n",
+ " def print_progress():\n",
+ " # Bresenham's algorithm. Yields the number of dots printed.\n",
+ " # This will always print 100 dots in max invocations.\n",
+ " dx = maxi\n",
+ " dy = self.size\n",
+ " d = dy - dx\n",
+ " for x in range(maxi):\n",
+ " k = 0\n",
+ " while d >= 0:\n",
+ " print('=', end=\"\", flush=True)\n",
+ " k += 1\n",
+ " d -= dx\n",
+ " d += dy\n",
+ " yield k\n",
+ " # Keep yielding the last result if there are too many steps.\n",
+ " while True:\n",
+ " yield k\n",
+ "\n",
+ " return print_progress"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "6xgS_8nDDIu8"
+ },
+ "source": [
+ "## Download and prepare the MNIST dataset"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "nbN7rmuF0VFB"
+ },
+ "outputs": [],
+ "source": [
+ "(x_train, train_labels), (x_test, test_labels) = tf.keras.datasets.mnist.load_data()\n",
+ "\n",
+ "train_data = tf.data.Dataset.from_tensor_slices((x_train, train_labels))\n",
+ "train_data = train_data.map(lambda x,y: (tf.expand_dims(tf.cast(x, tf.float32)/255.0, axis=-1),\n",
+ " tf.one_hot(y, depth=10)))\n",
+ "\n",
+ "BATCH_SIZE = 256\n",
+ "train_data = train_data.batch(BATCH_SIZE, drop_remainder=True)\n",
+ "train_data = train_data.cache()\n",
+ "train_data = train_data.shuffle(5000, reshuffle_each_iteration=True)\n",
+ "\n",
+ "test_data = tf.data.Dataset.from_tensor_slices((x_test, test_labels))\n",
+ "test_data = test_data.map(lambda x,y: (tf.expand_dims(tf.cast(x, tf.float32)/255.0, axis=-1),\n",
+ " tf.one_hot(y, depth=10)))\n",
+ "test_data = test_data.batch(10000)\n",
+ "test_data = test_data.cache()\n",
+ "\n",
+ "(one_batch, one_batch_labels) = next(iter(train_data)) # just one batch\n",
+ "(all_test_data, all_test_labels) = next(iter(test_data)) # all in one batch since batch size is 10000"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "LuZTo7SM3W_n"
+ },
+ "source": [
+ "## Configure training\n",
+ "This notebook will create and train a simple model for demonstration purposes."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "3vbKB4yZ3aTL"
+ },
+ "outputs": [],
+ "source": [
+ "# Training hyperparameters.\n",
+ "JAX_EPOCHS = 3\n",
+ "TF_EPOCHS = 7\n",
+ "STEPS_PER_EPOCH = len(train_labels)//BATCH_SIZE\n",
+ "LEARNING_RATE = 0.01\n",
+ "LEARNING_RATE_EXP_DECAY = 0.6\n",
+ "\n",
+ "# The learning rate schedule for JAX (with Optax).\n",
+ "jlr_decay = optax.exponential_decay(LEARNING_RATE, transition_steps=STEPS_PER_EPOCH, decay_rate=LEARNING_RATE_EXP_DECAY, staircase=True)\n",
+ "\n",
+ "# THe learning rate schedule for TensorFlow.\n",
+ "tflr_decay = tf.keras.optimizers.schedules.ExponentialDecay(initial_learning_rate=LEARNING_RATE, decay_steps=STEPS_PER_EPOCH, decay_rate=LEARNING_RATE_EXP_DECAY, staircase=True)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "Od3sMwQxtC34"
+ },
+ "source": [
+ "## Create the model using Flax"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "-ybqQF2zd2QX"
+ },
+ "outputs": [],
+ "source": [
+ "class ConvModel(flax.linen.Module):\n",
+ "\n",
+ " @flax.linen.compact\n",
+ " def __call__(self, x, train):\n",
+ " x = flax.linen.Conv(features=12, kernel_size=(3,3), padding=\"SAME\", use_bias=False)(x)\n",
+ " x = flax.linen.BatchNorm(use_running_average=not train, use_scale=False, use_bias=True)(x)\n",
+ " x = x.reshape((x.shape[0], -1)) # flatten\n",
+ " x = flax.linen.Dense(features=200, use_bias=True)(x)\n",
+ " x = flax.linen.BatchNorm(use_running_average=not train, use_scale=False, use_bias=True)(x)\n",
+ " x = flax.linen.Dropout(rate=0.3, deterministic=not train)(x)\n",
+ " x = flax.linen.relu(x)\n",
+ " x = flax.linen.Dense(features=10)(x)\n",
+ " #x = flax.linen.log_softmax(x)\n",
+ " return x\n",
+ "\n",
+ " # JAX differentiation requires a function `f(params, other_state, data, labels)` -> `loss` (as a single number).\n",
+ " # `jax.grad` will differentiate it against the fist argument.\n",
+ " # The user must split trainable and non-trainable variables into `params` and `other_state`.\n",
+ " # Must pass a different RNG key each time for the dropout mask to be different.\n",
+ " def loss(self, params, other_state, rng, data, labels, train):\n",
+ " logits, batch_stats = self.apply({'params': params, **other_state},\n",
+ " data,\n",
+ " mutable=['batch_stats'],\n",
+ " rngs={'dropout': rng},\n",
+ " train=train)\n",
+ " # The loss averaged across the batch dimension.\n",
+ " loss = optax.softmax_cross_entropy(logits, labels).mean()\n",
+ " return loss, batch_stats\n",
+ "\n",
+ " def predict(self, state, data):\n",
+ " logits = self.apply(state, data, train=False) # predict and accuracy disable dropout and use accumulated batch norm stats (train=False)\n",
+ " probabilities = flax.linen.log_softmax(logits)\n",
+ " return probabilities\n",
+ "\n",
+ " def accuracy(self, state, data, labels):\n",
+ " probabilities = self.predict(state, data)\n",
+ " predictions = jnp.argmax(probabilities, axis=-1)\n",
+ " dense_labels = jnp.argmax(labels, axis=-1)\n",
+ " accuracy = jnp.equal(predictions, dense_labels).mean()\n",
+ " return accuracy"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "7Cr0FRNFtHN4"
+ },
+ "source": [
+ "## Write the training step function"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "tmDwApcpgZzw"
+ },
+ "outputs": [],
+ "source": [
+ "# The training step.\n",
+ "@partial(jax.jit, static_argnums=[0]) # this forces jax.jit to recompile for every new model\n",
+ "def train_step(model, state, optimizer_state, rng, data, labels):\n",
+ "\n",
+ " other_state, params = state.pop('params') # differentiate only against 'params' which represents trainable variables\n",
+ " (loss, batch_stats), grads = jax.value_and_grad(model.loss, has_aux=True)(params, other_state, rng, data, labels, train=True)\n",
+ "\n",
+ " updates, optimizer_state = optimizer.update(grads, optimizer_state)\n",
+ " params = optax.apply_updates(params, updates)\n",
+ " new_state = state.copy(add_or_replace={**batch_stats, 'params': params})\n",
+ "\n",
+ " rng, _ = jax.random.split(rng)\n",
+ "\n",
+ " return new_state, optimizer_state, rng, loss"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "Zr16g6NzV4O9"
+ },
+ "source": [
+ "## Write the training loop"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "zbl5w-KUV7Qw"
+ },
+ "outputs": [],
+ "source": [
+ "def train(model, state, optimizer_state, train_data, epochs, losses, avg_losses, eval_losses, eval_accuracies):\n",
+ " p = Progress(STEPS_PER_EPOCH)\n",
+ " rng = jax.random.PRNGKey(0)\n",
+ " for epoch in range(epochs):\n",
+ "\n",
+ " # This is where the learning rate schedule state is stored in the optimizer state.\n",
+ " optimizer_step = optimizer_state[1].count\n",
+ "\n",
+ " # Run an epoch of training.\n",
+ " for step, (data, labels) in enumerate(train_data):\n",
+ " p.step(reset=(step==0))\n",
+ " state, optimizer_state, rng, loss = train_step(model, state, optimizer_state, rng, data.numpy(), labels.numpy())\n",
+ " losses.append(loss)\n",
+ " avg_loss = np.mean(losses[-step:])\n",
+ " avg_losses.append(avg_loss)\n",
+ "\n",
+ " # Run one epoch of evals (10,000 test images in a single batch).\n",
+ " other_state, params = state.pop('params')\n",
+ " # Gotcha: must discard modified batch_stats here\n",
+ " eval_loss, _ = model.loss(params, other_state, rng, all_test_data.numpy(), all_test_labels.numpy(), train=False)\n",
+ " eval_losses.append(eval_loss)\n",
+ " eval_accuracy = model.accuracy(state, all_test_data.numpy(), all_test_labels.numpy())\n",
+ " eval_accuracies.append(eval_accuracy)\n",
+ "\n",
+ " print(\"\\nEpoch\", epoch, \"train loss:\", avg_loss, \"eval loss:\", eval_loss, \"eval accuracy\", eval_accuracy, \"lr:\", jlr_decay(optimizer_step))\n",
+ "\n",
+ " return state, optimizer_state"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "DGB3W5g0Wt1H"
+ },
+ "source": [
+ "## Create the model and the optimizer (with Optax)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "mW5mkmCWtN8W"
+ },
+ "outputs": [],
+ "source": [
+ "# The model.\n",
+ "model = ConvModel()\n",
+ "state = model.init({'params':jax.random.PRNGKey(0), 'dropout':jax.random.PRNGKey(0)}, one_batch, train=True) # Flax allows a separate RNG for \"dropout\"\n",
+ "\n",
+ "# The optimizer.\n",
+ "optimizer = optax.adam(learning_rate=jlr_decay) # Gotcha: it does not seem to be possible to pass just a callable as LR, must be an Optax Schedule\n",
+ "optimizer_state = optimizer.init(state['params'])\n",
+ "\n",
+ "losses=[]\n",
+ "avg_losses=[]\n",
+ "eval_losses=[]\n",
+ "eval_accuracies=[]"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "FJdsKghBNF"
+ },
+ "source": [
+ "## Train the model"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "nmcofTTBZSIb"
+ },
+ "outputs": [],
+ "source": [
+ "new_state, new_optimizer_state = train(model, state, optimizer_state, train_data, JAX_EPOCHS+TF_EPOCHS, losses, avg_losses, eval_losses, eval_accuracies)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "n_20vgvDXB5r"
+ },
+ "outputs": [],
+ "source": [
+ "display_train_curves(losses, avg_losses, eval_losses, eval_accuracies, len(eval_losses), STEPS_PER_EPOCH, ignore_first_n=1*STEPS_PER_EPOCH)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "0lT3cdENCBzL"
+ },
+ "source": [
+ "## Partially train the model\n",
+ "\n",
+ "You will continue training the model in TensorFlow shortly."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "KT-xqj5N7C6L"
+ },
+ "outputs": [],
+ "source": [
+ "model = ConvModel()\n",
+ "state = model.init({'params':jax.random.PRNGKey(0), 'dropout':jax.random.PRNGKey(0)}, one_batch, train=True) # Flax allows a separate RNG for \"dropout\"\n",
+ "\n",
+ "# The optimizer.\n",
+ "optimizer = optax.adam(learning_rate=jlr_decay) # LR must be an Optax LR Schedule\n",
+ "optimizer_state = optimizer.init(state['params'])\n",
+ "\n",
+ "losses, avg_losses, eval_losses, eval_accuracies = [], [], [], []"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "oa362HMDbzDE"
+ },
+ "outputs": [],
+ "source": [
+ "state, optimizer_state = train(model, state, optimizer_state, train_data, JAX_EPOCHS, losses, avg_losses, eval_losses, eval_accuracies)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "0IyZtUPPCt0y"
+ },
+ "outputs": [],
+ "source": [
+ "display_train_curves(losses, avg_losses, eval_losses, eval_accuracies, len(eval_losses), STEPS_PER_EPOCH, ignore_first_n=1*STEPS_PER_EPOCH)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "uNtlSaOCCumB"
+ },
+ "source": [
+ "## Save just enough for inference\n",
+ "\n",
+ "If your goal is to deploy your JAX model (so you can run inference using `model.predict()`), simply exporting it to [SavedModel](https://www.tensorflow.org/guide/saved_model) is sufficient. This section demonstrates how to accomplish that."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "O653B3-5H8FL"
+ },
+ "outputs": [],
+ "source": [
+ "# Test data with a different batch size to test polymorphic shapes.\n",
+ "x, y = next(iter(train_data.unbatch().batch(13)))\n",
+ "\n",
+ "m = tf.Module()\n",
+ "# Wrap the JAX state in `tf.Variable` (needed when calling the converted JAX function.\n",
+ "state_vars = tf.nest.map_structure(tf.Variable, state)\n",
+ "# Keep the wrapped state as flat list (needed in TensorFlow fine-tuning).\n",
+ "m.vars = tf.nest.flatten(state_vars)\n",
+ "# Convert the desired JAX function (`model.predict`).\n",
+ "predict_fn = jax2tf.convert(model.predict, polymorphic_shapes=[\"...\", \"(b, 28, 28, 1)\"])\n",
+ "# Wrap the converted function in `tf.function` with the correct `tf.TensorSpec` (necessary for dynamic shapes to work).\n",
+ "@tf.function(autograph=False, input_signature=[tf.TensorSpec(shape=(None, 28, 28, 1), dtype=tf.float32)])\n",
+ "def predict(data):\n",
+ " return predict_fn(state_vars, data)\n",
+ "m.predict = predict\n",
+ "tf.saved_model.save(m, \"./\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "8HFx67zStgvo"
+ },
+ "outputs": [],
+ "source": [
+ "# Test the converted function.\n",
+ "print(\"Converted function predictions:\", np.argmax(m.predict(x).numpy(), axis=-1))\n",
+ "# Reload the model.\n",
+ "reloaded_model = tf.saved_model.load(\"./\")\n",
+ "# Test the reloaded converted function (the result should be the same).\n",
+ "print(\"Reloaded function predictions:\", np.argmax(reloaded_model.predict(x).numpy(), axis=-1))"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "eEk8wv4HJu94"
+ },
+ "source": [
+ "## Save everything\n",
+ "If your goal is a comprehensive export (useful if you're planning on brining the model into TensorFlow for fine-tuning, fusion, etc), this section demonstrates how to save the model so you can access methods including:\n",
+ "\n",
+ " - model.predict\n",
+ " - model.accuracy\n",
+ " - model.loss (including train=True/False bool, RNG for dropout and BatchNorm state updates)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "9mty52pmvDDp"
+ },
+ "outputs": [],
+ "source": [
+ "from collections import abc\n",
+ "\n",
+ "def _fix_frozen(d):\n",
+ " \"\"\"Changes any mappings (e.g. frozendict) back to dict.\"\"\"\n",
+ " if isinstance(d, list):\n",
+ " return [_fix_frozen(v) for v in d]\n",
+ " elif isinstance(d, tuple):\n",
+ " return tuple(_fix_frozen(v) for v in d)\n",
+ " elif not isinstance(d, abc.Mapping):\n",
+ " return d\n",
+ " d = dict(d)\n",
+ " for k, v in d.items():\n",
+ " d[k] = _fix_frozen(v)\n",
+ " return d"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "3HEsKNXbCwXw"
+ },
+ "outputs": [],
+ "source": [
+ "class TFModel(tf.Module):\n",
+ " def __init__(self, state, model):\n",
+ " super().__init__()\n",
+ "\n",
+ " # Special care needed for the train=True/False parameter in the loss\n",
+ " @jax.jit\n",
+ " def loss_with_train_bool(state, rng, data, labels, train):\n",
+ " other_state, params = state.pop('params')\n",
+ " loss, batch_stats = jax.lax.cond(train,\n",
+ " lambda state, data, labels: model.loss(params, other_state, rng, data, labels, train=True),\n",
+ " lambda state, data, labels: model.loss(params, other_state, rng, data, labels, train=False),\n",
+ " state, data, labels)\n",
+ " # must use JAX to split the RNG, therefore, must do it in a @jax.jit function\n",
+ " new_rng, _ = jax.random.split(rng)\n",
+ " return loss, batch_stats, new_rng\n",
+ "\n",
+ " self.state_vars = tf.nest.map_structure(tf.Variable, state)\n",
+ " self.vars = tf.nest.flatten(self.state_vars)\n",
+ " self.jax_rng = tf.Variable(jax.random.PRNGKey(0))\n",
+ "\n",
+ " self.loss_fn = jax2tf.convert(loss_with_train_bool, polymorphic_shapes=[\"...\", \"...\", \"(b, 28, 28, 1)\", \"(b, 10)\", \"...\"])\n",
+ " self.accuracy_fn = jax2tf.convert(model.accuracy, polymorphic_shapes=[\"...\", \"(b, 28, 28, 1)\", \"(b, 10)\"])\n",
+ " self.predict_fn = jax2tf.convert(model.predict, polymorphic_shapes=[\"...\", \"(b, 28, 28, 1)\"])\n",
+ "\n",
+ " # Must specify TensorSpec manually for variable batch size to work\n",
+ " @tf.function(autograph=False, input_signature=[tf.TensorSpec(shape=(None, 28, 28, 1), dtype=tf.float32)])\n",
+ " def predict(self, data):\n",
+ " # Make sure the TfModel.predict function implicitly use self.state_vars and not the JAX state directly\n",
+ " # otherwise, all model weights would be embedded in the TF graph as constants.\n",
+ " return self.predict_fn(self.state_vars, data)\n",
+ "\n",
+ " @tf.function(input_signature=[tf.TensorSpec(shape=(None, 28, 28, 1), dtype=tf.float32),\n",
+ " tf.TensorSpec(shape=(None, 10), dtype=tf.float32)],\n",
+ " autograph=False)\n",
+ " def train_loss(self, data, labels):\n",
+ " loss, batch_stats, new_rng = self.loss_fn(self.state_vars, self.jax_rng, data, labels, True)\n",
+ " # update batch norm stats\n",
+ " flat_vars = tf.nest.flatten(self.state_vars['batch_stats'])\n",
+ " flat_values = tf.nest.flatten(batch_stats['batch_stats'])\n",
+ " for var, val in zip(flat_vars, flat_values):\n",
+ " var.assign(val)\n",
+ " # update RNG\n",
+ " self.jax_rng.assign(new_rng)\n",
+ " return loss\n",
+ "\n",
+ " @tf.function(input_signature=[tf.TensorSpec(shape=(None, 28, 28, 1), dtype=tf.float32),\n",
+ " tf.TensorSpec(shape=(None, 10), dtype=tf.float32)],\n",
+ " autograph=False)\n",
+ " def eval_loss(self, data, labels):\n",
+ " loss, batch_stats, new_rng = self.loss_fn(self.state_vars, self.jax_rng, data, labels, False)\n",
+ " return loss\n",
+ "\n",
+ " @tf.function(input_signature=[tf.TensorSpec(shape=(None, 28, 28, 1), dtype=tf.float32),\n",
+ " tf.TensorSpec(shape=(None, 10), dtype=tf.float32)],\n",
+ " autograph=False)\n",
+ " def accuracy(self, data, labels):\n",
+ " return self.accuracy_fn(self.state_vars, data, labels)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "znJrAVpcxO9u"
+ },
+ "outputs": [],
+ "source": [
+ "# Instantiate the model.\n",
+ "tf_model = TFModel(state, model)\n",
+ "\n",
+ "# Save the model.\n",
+ "tf.saved_model.save(tf_model, \"./\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "Y02DHEwTjNzV"
+ },
+ "source": [
+ "## Reload the model"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "i75yS3v2jPpM"
+ },
+ "outputs": [],
+ "source": [
+ "reloaded_model = tf.saved_model.load(\"./\")\n",
+ "\n",
+ "# Test if it works and that the batch size is indeed variable.\n",
+ "x,y = next(iter(train_data.unbatch().batch(13)))\n",
+ "print(np.argmax(reloaded_model.predict(x).numpy(), axis=-1))\n",
+ "x,y = next(iter(train_data.unbatch().batch(20)))\n",
+ "print(np.argmax(reloaded_model.predict(x).numpy(), axis=-1))\n",
+ "\n",
+ "print(reloaded_model.accuracy(one_batch, one_batch_labels))\n",
+ "print(reloaded_model.accuracy(all_test_data, all_test_labels))"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "DiwEAwQmlx1x"
+ },
+ "source": [
+ "## Continue training the converted JAX model in TensorFlow"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "MubFcO_jl2vE"
+ },
+ "outputs": [],
+ "source": [
+ "optimizer = tf.keras.optimizers.Adam(learning_rate=tflr_decay)\n",
+ "\n",
+ "# Set the iteration step for the learning rate to resume from where it left off in JAX.\n",
+ "optimizer.iterations.assign(len(eval_losses)*STEPS_PER_EPOCH)\n",
+ "\n",
+ "p = Progress(STEPS_PER_EPOCH)\n",
+ "\n",
+ "for epoch in range(JAX_EPOCHS, JAX_EPOCHS+TF_EPOCHS):\n",
+ "\n",
+ " # This is where the learning rate schedule state is stored in the optimizer state.\n",
+ " optimizer_step = optimizer.iterations\n",
+ "\n",
+ " for step, (data, labels) in enumerate(train_data):\n",
+ " p.step(reset=(step==0))\n",
+ " with tf.GradientTape() as tape:\n",
+ " #loss = reloaded_model.loss(data, labels, True)\n",
+ " loss = reloaded_model.train_loss(data, labels)\n",
+ " grads = tape.gradient(loss, reloaded_model.vars)\n",
+ " optimizer.apply_gradients(zip(grads, reloaded_model.vars))\n",
+ " losses.append(loss)\n",
+ " avg_loss = np.mean(losses[-step:])\n",
+ " avg_losses.append(avg_loss)\n",
+ "\n",
+ " eval_loss = reloaded_model.eval_loss(all_test_data.numpy(), all_test_labels.numpy()).numpy()\n",
+ " eval_losses.append(eval_loss)\n",
+ " eval_accuracy = reloaded_model.accuracy(all_test_data.numpy(), all_test_labels.numpy()).numpy()\n",
+ " eval_accuracies.append(eval_accuracy)\n",
+ "\n",
+ " print(\"\\nEpoch\", epoch, \"train loss:\", avg_loss, \"eval loss:\", eval_loss, \"eval accuracy\", eval_accuracy, \"lr:\", tflr_decay(optimizer.iterations).numpy())"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "50V1FSmI6UTk"
+ },
+ "outputs": [],
+ "source": [
+ "display_train_curves(losses, avg_losses, eval_losses, eval_accuracies, len(eval_losses), STEPS_PER_EPOCH, ignore_first_n=2*STEPS_PER_EPOCH)\n",
+ "\n",
+ "# The loss takes a hit when the training restarts, but does not go back to random levels.\n",
+ "# This is likely caused by the optimizer momentum being reinitialized."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "L7lSziW0K0ny"
+ },
+ "source": [
+ "## Next steps\n",
+ "You can learn more about [JAX](https://jax.readthedocs.io/en/latest/index.html) and [Flax](https://flax.readthedocs.io/en/latest) on their documentation websites which contain detailed guides and examples. If you're new to JAX, be sure to explore the [JAX 101 tutorials](https://jax.readthedocs.io/en/latest/jax-101/index.html), and check out the [Flax quickstart](https://flax.readthedocs.io/en/latest/getting_started.html). To learn more about converting JAX models to TensorFlow format, check out the [jax2tf](https://github.com/google/jax/tree/main/jax/experimental/jax2tf) utility on GitHub. If you're interested in converting JAX models to run in the browser with TensorFlow.js, visit [JAX on the Web with TensorFlow.js](https://blog.tensorflow.org/2022/08/jax-on-web-with-tensorflowjs.html). If you'd like to prepare JAX models to run in TensorFLow Lite, visit the [JAX Model Conversion For TFLite](https://www.tensorflow.org/lite/examples/jax_conversion/overview) guide."
+ ]
+ }
+ ],
+ "metadata": {
+ "accelerator": "GPU",
+ "colab": {
+ "name": "jax2tf.ipynb",
+ "toc_visible": true
+ },
+ "kernelspec": {
+ "display_name": "Python 3",
+ "name": "python3"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}
diff --git a/site/en/guide/keras.md b/site/en/guide/keras.md
new file mode 100644
index 00000000000..3dee7be3aa1
--- /dev/null
+++ b/site/en/guide/keras.md
@@ -0,0 +1,135 @@
+# Keras: The high-level API for TensorFlow
+
+Keras is the high-level API of the TensorFlow platform. It provides an
+approachable, highly-productive interface for solving machine learning (ML)
+problems, with a focus on modern deep learning. Keras covers every step of the
+machine learning workflow, from data processing to hyperparameter tuning to
+deployment. It was developed with a focus on enabling fast experimentation.
+
+With Keras, you have full access to the scalability and cross-platform
+capabilities of TensorFlow. You can run Keras on a TPU Pod or large clusters of
+GPUs, and you can export Keras models to run in the browser or on mobile
+devices. You can also serve Keras models via a web API.
+
+Keras is designed to reduce cognitive load by achieving the following goals:
+
+* Offer simple, consistent interfaces.
+* Minimize the number of actions required for common use cases.
+* Provide clear, actionable error messages.
+* Follow the principle of progressive disclosure of complexity: It's easy to get
+ started, and you can complete advanced workflows by learning as you go.
+* Help you write concise, readable code.
+
+## Who should use Keras
+
+The short answer is that every TensorFlow user should use the Keras APIs by
+default. Whether you're an engineer, a researcher, or an ML practitioner, you
+should start with Keras.
+
+There are a few use cases (for example, building tools on top of TensorFlow or
+developing your own high-performance platform) that require the low-level
+[TensorFlow Core APIs](https://www.tensorflow.org/guide/core). But if your use
+case doesn't fall into one
+of the
+[Core API applications](https://www.tensorflow.org/guide/core#core_api_applications),
+you should prefer Keras.
+
+## Keras API components
+
+The core data structures of Keras are [layers](https://keras.io/api/layers/) and
+[models](https://keras.io/api/models/). A layer is a simple input/output
+transformation, and a model is a directed acyclic graph (DAG) of layers.
+
+### Layers
+
+The `tf.keras.layers.Layer` class is the fundamental abstraction in Keras. A
+`Layer` encapsulates a state (weights) and some computation (defined in the
+`tf.keras.layers.Layer.call` method).
+
+Weights created by layers can be trainable or non-trainable. Layers are
+recursively composable: If you assign a layer instance as an attribute of
+another layer, the outer layer will start tracking the weights created by the
+inner layer.
+
+You can also use layers to handle data preprocessing tasks like normalization
+and text vectorization. Preprocessing layers can be included directly into a
+model, either during or after training, which makes the model portable.
+
+### Models
+
+A model is an object that groups layers together and that can be trained on
+data.
+
+The simplest type of model is the
+[`Sequential` model](https://www.tensorflow.org/guide/keras/sequential_model),
+which is a linear stack of layers. For more complex architectures, you can
+either use the
+[Keras functional API](https://www.tensorflow.org/guide/keras/functional_api),
+which lets you build arbitrary graphs of layers, or
+[use subclassing to write models from scratch](https://www.tensorflow.org/guide/keras/making_new_layers_and_models_via_subclassing).
+
+The `tf.keras.Model` class features built-in training and evaluation methods:
+
+* `tf.keras.Model.fit`: Trains the model for a fixed number of epochs.
+* `tf.keras.Model.predict`: Generates output predictions for the input samples.
+* `tf.keras.Model.evaluate`: Returns the loss and metrics values for the model;
+ configured via the `tf.keras.Model.compile` method.
+
+These methods give you access to the following built-in training features:
+
+* [Callbacks](https://www.tensorflow.org/api_docs/python/tf/keras/callbacks).
+ You can leverage built-in callbacks for early stopping, model checkpointing,
+ and [TensorBoard](https://www.tensorflow.org/tensorboard) monitoring. You can
+ also
+ [implement custom callbacks](https://www.tensorflow.org/guide/keras/writing_your_own_callbacks).
+* [Distributed training](https://www.tensorflow.org/guide/keras/distributed_training).
+ You can easily scale up your training to multiple GPUs, TPUs, or devices.
+* Step fusing. With the `steps_per_execution` argument in
+ `tf.keras.Model.compile`, you can process multiple batches in a single
+ `tf.function` call, which greatly improves device utilization on TPUs.
+
+For a detailed overview of how to use `fit`, see the
+[training and evaluation guide](https://www.tensorflow.org/guide/keras/training_with_built_in_methods).
+To learn how to customize the built-in training and evaluation loops, see
+[Customizing what happens in `fit()`](https://www.tensorflow.org/guide/keras/customizing_what_happens_in_fit).
+
+### Other APIs and tools
+
+Keras provides many other APIs and tools for deep learning, including:
+
+* [Optimizers](https://keras.io/api/optimizers/)
+* [Metrics](https://keras.io/api/metrics/)
+* [Losses](https://keras.io/api/losses/)
+* [Data loading utilities](https://keras.io/api/data_loading/)
+
+For a full list of available APIs, see the
+[Keras API reference](https://keras.io/api/). To learn more about other Keras
+projects and initiatives, see
+[The Keras ecosystem](https://keras.io/getting_started/ecosystem/).
+
+## Next steps
+
+To get started using Keras with TensorFlow, check out the following topics:
+
+* [The Sequential model](https://www.tensorflow.org/guide/keras/sequential_model)
+* [The Functional API](https://www.tensorflow.org/guide/keras/functional)
+* [Training & evaluation with the built-in methods](https://www.tensorflow.org/guide/keras/training_with_built_in_methods)
+* [Making new layers and models via subclassing](https://www.tensorflow.org/guide/keras/custom_layers_and_models)
+* [Serialization and saving](https://www.tensorflow.org/guide/keras/save_and_serialize)
+* [Working with preprocessing layers](https://www.tensorflow.org/guide/keras/preprocessing_layers)
+* [Customizing what happens in fit()](https://www.tensorflow.org/guide/keras/customizing_what_happens_in_fit)
+* [Writing a training loop from scratch](https://www.tensorflow.org/guide/keras/writing_a_training_loop_from_scratch)
+* [Working with RNNs](https://www.tensorflow.org/guide/keras/rnn)
+* [Understanding masking & padding](https://www.tensorflow.org/guide/keras/masking_and_padding)
+* [Writing your own callbacks](https://www.tensorflow.org/guide/keras/custom_callback)
+* [Transfer learning & fine-tuning](https://www.tensorflow.org/guide/keras/transfer_learning)
+* [Multi-GPU and distributed training](https://www.tensorflow.org/guide/keras/distributed_training)
+
+To learn more about Keras, see the following topics at
+[keras.io](http://keras.io):
+
+* [About Keras](https://keras.io/about/)
+* [Introduction to Keras for Engineers](https://keras.io/getting_started/intro_to_keras_for_engineers/)
+* [Introduction to Keras for Researchers](https://keras.io/getting_started/intro_to_keras_for_researchers/)
+* [Keras API reference](https://keras.io/api/)
+* [The Keras ecosystem](https://keras.io/getting_started/ecosystem/)
\ No newline at end of file
diff --git a/site/en/guide/migrate/canned_estimators.ipynb b/site/en/guide/migrate/canned_estimators.ipynb
index 37e9dbe8512..68859511a84 100644
--- a/site/en/guide/migrate/canned_estimators.ipynb
+++ b/site/en/guide/migrate/canned_estimators.ipynb
@@ -37,7 +37,7 @@
"id": "77z2OchJTk0l"
},
"source": [
- "# Migration Examples: Canned Estimators\n",
+ "# Migration examples: Canned Estimators\n",
"\n",
"
\n",
"
\n",
@@ -67,15 +67,15 @@
"source": [
"Canned (or Premade) Estimators have traditionally been used in TensorFlow 1 as quick and easy ways to train models for a variety of typical use cases. TensorFlow 2 provides straightforward approximate substitutes for a number of them by way of Keras models. For those canned estimators that do not have built-in TensorFlow 2 substitutes, you can still build your own replacement fairly easily.\n",
"\n",
- "This guide walks through a few examples of direct equivalents and custom substitutions to demonstrate how TensorFlow 1's `tf.estimator`-derived models can be migrated to TF2 with Keras.\n",
+ "This guide will walk you through a few examples of direct equivalents and custom substitutions to demonstrate how TensorFlow 1's `tf.estimator`-derived models can be migrated to TensorFlow 2 with Keras.\n",
"\n",
"Namely, this guide includes examples for migrating:\n",
"* From `tf.estimator`'s `LinearEstimator`, `Classifier` or `Regressor` in TensorFlow 1 to Keras `tf.compat.v1.keras.models.LinearModel` in TensorFlow 2\n",
"* From `tf.estimator`'s `DNNEstimator`, `Classifier` or `Regressor` in TensorFlow 1 to a custom Keras DNN ModelKeras in TensorFlow 2\n",
"* From `tf.estimator`'s `DNNLinearCombinedEstimator`, `Classifier` or `Regressor` in TensorFlow 1 to `tf.compat.v1.keras.models.WideDeepModel` in TensorFlow 2\n",
- "* From `tf.estimator`'s `BoostedTreesEstimator`, `Classifier` or `Regressor` in TensorFlow 1 to `tf.compat.v1.keras.models.WideDeepModel` in TensorFlow 2\n",
+ "* From `tf.estimator`'s `BoostedTreesEstimator`, `Classifier` or `Regressor` in TensorFlow 1 to `tfdf.keras.GradientBoostedTreesModel` in TensorFlow 2\n",
"\n",
- "A common precursor to the training of a model is feature preprocessing, which is done for TensorFlow 1 Estimator models with `tf.feature_column`. For more information on feature preprocessing in TensorFlow 2, see [this guide on migrating feature columns](migrating_feature_columns.ipynb)."
+ "A common precursor to the training of a model is feature preprocessing, which is done for TensorFlow 1 Estimator models with `tf.feature_column`. For more information on feature preprocessing in TensorFlow 2, see [this guide on migrating from feature columns to the Keras preprocessing layers API](migrating_feature_columns.ipynb)."
]
},
{
@@ -108,11 +108,11 @@
},
"outputs": [],
"source": [
- "import keras\n",
"import pandas as pd\n",
"import tensorflow as tf\n",
"import tensorflow.compat.v1 as tf1\n",
- "import tensorflow_decision_forests as tfdf\n"
+ "import tensorflow_decision_forests as tfdf\n",
+ "from tensorflow import keras\n"
]
},
{
@@ -183,7 +183,7 @@
"id": "bYSgoezeMrpI"
},
"source": [
- "and create a method to instantiate a simplistic sample optimizer to use with our various TensorFlow 1 Estimator and TensorFlow 2 Keras models."
+ "and create a method to instantiate a simplistic sample optimizer to use with various TensorFlow 1 Estimator and TensorFlow 2 Keras models."
]
},
{
@@ -196,7 +196,7 @@
"source": [
"def create_sample_optimizer(tf_version):\n",
" if tf_version == 'tf1':\n",
- " optimizer = lambda: tf.keras.optimizers.Ftrl(\n",
+ " optimizer = lambda: tf.keras.optimizers.legacy.Ftrl(\n",
" l1_regularization_strength=0.001,\n",
" learning_rate=tf1.train.exponential_decay(\n",
" learning_rate=0.1,\n",
@@ -204,7 +204,7 @@
" decay_steps=10000,\n",
" decay_rate=0.9))\n",
" elif tf_version == 'tf2':\n",
- " optimizer = tf.keras.optimizers.Ftrl(\n",
+ " optimizer = tf.keras.optimizers.legacy.Ftrl(\n",
" l1_regularization_strength=0.001,\n",
" learning_rate=tf.keras.optimizers.schedules.ExponentialDecay(\n",
" initial_learning_rate=0.1, decay_steps=10000, decay_rate=0.9))\n",
@@ -226,7 +226,7 @@
"id": "_O7fyhCnpvED"
},
"source": [
- "### TF1: Using LinearEstimator"
+ "### TensorFlow 1: Using LinearEstimator"
]
},
{
@@ -270,7 +270,7 @@
"id": "KEmzBjfnsxwT"
},
"source": [
- "### TF2: Using Keras LinearModel"
+ "### TensorFlow 2: Using Keras LinearModel"
]
},
{
@@ -311,7 +311,7 @@
"id": "YKl6XZ7Bp1t5"
},
"source": [
- "### TF1: Using DNNEstimator"
+ "### TensorFlow 1: Using DNNEstimator"
]
},
{
@@ -320,7 +320,7 @@
"id": "J7wJUmgypln8"
},
"source": [
- "In TensorFlow 1, you can use `tf.estimator.DNNEstimator` to create a baseline DNN model for regression and classification problems."
+ "In TensorFlow 1, you can use `tf.estimator.DNNEstimator` to create a baseline deep neural network (DNN) model for regression and classification problems."
]
},
{
@@ -357,7 +357,7 @@
"id": "6xJz6px6pln-"
},
"source": [
- "### TF2: Using Keras to Create a Custom DNN Model"
+ "### TensorFlow 2: Using Keras to create a custom DNN model"
]
},
{
@@ -368,7 +368,7 @@
"source": [
"In TensorFlow 2, you can create a custom DNN model to substitute for one generated by `tf.estimator.DNNEstimator`, with similar levels of user-specified customization (for instance, as in the previous example, the ability to customize a chosen model optimizer).\n",
"\n",
- "A similar workflow can be used to replace `tf.estimator.experimental.RNNEstimator` with a Keras RNN Model. Keras provides a number of built-in, customizable choices by way of `tf.keras.layers.RNN`, `tf.keras.layers.LSTM`, and `tf.keras.layers.GRU` - see [here](https://www.tensorflow.org/guide/keras/rnn#built-in_rnn_layers_a_simple_example) for more details."
+ "A similar workflow can be used to replace `tf.estimator.experimental.RNNEstimator` with a Keras recurrent neural network (RNN) model. Keras provides a number of built-in, customizable choices by way of `tf.keras.layers.RNN`, `tf.keras.layers.LSTM`, and `tf.keras.layers.GRU`. To learn more, check out the _Built-in RNN layers: a simple example_ section of [RNN with Keras guide](https://www.tensorflow.org/guide/keras/rnn)."
]
},
{
@@ -413,7 +413,7 @@
"id": "GfRaObf5g4TU"
},
"source": [
- "### TF1: Using DNNLinearCombinedEstimator"
+ "### TensorFlow 1: Using DNNLinearCombinedEstimator"
]
},
{
@@ -464,7 +464,7 @@
"id": "BeMikL5ug4TX"
},
"source": [
- "### TF2: Using Keras WideDeepModel"
+ "### TensorFlow 2: Using Keras WideDeepModel"
]
},
{
@@ -477,7 +477,7 @@
"\n",
"This `WideDeepModel` is constructed on the basis of a constituent `LinearModel` and a custom DNN Model, both of which are discussed in the preceding two examples. A custom linear model can also be used in place of the built-in Keras `LinearModel` if desired.\n",
"\n",
- "If you would like to build your own model instead of a canned estimator, check out [how to build a `keras.Sequential` model](https://www.tensorflow.org/guide/keras/sequential_model). For more information on custom training and optimizers you can also checkout [this guide](https://www.tensorflow.org/tutorials/customization/custom_training_walkthrough)."
+ "If you would like to build your own model instead of using a canned estimator, check out the [Keras Sequential model](https://www.tensorflow.org/guide/keras/sequential_model) guide. For more information on custom training and optimizers, check out the [Custom training: walkthrough](https://www.tensorflow.org/tutorials/customization/custom_training_walkthrough) guide."
]
},
{
@@ -532,7 +532,7 @@
"id": "_3mCQVDSeOKD"
},
"source": [
- "### TF1: Using BoostedTreesEstimator"
+ "### TensorFlow 1: Using BoostedTreesEstimator"
]
},
{
@@ -578,7 +578,7 @@
"id": "eNuLP6BeeOKF"
},
"source": [
- "### TF2: Using TensorFlow Decision Forests"
+ "### TensorFlow 2: Using TensorFlow Decision Forests"
]
},
{
@@ -620,7 +620,7 @@
"id": "B1qTdAS-VpXk"
},
"source": [
- "Create a TensorFlow dataset. Note that Decision Forests support natively many types of features and do not need pre-processing."
+ "Create a TensorFlow dataset. Note that Decision Forests natively support many types of features and do not need pre-processing."
]
},
{
@@ -689,9 +689,9 @@
"id": "Z22UJ5SUqToQ"
},
"source": [
- "Gradient Boosted Trees is just one of the many decision forests algorithms avaiable in TensorFlow Decision Forests. For example, Random Forests (available as [tfdf.keras.GradientBoostedTreesModel](https://www.tensorflow.org/decision_forests/api_docs/python/tfdf/keras/RandomForestModel) is very resistant to overfitting) while CART (available as [tfdf.keras.CartModel](https://www.tensorflow.org/decision_forests/api_docs/python/tfdf/keras/CartModel)) is great for model interpretation.\n",
+ "Gradient Boosted Trees is just one of the many decision forest algorithms available in TensorFlow Decision Forests. For example, Random Forests (available as [tfdf.keras.GradientBoostedTreesModel](https://www.tensorflow.org/decision_forests/api_docs/python/tfdf/keras/RandomForestModel) is very resistant to overfitting) while CART (available as [tfdf.keras.CartModel](https://www.tensorflow.org/decision_forests/api_docs/python/tfdf/keras/CartModel)) is great for model interpretation.\n",
"\n",
- "In the next example, we train and plot a Random Forest model."
+ "In the next example, train and plot a Random Forest model."
]
},
{
@@ -718,7 +718,7 @@
"id": "Z0QYolhoZb_k"
},
"source": [
- "Finally, in the next example, we train and evaluate a CART model."
+ "In the final example, train and evaluate a CART model."
]
},
{
@@ -742,7 +742,6 @@
"colab": {
"collapsed_sections": [],
"name": "canned_estimators.ipynb",
- "provenance": [],
"toc_visible": true
},
"kernelspec": {
diff --git a/site/en/guide/migrate/early_stopping.ipynb b/site/en/guide/migrate/early_stopping.ipynb
index fec8144348a..1c1712e975b 100644
--- a/site/en/guide/migrate/early_stopping.ipynb
+++ b/site/en/guide/migrate/early_stopping.ipynb
@@ -457,7 +457,7 @@
"epochs = 100\n",
"patience = 5\n",
"wait = 0\n",
- "best = 0\n",
+ "best = float('inf')\n",
"\n",
"for epoch in range(epochs):\n",
" print(\"\\nStart of epoch %d\" % (epoch,))\n",
@@ -486,7 +486,7 @@
" # The early stopping strategy: stop the training if `val_loss` does not\n",
" # decrease over a certain number of epochs.\n",
" wait += 1\n",
- " if val_loss > best:\n",
+ " if val_loss < best:\n",
" best = val_loss\n",
" wait = 0\n",
" if wait >= patience:\n",
diff --git a/site/en/guide/migrate/evaluator.ipynb b/site/en/guide/migrate/evaluator.ipynb
index e81863edb81..c8f848e4406 100644
--- a/site/en/guide/migrate/evaluator.ipynb
+++ b/site/en/guide/migrate/evaluator.ipynb
@@ -13,6 +13,7 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
+ "cellView": "form",
"id": "HMUDt0CiUJk9"
},
"outputs": [],
@@ -121,7 +122,7 @@
"\n",
"In TensorFlow 1, you can configure a `tf.estimator` to evaluate the estimator using `tf.estimator.train_and_evaluate`.\n",
"\n",
- "In this example, start by defining the `tf.estimator.Estimator` and speciyfing training and evaluation specifications:"
+ "In this example, start by defining the `tf.estimator.Estimator` and specifying training and evaluation specifications:"
]
},
{
@@ -279,7 +280,6 @@
"colab": {
"collapsed_sections": [],
"name": "evaluator.ipynb",
- "provenance": [],
"toc_visible": true
},
"kernelspec": {
diff --git a/site/en/guide/migrate/images/tensorboard_TF1.png b/site/en/guide/migrate/images/tensorboard_TF1.png
index 18b7bbb12cf..294fbbcc5b5 100644
Binary files a/site/en/guide/migrate/images/tensorboard_TF1.png and b/site/en/guide/migrate/images/tensorboard_TF1.png differ
diff --git a/site/en/guide/migrate/images/tensorboard_TF2.png b/site/en/guide/migrate/images/tensorboard_TF2.png
index 55abb91fe6f..bbad8768210 100644
Binary files a/site/en/guide/migrate/images/tensorboard_TF2.png and b/site/en/guide/migrate/images/tensorboard_TF2.png differ
diff --git a/site/en/guide/migrate/migrating_feature_columns.ipynb b/site/en/guide/migrate/migrating_feature_columns.ipynb
index 500894efa8f..b2dbc5fe7c0 100644
--- a/site/en/guide/migrate/migrating_feature_columns.ipynb
+++ b/site/en/guide/migrate/migrating_feature_columns.ipynb
@@ -654,17 +654,17 @@
"source": [
"categorical_col = tf1.feature_column.categorical_column_with_identity(\n",
" 'type', num_buckets=one_hot_dims)\n",
- "# Convert index to one-hot; e.g. [2] -> [0,0,1].\n",
+ "# Convert index to one-hot; e.g., [2] -> [0,0,1].\n",
"indicator_col = tf1.feature_column.indicator_column(categorical_col)\n",
"\n",
- "# Convert strings to indices; e.g. ['small'] -> [1].\n",
+ "# Convert strings to indices; e.g., ['small'] -> [1].\n",
"vocab_col = tf1.feature_column.categorical_column_with_vocabulary_list(\n",
" 'size', vocabulary_list=vocab, num_oov_buckets=1)\n",
"# Embed the indices.\n",
"embedding_col = tf1.feature_column.embedding_column(vocab_col, embedding_dims)\n",
"\n",
"normalizer_fn = lambda x: (x - weight_mean) / math.sqrt(weight_variance)\n",
- "# Normalize the numeric inputs; e.g. [2.0] -> [0.0].\n",
+ "# Normalize the numeric inputs; e.g., [2.0] -> [0.0].\n",
"numeric_col = tf1.feature_column.numeric_column(\n",
" 'weight', normalizer_fn=normalizer_fn)\n",
"\n",
@@ -727,12 +727,12 @@
" 'size': tf.keras.Input(shape=(), dtype='string'),\n",
" 'weight': tf.keras.Input(shape=(), dtype='float32'),\n",
"}\n",
- "# Convert index to one-hot; e.g. [2] -> [0,0,1].\n",
+ "# Convert index to one-hot; e.g., [2] -> [0,0,1].\n",
"type_output = tf.keras.layers.CategoryEncoding(\n",
" one_hot_dims, output_mode='one_hot')(inputs['type'])\n",
- "# Convert size strings to indices; e.g. ['small'] -> [1].\n",
+ "# Convert size strings to indices; e.g., ['small'] -> [1].\n",
"size_output = tf.keras.layers.StringLookup(vocabulary=vocab)(inputs['size'])\n",
- "# Normalize the numeric inputs; e.g. [2.0] -> [0.0].\n",
+ "# Normalize the numeric inputs; e.g., [2.0] -> [0.0].\n",
"weight_output = tf.keras.layers.Normalization(\n",
" axis=None, mean=weight_mean, variance=weight_variance)(inputs['weight'])\n",
"outputs = {\n",
@@ -858,7 +858,7 @@
"id": "O01VQIxCWBxU"
},
"source": [
- "This composed model can be saved as a [SavedModel](https://www.tensorflow.org/guide/saved_model) for later use."
+ "This composed model can be saved as a `.keras` file for later use."
]
},
{
@@ -869,8 +869,8 @@
},
"outputs": [],
"source": [
- "inference_model.save('model')\n",
- "restored_model = tf.keras.models.load_model('model')\n",
+ "inference_model.save('model.keras')\n",
+ "restored_model = tf.keras.models.load_model('model.keras')\n",
"restored_model.predict(predict_dataset)"
]
},
diff --git a/site/en/guide/migrate/migration_debugging.ipynb b/site/en/guide/migrate/migration_debugging.ipynb
index 6a2afa27425..25cb7f9065f 100644
--- a/site/en/guide/migrate/migration_debugging.ipynb
+++ b/site/en/guide/migrate/migration_debugging.ipynb
@@ -37,7 +37,7 @@
"id": "77z2OchJTk0l"
},
"source": [
- "# Debug TF2 Migrated Training Pipeline\n",
+ "# Debug a TensorFlow 2 migrated training pipeline\n",
"\n",
"
\n",
"
\n",
@@ -67,12 +67,12 @@
"id": "zTwPu-w6M5sz"
},
"source": [
- "This notebook demonstrates how to debug training pipeline when migrating to TF2. It consists of following components:\n",
+ "This notebook demonstrates how to debug a training pipeline when migrating to TensorFlow 2 (TF2). It consists of following components:\n",
"1. Suggested steps and code samples for debugging training pipeline\n",
"2. Tools for debugging\n",
"3. Other related resources\n",
"\n",
- "One assumption is you have TF1.x code and trained models for comparison, and you want to build a TF2 model that achieves similar validation accuracy.\n",
+ "One assumption is you have the TensorFlow 1 (TF1.x) code and trained models for comparison, and you want to build a TF2 model that achieves similar validation accuracy.\n",
"\n",
"This notebook does **NOT** cover debugging performance issues for training/inference speed or memory usage."
]
@@ -100,7 +100,7 @@
" disabled\n",
" * Align random number generation, check numerical equivalence in inference\n",
" * (Optional) Check checkpoints are loaded properly and TF1.x/TF2 models\n",
- " generate identitcal output\n",
+ " generate identical output\n",
"\n",
" b. On single GPU/TPU device\n",
"\n",
@@ -124,11 +124,11 @@
"\n",
" d. With multi-device strategies (check the intro for [MultiProcessRunner](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/distribute/multi_process_runner.py#L108) at the bottom)\n",
"\n",
- "4. End-to-end covergence testing on real dataset\n",
+ "4. End-to-end convergence testing on real dataset\n",
"\n",
" a. Check training behaviors with TensorBoard\n",
"\n",
- " * use simple optimizers e.g. SGD and simple distribution strategies e.g.\n",
+ " * use simple optimizers e.g., SGD and simple distribution strategies e.g.\n",
" `tf.distribute.OneDeviceStrategy` first\n",
" * training metrics\n",
" * evaluation metrics\n",
@@ -151,17 +151,6 @@
"## Setup"
]
},
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "id": "sopP--i7-LaF"
- },
- "outputs": [],
- "source": [
- "!pip uninstall -y -q tensorflow"
- ]
- },
{
"cell_type": "code",
"execution_count": null,
@@ -170,9 +159,8 @@
},
"outputs": [],
"source": [
- "# Install tf-nightly as the DeterministicRandomTestTool is only available in\n",
- "# Tensorflow 2.8\n",
- "!pip install -q tf-nightly"
+ "# The `DeterministicRandomTestTool` is only available from Tensorflow 2.8:\n",
+ "!pip install -q \"tensorflow==2.9.*\""
]
},
{
@@ -416,7 +404,7 @@
" decay_steps=params['decay_steps'],\n",
" end_learning_rate=params['end_lr'],\n",
" power=params['lr_power']) \n",
- " self.optimizer = tf.keras.optimizers.SGD(learning_rate_fn)\n",
+ " self.optimizer = tf.keras.optimizers.legacy.SGD(learning_rate_fn)\n",
" self.compiled_loss = tf.keras.losses.CategoricalCrossentropy(from_logits=True)\n",
" self.logs = {\n",
" 'lr': [],\n",
@@ -586,7 +574,7 @@
" # adopt different tolerance strategies before and after 10 steps\n",
" first_n_step = 10\n",
"\n",
- " # abosolute difference is limited below 1e-5\n",
+ " # absolute difference is limited below 1e-5\n",
" # set `equal_nan` to be False to detect potential NaN loss issues\n",
" abosolute_tolerance = 1e-5\n",
" np.testing.assert_allclose(\n",
@@ -622,7 +610,7 @@
"\n",
"tf.print vs print/logging.info\n",
"\n",
- "- With configurable arguments, `tf.print` can recursively display show first and last few elements of each dimension for printed tensors. Check the [API docs](https://www.tensorflow.org/api_docs/python/tf/print) for details.\n",
+ "- With configurable arguments, `tf.print` can recursively display the first and last few elements of each dimension for printed tensors. Check the [API docs](https://www.tensorflow.org/api_docs/python/tf/print) for details.\n",
"- For eager execution, both `print` and `tf.print` print the value of the tensor. But `print` may involve device-to-host copy, which can potentially slow down your code. \n",
"- For graph mode including usage inside `tf.function`, you need to use `tf.print` to print the actual tensor value. `tf.print` is compiled into an op in the graph, whereas `print` and `logging.info` only log at tracing time, which is often not what you want. \n",
"- `tf.print` also supports printing composite tensors like `tf.RaggedTensor` and `tf.sparse.SparseTensor`.\n",
diff --git a/site/en/guide/migrate/model_mapping.ipynb b/site/en/guide/migrate/model_mapping.ipynb
index edb411a7270..2d4582839c0 100644
--- a/site/en/guide/migrate/model_mapping.ipynb
+++ b/site/en/guide/migrate/model_mapping.ipynb
@@ -1048,6 +1048,7 @@
"source": [
"random_tool = v1.keras.utils.DeterministicRandomTestTool(mode='num_random_ops')\n",
"with random_tool.scope():\n",
+ " tf.keras.utils.set_random_seed(42)\n",
" layer = CompatModel(10)\n",
"\n",
" inputs = tf.random.normal(shape=(10, 5, 5, 5))\n",
@@ -1069,6 +1070,7 @@
"source": [
"random_tool = v1.keras.utils.DeterministicRandomTestTool(mode='num_random_ops')\n",
"with random_tool.scope():\n",
+ " tf.keras.utils.set_random_seed(42)\n",
" layer = PartiallyMigratedModel(10)\n",
"\n",
" inputs = tf.random.normal(shape=(10, 5, 5, 5))\n",
@@ -1142,6 +1144,7 @@
"source": [
"random_tool = v1.keras.utils.DeterministicRandomTestTool(mode='num_random_ops')\n",
"with random_tool.scope():\n",
+ " tf.keras.utils.set_random_seed(42)\n",
" layer = NearlyFullyNativeModel(10)\n",
"\n",
" inputs = tf.random.normal(shape=(10, 5, 5, 5))\n",
@@ -1215,6 +1218,7 @@
"source": [
"random_tool = v1.keras.utils.DeterministicRandomTestTool(mode='num_random_ops')\n",
"with random_tool.scope():\n",
+ " tf.keras.utils.set_random_seed(42)\n",
" layer = FullyNativeModel(10)\n",
"\n",
" inputs = tf.random.normal(shape=(10, 5, 5, 5))\n",
@@ -1249,7 +1253,7 @@
"\n",
"The above migration process to native TF2 APIs changed both the variable names (as Keras APIs produce very different weight names), and the object-oriented paths that point to different weights in the model. The impact of these changes is that they will have broken both any existing TF1-style name-based checkpoints or TF2-style object-oriented checkpoints.\n",
"\n",
- "However, in some cases, you might be able to take your original name-based checkpoint and find a mapping of the variables to their new names with approaches like the one detailed in the [Reusing TF1.x checkpoints guide](./reusing_checkpoints.ipynb).\n",
+ "However, in some cases, you might be able to take your original name-based checkpoint and find a mapping of the variables to their new names with approaches like the one detailed in the [Reusing TF1.x checkpoints guide](./migrating_checkpoints.ipynb).\n",
"\n",
"Some tips to making this feasible are as follows:\n",
"- Variables still all have a `name` argument you can set.\n",
diff --git a/site/en/guide/migrate/multi_worker_cpu_gpu_training.ipynb b/site/en/guide/migrate/multi_worker_cpu_gpu_training.ipynb
index e4c30e2fa5c..8a95cb903d6 100644
--- a/site/en/guide/migrate/multi_worker_cpu_gpu_training.ipynb
+++ b/site/en/guide/migrate/multi_worker_cpu_gpu_training.ipynb
@@ -369,7 +369,7 @@
"\n",
"with strategy.scope():\n",
" model = tf.keras.models.Sequential([tf.keras.layers.Dense(1)])\n",
- " optimizer = tf.keras.optimizers.Adagrad(learning_rate=0.05)\n",
+ " optimizer = tf.keras.optimizers.legacy.Adagrad(learning_rate=0.05)\n",
" model.compile(optimizer, \"mse\")\n",
"\n",
"model.fit(dataset, epochs=5, steps_per_epoch=10)"
diff --git a/site/en/guide/migrate/saved_model.ipynb b/site/en/guide/migrate/saved_model.ipynb
index f875b3f9a6c..e7e8ce8daa1 100644
--- a/site/en/guide/migrate/saved_model.ipynb
+++ b/site/en/guide/migrate/saved_model.ipynb
@@ -352,7 +352,11 @@
"source": [
"### Save and export a SavedModel defined with Keras\n",
"\n",
- "The Keras APIs for saving and exporting—`Mode.save` or `tf.keras.models.save_model`—can export a SavedModel from a `tf.keras.Model`. Check out the [Save and load Keras models](../..guide/keras/save_and_serialize) for more details."
+ "\n",
+ "Deprecated: For Keras objects, it's recommended to use the new high-level `.keras` format and `tf.keras.Model.export`, as demonstrated in the guide [here](https://www.tensorflow.org/guide/keras/save_and_serialize). The low-level SavedModel format continues to be supported for existing code.\n",
+ "\n",
+ "\n",
+ "The Keras APIs for saving and exporting—`Model.save` or `tf.keras.models.save_model`—can export a SavedModel from a `tf.keras.Model`. Check out the [Save and load Keras models](../..guide/keras/save_and_serialize) for more details."
]
},
{
@@ -506,6 +510,9 @@
"source": [
"### TensorFlow 2: Load a model saved with Keras\n",
"\n",
+ "Deprecated: For Keras objects, it's recommended to use the new high-level `.keras` format and `tf.keras.Model.export`, as demonstrated in the guide [here](https://www.tensorflow.org/guide/keras/save_and_serialize). The low-level SavedModel format continues to be supported for existing code.\n",
+ "\n",
+ "\n",
"The Keras loading API—`tf.keras.models.load_model`—allows you to reload a saved model back into a Keras Model object. Note that this only allows you to load SavedModels saved with Keras (`Model.save` or `tf.keras.models.save_model`).\n",
"\n",
"Models saved with `tf.saved_model.save` should be loaded with `tf.saved_model.load`. You can load a Keras model saved with `Model.save` using `tf.saved_model.load` but you will only get the TensorFlow graph. Refer to the `tf.keras.models.load_model` API docs and [Save and load Keras models](https://www.tensorflow.org/guide/keras/save_and_serialize#savedmodel_format) guide for details."
@@ -748,7 +755,6 @@
"colab": {
"collapsed_sections": [],
"name": "saved_model.ipynb",
- "provenance": [],
"toc_visible": true
},
"kernelspec": {
diff --git a/site/en/guide/migrate/tensorboard.ipynb b/site/en/guide/migrate/tensorboard.ipynb
index c8ec222b621..ea0cd72b47e 100644
--- a/site/en/guide/migrate/tensorboard.ipynb
+++ b/site/en/guide/migrate/tensorboard.ipynb
@@ -218,10 +218,10 @@
"\n",
"def create_model():\n",
" return tf.keras.models.Sequential([\n",
- " tf.keras.layers.Flatten(input_shape=(28, 28)),\n",
- " tf.keras.layers.Dense(512, activation='relu'),\n",
- " tf.keras.layers.Dropout(0.2),\n",
- " tf.keras.layers.Dense(10, activation='softmax')\n",
+ " tf.keras.layers.Flatten(input_shape=(28, 28), name='layers_flatten'),\n",
+ " tf.keras.layers.Dense(512, activation='relu', name='layers_dense'),\n",
+ " tf.keras.layers.Dropout(0.2, name='layers_dropout'),\n",
+ " tf.keras.layers.Dense(10, activation='softmax', name='layers_dense_2')\n",
" ])\n",
"\n",
"model = create_model()\n",
@@ -279,7 +279,6 @@
"colab": {
"collapsed_sections": [],
"name": "tensorboard.ipynb",
- "provenance": [],
"toc_visible": true
},
"kernelspec": {
diff --git a/site/en/guide/migrate/tf1_vs_tf2.ipynb b/site/en/guide/migrate/tf1_vs_tf2.ipynb
index f4b2cd85b62..60791f72680 100644
--- a/site/en/guide/migrate/tf1_vs_tf2.ipynb
+++ b/site/en/guide/migrate/tf1_vs_tf2.ipynb
@@ -131,7 +131,7 @@
"\n",
"### Other API changes\n",
"\n",
- "* TF2 features significant improvements to the device placement algorithms which renders the usage of `tf.colocate_with` unnecessary. If removing it causes a performance degrade [please file a bug](https://github.com/tensorflow/tensorflow/issues).\n",
+ "* TF2 features significant improvements to the device placement algorithms which renders the usage of `tf.colocate_with` unnecessary. If removing it causes a performance degradation, [please file a bug](https://github.com/tensorflow/tensorflow/issues).\n",
"\n",
"* Replace all usage of `tf.v1.ConfigProto` with equivalent functions from `tf.config`."
]
@@ -159,7 +159,7 @@
"source": [
"## No more globals\n",
"\n",
- "TF1.x relied heavily on implicit global namespaces and collections. When you called `tf.Variable`, it would be put into a collection in the default graph, and it would remain there, even if you lost track of the Python variable pointing to it. You could then recover that `tf.Variable`, but only if you knew the name that it had been created with. This was difficult to do if you were not in control of the variable's creation. As a result, all sorts of mechanisms proliferated to\n",
+ "TF1.x relied heavily on implicit global namespaces and collections. When you call `tf.Variable`, it would be put into a collection in the default graph, and it would remain there, even if you lost track of the Python variable pointing to it. You could then recover that `tf.Variable`, but only if you knew the name that it had been created with. This was difficult to do if you were not in control of the variable's creation. As a result, all sorts of mechanisms proliferated to\n",
"attempt to help you find your variables again, and for frameworks to find\n",
"user-created variables. Some of these include: variable scopes, global collections, helper methods like `tf.get_global_step` and `tf.global_variables_initializer`, optimizers implicitly\n",
"computing gradients over all trainable variables, and so on. TF2 eliminates all of these mechanisms ([Variables 2.0 RFC](https://github.com/tensorflow/community/pull/11)) in favor of the default mechanism - you keep track of your variables. If you lose track of a `tf.Variable`, it gets garbage collected.\n",
@@ -340,7 +340,7 @@
" print(e) # is out of scope and cannot be used here.\n",
"```\n",
"\n",
- "The most straightfoward solution is ensuring that the variable creation and dataset creation are both outside of the `tf.function` call. For example:\n",
+ "The most straightforward solution is ensuring that the variable creation and dataset creation are both outside of the `tf.function` call. For example:\n",
"\n",
"```python\n",
"class Model(tf.Module):\n",
@@ -681,7 +681,7 @@
"source": [
"### `ResourceVariables` instead of `ReferenceVariables`\n",
"\n",
- "`ResourceVariables` have stronger read-write consistency guarantees than `ReferenceVariables`. This leads to more predictable, easier-to-reason about semantics about whether or not you will observe the result of a previous write when using your variables. This change is extremely unlikely to cause existing code to raise errors or to break silently.\n",
+ "`ResourceVariables` have stronger read-write consistency guarantees than `ReferenceVariables`. This leads to more predictable, easier-to-reason semantics about whether or not you will observe the result of a previous write when using your variables. This change is extremely unlikely to cause existing code to raise errors or to break silently.\n",
"\n",
"However, it is ***possible though unlikely*** that these stronger consistency guarantees may increase the memory usage of your specific program. Please file an [issue](https://github.com/tensorflow/tensorflow/issues) if you find this to be the case. Additionally, if you have unit tests relying on exact string comparisons against the operator names in a graph corresponding to variable reads, be aware that enabling resource variables may slightly change the name of these operators.\n",
"\n",
@@ -1027,28 +1027,12 @@
},
"source": [
"### Hashing tensors and variables\n",
- "With TF1.x behaviors you used to be able to directly add variables and tensors to data structures that require hashing, such as `set` and `dict` keys."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "id": "fxi7EgKdBpd5"
- },
- "outputs": [],
- "source": [
- "tf.compat.v1.disable_tensor_equality()\n",
+ "With TF1.x behaviors you used to be able to directly add variables and tensors to data structures that require hashing, such as `set` and `dict` keys.\n",
+ "```python\n",
"x = tf.Variable(0.0)\n",
- "set([x, tf.constant(2.0)])"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {
- "id": "LFgjR-PSajVQ"
- },
- "source": [
+ "set([x, tf.constant(2.0)])\n",
+ "```\n",
+ "\n",
"However, in TF2 with tensor equality enabled, tensors and variables are made unhashable due to the `==` and `!=` operator semantics changing to value equality checks."
]
},
@@ -1135,7 +1119,6 @@
"colab": {
"collapsed_sections": [],
"name": "tf1_vs_tf2.ipynb",
- "provenance": [],
"toc_visible": true
},
"kernelspec": {
diff --git a/site/en/guide/migrate/tflite.ipynb b/site/en/guide/migrate/tflite.ipynb
index fd3695e5434..0426655ee1a 100644
--- a/site/en/guide/migrate/tflite.ipynb
+++ b/site/en/guide/migrate/tflite.ipynb
@@ -67,14 +67,14 @@
"id": "meUTrR4I6m1C"
},
"source": [
- "[TensorFlow Lite](https://www.tensorflow.org/lite/guide) (TFLite) is a set of tools that helps developers run ML inference on-device (mobile, embedded, and IoT devices). The [TFLite converter](https://www.tensorflow.org/lite/convert) is one such tool that converts exisiting TF models into an optimized TFLite model format that can be efficiently run on-device.\n",
+ "[TensorFlow Lite](https://www.tensorflow.org/lite/guide) (TFLite) is a set of tools that helps developers run ML inference on-device (mobile, embedded, and IoT devices). The [TFLite converter](https://www.tensorflow.org/lite/convert) is one such tool that converts existing TF models into an optimized TFLite model format that can be efficiently run on-device.\n",
"\n",
"In this doc, you'll learn what changes you need to make to your TF to TFLite conversion code, followed by a few examples that do the same.\n",
"\n",
"\n",
"## Changes to your TF to TFLite conversion code\n",
"\n",
- "* If you're using a legacy TF1 model format (Keras file, frozen GraphDef, checkpoints, tf.Session, etc), update it to TF1/TF2 SavedModel and use the TF2 converter API `tf.lite.TFLiteConverter.from_saved_model(...)` to convert it to a TFLite model (refer to Table 1).\n",
+ "* If you're using a legacy TF1 model format (such as Keras file, frozen GraphDef, checkpoints, tf.Session), update it to TF1/TF2 SavedModel and use the TF2 converter API `tf.lite.TFLiteConverter.from_saved_model(...)` to convert it to a TFLite model (refer to Table 1).\n",
"\n",
"* Update the converter API flags (refer to Table 2).\n",
"* Remove legacy APIs such as `tf.lite.constants`. (eg: Replace `tf.lite.constants.INT8` with `tf.int8`)\n",
@@ -125,7 +125,7 @@
"source": [
"## Examples\n",
"\n",
- "You'll now walkthrough some examples to convert legacy TF1 models to TF1/TF2 SavedModels and then convert it to TF2 TFLite models.\n",
+ "You'll now walk through some examples to convert legacy TF1 models to TF1/TF2 SavedModels and then convert them to TF2 TFLite models.\n",
"\n",
"### Setup\n",
"\n",
@@ -400,7 +400,7 @@
"with tf.Graph().as_default() as g:\n",
" tf.graph_util.import_graph_def(gdef, name=\"\")\n",
"\n",
- "# Lookup the input and output tensors.\n",
+ "# Look up the input and output tensors.\n",
"input_tensor = g.get_tensor_by_name('input:0') \n",
"output_tensor = g.get_tensor_by_name('MobilenetV1/Predictions/Softmax:0')\n",
"\n",
@@ -436,7 +436,6 @@
"colab": {
"collapsed_sections": [],
"name": "tflite.ipynb",
- "provenance": [],
"toc_visible": true
},
"kernelspec": {
diff --git a/site/en/guide/migrate/tpu_embedding.ipynb b/site/en/guide/migrate/tpu_embedding.ipynb
index 0a5b8882ef2..44105ea984b 100644
--- a/site/en/guide/migrate/tpu_embedding.ipynb
+++ b/site/en/guide/migrate/tpu_embedding.ipynb
@@ -476,7 +476,10 @@
"source": [
"strategy = tf.distribute.TPUStrategy(cluster_resolver)\n",
"with strategy.scope():\n",
- " optimizer = tf.keras.optimizers.Adagrad(learning_rate=0.05)\n",
+ " if hasattr(tf.keras.optimizers, \"legacy\"):\n",
+ " optimizer = tf.keras.optimizers.legacy.Adagrad(learning_rate=0.05)\n",
+ " else:\n",
+ " optimizer = tf.keras.optimizers.Adagrad(learning_rate=0.05)\n",
" dense_input = tf.keras.Input(shape=(2,), dtype=tf.float32, batch_size=global_batch_size)\n",
" sparse_input = tf.keras.Input(shape=(), dtype=tf.int32, batch_size=global_batch_size)\n",
" embedded_input = tfrs.layers.embedding.TPUEmbedding(\n",
diff --git a/site/en/guide/migrate/validate_correctness.ipynb b/site/en/guide/migrate/validate_correctness.ipynb
index 458d03ad48f..a0555cdd55c 100644
--- a/site/en/guide/migrate/validate_correctness.ipynb
+++ b/site/en/guide/migrate/validate_correctness.ipynb
@@ -1254,7 +1254,7 @@
"source": [
"## Step 3b or 4b (optional): Testing with pre-existing checkpoints\n",
"\n",
- "After step 3 or step 4 above, it can be useful to run your numerical equivalence tests when starting from pre-existing name-based checkpoints if you have some. This can test both that your legacy checkpoint loading is working correctly and that the model itself is working right. The [Reusing TF1.x checkpoints guide](./reuse_checkpoints.ipynb) covers how to reuse your pre-existing TF1.x checkpoints and transfer them over to TF2 checkpoints.\n"
+ "After step 3 or step 4 above, it can be useful to run your numerical equivalence tests when starting from pre-existing name-based checkpoints if you have some. This can test both that your legacy checkpoint loading is working correctly and that the model itself is working right. The [Reusing TF1.x checkpoints guide](./migrating_checkpoints.ipynb) covers how to reuse your pre-existing TF1.x checkpoints and transfer them over to TF2 checkpoints.\n"
]
},
{
diff --git a/site/en/guide/mixed_precision.ipynb b/site/en/guide/mixed_precision.ipynb
index 05d29122211..84a93a64191 100644
--- a/site/en/guide/mixed_precision.ipynb
+++ b/site/en/guide/mixed_precision.ipynb
@@ -70,7 +70,7 @@
"source": [
"## Overview\n",
"\n",
- "Mixed precision is the use of both 16-bit and 32-bit floating-point types in a model during training to make it run faster and use less memory. By keeping certain parts of the model in the 32-bit types for numeric stability, the model will have a lower step time and train equally as well in terms of the evaluation metrics such as accuracy. This guide describes how to use the Keras mixed precision API to speed up your models. Using this API can improve performance by more than 3 times on modern GPUs and 60% on TPUs."
+ "Mixed precision is the use of both 16-bit and 32-bit floating-point types in a model during training to make it run faster and use less memory. By keeping certain parts of the model in the 32-bit types for numeric stability, the model will have a lower step time and train equally as well in terms of the evaluation metrics such as accuracy. This guide describes how to use the Keras mixed precision API to speed up your models. Using this API can improve performance by more than 3 times on modern GPUs, 60% on TPUs and more than 2 times on latest Intel CPUs."
]
},
{
@@ -81,7 +81,7 @@
"source": [
"Today, most models use the float32 dtype, which takes 32 bits of memory. However, there are two lower-precision dtypes, float16 and bfloat16, each which take 16 bits of memory instead. Modern accelerators can run operations faster in the 16-bit dtypes, as they have specialized hardware to run 16-bit computations and 16-bit dtypes can be read from memory faster.\n",
"\n",
- "NVIDIA GPUs can run operations in float16 faster than in float32, and TPUs can run operations in bfloat16 faster than float32. Therefore, these lower-precision dtypes should be used whenever possible on those devices. However, variables and a few computations should still be in float32 for numeric reasons so that the model trains to the same quality. The Keras mixed precision API allows you to use a mix of either float16 or bfloat16 with float32, to get the performance benefits from float16/bfloat16 and the numeric stability benefits from float32.\n",
+ "NVIDIA GPUs can run operations in float16 faster than in float32, and TPUs and supporting Intel CPUs can run operations in bfloat16 faster than float32. Therefore, these lower-precision dtypes should be used whenever possible on those devices. However, variables and a few computations should still be in float32 for numeric reasons so that the model trains to the same quality. The Keras mixed precision API allows you to use a mix of either float16 or bfloat16 with float32, to get the performance benefits from float16/bfloat16 and the numeric stability benefits from float32.\n",
"\n",
"Note: In this guide, the term \"numeric stability\" refers to how a model's quality is affected by the use of a lower-precision dtype instead of a higher precision dtype. An operation is \"numerically unstable\" in float16 or bfloat16 if running it in one of those dtypes causes the model to have worse evaluation accuracy or other metrics compared to running the operation in float32."
]
@@ -118,9 +118,11 @@
"source": [
"## Supported hardware\n",
"\n",
- "While mixed precision will run on most hardware, it will only speed up models on recent NVIDIA GPUs and Cloud TPUs. NVIDIA GPUs support using a mix of float16 and float32, while TPUs support a mix of bfloat16 and float32.\n",
+ "While mixed precision will run on most hardware, it will only speed up models on recent NVIDIA GPUs, Cloud TPUs and recent Intel CPUs. NVIDIA GPUs support using a mix of float16 and float32, while TPUs and Intel CPUs support a mix of bfloat16 and float32.\n",
"\n",
- "Among NVIDIA GPUs, those with compute capability 7.0 or higher will see the greatest performance benefit from mixed precision because they have special hardware units, called Tensor Cores, to accelerate float16 matrix multiplications and convolutions. Older GPUs offer no math performance benefit for using mixed precision, however memory and bandwidth savings can enable some speedups. You can look up the compute capability for your GPU at NVIDIA's [CUDA GPU web page](https://developer.nvidia.com/cuda-gpus). Examples of GPUs that will benefit most from mixed precision include RTX GPUs, the V100, and the A100."
+ "Among NVIDIA GPUs, those with compute capability 7.0 or higher will see the greatest performance benefit from mixed precision because they have special hardware units, called Tensor Cores, to accelerate float16 matrix multiplications and convolutions. Older GPUs offer no math performance benefit for using mixed precision, however memory and bandwidth savings can enable some speedups. You can look up the compute capability for your GPU at NVIDIA's [CUDA GPU web page](https://developer.nvidia.com/cuda-gpus). Examples of GPUs that will benefit most from mixed precision include RTX GPUs, the V100, and the A100.\n",
+ "\n",
+ "Among Intel CPUs, starting with the 4th Gen Intel Xeon Processors (code name Sapphire Rapids), will see the greatest performance benefit from mixed precision as they can accelerate bfloat16 computations using AMX instructions (requires Tensorflow 2.12 or later)."
]
},
{
@@ -129,7 +131,7 @@
"id": "-q2hisD60F0_"
},
"source": [
- "Note: If running this guide in Google Colab, the GPU runtime typically has a P100 connected. The P100 has compute capability 6.0 and is not expected to show a significant speedup.\n",
+ "Note: If running this guide in Google Colab, the GPU runtime typically has a P100 connected. The P100 has compute capability 6.0 and is not expected to show a significant speedup. If running on CPU runtime, there may be a slow down as the runtime likely has a CPU without AMX.\n",
"\n",
"You can check your GPU type with the following. The command only exists if the\n",
"NVIDIA drivers are installed, so the following will raise an error otherwise."
@@ -154,7 +156,7 @@
"source": [
"All Cloud TPUs support bfloat16.\n",
"\n",
- "Even on CPUs and older GPUs, where no speedup is expected, mixed precision APIs can still be used for unit testing, debugging, or just to try out the API. On CPUs, mixed precision will run significantly slower, however."
+ "Even on older Intel CPUs, other x86 CPUs without AMX, and older GPUs, where no speedup is expected, mixed precision APIs can still be used for unit testing, debugging, or just to try out the API. However, mixed_bfloat16 on CPUs without AMX instructions and mixed_float16 on all x86 CPUs will run significantly slower."
]
},
{
@@ -235,7 +237,7 @@
"id": "MOFEcna28o4T"
},
"source": [
- "As mentioned before, the `mixed_float16` policy will most significantly improve performance on NVIDIA GPUs with compute capability of at least 7.0. The policy will run on other GPUs and CPUs but may not improve performance. For TPUs, the `mixed_bfloat16` policy should be used instead."
+ "As mentioned before, the `mixed_float16` policy will most significantly improve performance on NVIDIA GPUs with compute capability of at least 7.0. The policy will run on other GPUs and CPUs but may not improve performance. For TPUs and CPUs, the `mixed_bfloat16` policy should be used instead."
]
},
{
@@ -411,7 +413,7 @@
"id": "0Sm8FJHegVRN"
},
"source": [
- "This example cast the input data from int8 to float32. You don't cast to float16 since the division by 255 is on the CPU, which runs float16 operations slower than float32 operations. In this case, the performance difference in negligible, but in general you should run input processing math in float32 if it runs on the CPU. The first layer of the model will cast the inputs to float16, as each layer casts floating-point inputs to its compute dtype.\n",
+ "This example casts the input data from int8 to float32. You don't cast to float16 since the division by 255 is on the CPU, which runs float16 operations slower than float32 operations. In this case, the performance difference is negligible, but in general you should run input processing math in float32 if it runs on the CPU. The first layer of the model will cast the inputs to float16, as each layer casts floating-point inputs to its compute dtype.\n",
"\n",
"The initial weights of the model are retrieved. This will allow training from scratch again by loading the weights."
]
@@ -465,7 +467,7 @@
" \n",
"If you are running this guide in Colab, you can compare the performance of mixed precision with float32. To do so, change the policy from `mixed_float16` to `float32` in the \"Setting the dtype policy\" section, then rerun all the cells up to this point. On GPUs with compute capability 7.X, you should see the time per step significantly increase, indicating mixed precision sped up the model. Make sure to change the policy back to `mixed_float16` and rerun the cells before continuing with the guide.\n",
"\n",
- "On GPUs with compute capability of at least 8.0 (Ampere GPUs and above), you likely will see no performance improvement in the toy model in this guide when using mixed precision compared to float32. This is due to the use of [TensorFloat-32](https://www.tensorflow.org/api_docs/python/tf/config/experimental/enable_tensor_float_32_execution), which automatically uses lower precision math in certain float32 ops such as `tf.linalg.matmul`. TensorFloat-32 gives some of the performance advantages of mixed precision when using float32. However, in real-world models, you will still typically see significantly performance improvements from mixed precision due to memory bandwidth savings and ops which TensorFloat-32 does not support.\n",
+ "On GPUs with compute capability of at least 8.0 (Ampere GPUs and above), you likely will see no performance improvement in the toy model in this guide when using mixed precision compared to float32. This is due to the use of [TensorFloat-32](https://www.tensorflow.org/api_docs/python/tf/config/experimental/enable_tensor_float_32_execution), which automatically uses lower precision math in certain float32 ops such as `tf.linalg.matmul`. TensorFloat-32 gives some of the performance advantages of mixed precision when using float32. However, in real-world models, you will still typically experience significant performance improvements from mixed precision due to memory bandwidth savings and ops which TensorFloat-32 does not support.\n",
"\n",
"If running mixed precision on a TPU, you will not see as much of a performance gain compared to running mixed precision on GPUs, especially pre-Ampere GPUs. This is because TPUs do certain ops in bfloat16 under the hood even with the default dtype policy of float32. This is similar to how Ampere GPUs use TensorFloat-32 by default. Compared to Ampere GPUs, TPUs typically see less performance gains with mixed precision on real-world models.\n",
"\n",
@@ -480,7 +482,9 @@
"source": [
"## Loss scaling\n",
"\n",
- "Loss scaling is a technique which `tf.keras.Model.fit` automatically performs with the `mixed_float16` policy to avoid numeric underflow. This section describes what loss scaling is and the next section describes how to use it with a custom training loop."
+ "Loss scaling is a technique which `tf.keras.Model.fit` automatically performs with the `mixed_float16` policy to avoid numeric underflow. This section describes what loss scaling is and the next section describes how to use it with a custom training loop.\n",
+ "\n",
+ "Note: When using `mixed_bfloat16` policy, there is no need to do loss scaling."
]
},
{
@@ -612,7 +616,7 @@
"id": "FVy5gnBqTE9z"
},
"source": [
- "If you want, it is possible choose an explicit loss scale or otherwise customize the loss scaling behavior, but it is highly recommended to keep the default loss scaling behavior, as it has been found to work well on all known models. See the `tf.keras.mixed_precision.LossScaleOptimizer` documention if you want to customize the loss scaling behavior."
+ "If you want, it is possible choose an explicit loss scale or otherwise customize the loss scaling behavior, but it is highly recommended to keep the default loss scaling behavior, as it has been found to work well on all known models. See the `tf.keras.mixed_precision.LossScaleOptimizer` documentation if you want to customize the loss scaling behavior."
]
},
{
@@ -806,16 +810,17 @@
"source": [
"## Summary\n",
"\n",
- "- You should use mixed precision if you use TPUs or NVIDIA GPUs with at least compute capability 7.0, as it will improve performance by up to 3x.\n",
+ "- You should use mixed precision if you use TPUs, NVIDIA GPUs with at least compute capability 7.0, or Intel CPUs with support for AMX instructions, as it will improve performance by up to 3x.\n",
"- You can use mixed precision with the following lines:\n",
"\n",
" ```python\n",
- " # On TPUs, use 'mixed_bfloat16' instead\n",
+ " # On TPUs and CPUs, use 'mixed_bfloat16' instead\n",
" mixed_precision.set_global_policy('mixed_float16')\n",
" ```\n",
"\n",
"* If your model ends in softmax, make sure it is float32. And regardless of what your model ends in, make sure the output is float32.\n",
"* If you use a custom training loop with `mixed_float16`, in addition to the above lines, you need to wrap your optimizer with a `tf.keras.mixed_precision.LossScaleOptimizer`. Then call `optimizer.get_scaled_loss` to scale the loss, and `optimizer.get_unscaled_gradients` to unscale the gradients.\n",
+ "* If you use a custom training loop with `mixed_bfloat16`, setting the global_policy mentioned above is sufficient.\n",
"* Double the training batch size if it does not reduce evaluation accuracy\n",
"* On GPUs, ensure most tensor dimensions are a multiple of $8$ to maximize performance\n",
"\n",
diff --git a/site/en/guide/profiler.md b/site/en/guide/profiler.md
index 1cd19c109fe..dee8a5a84af 100644
--- a/site/en/guide/profiler.md
+++ b/site/en/guide/profiler.md
@@ -55,7 +55,7 @@ found.
When you run profiling with CUDA® Toolkit in a Docker environment or on Linux,
you may encounter issues related to insufficient CUPTI privileges
(`CUPTI_ERROR_INSUFFICIENT_PRIVILEGES`). Go to the
-[NVIDIA Developer Docs](https://developer.nvidia.com/nvidia-development-tools-solutions-ERR_NVGPUCTRPERM-permission-issue-performance-counters){:.external}
+[NVIDIA Developer Docs](https://developer.nvidia.com/nvidia-development-tools-solutions-ERR_NVGPUCTRPERM-permission-issue-performance-counters)
to learn more about how you can resolve these issues on Linux.
To resolve CUPTI privilege issues in a Docker environment, run
@@ -694,7 +694,7 @@ first few batches to avoid inaccuracies due to initialization overhead.
An example for profiling multiple workers:
```python
- # E.g. your worker IP addresses are 10.0.0.2, 10.0.0.3, 10.0.0.4, and you
+ # E.g., your worker IP addresses are 10.0.0.2, 10.0.0.3, 10.0.0.4, and you
# would like to profile for a duration of 2 seconds.
tf.profiler.experimental.client.trace(
'grpc://10.0.0.2:8466,grpc://10.0.0.3:8466,grpc://10.0.0.4:8466',
@@ -845,7 +845,7 @@ more efficient by casting to different data types after applying
spatial transformations, such as flipping, cropping, rotating, etc.
Note: Some ops like `tf.image.resize` transparently change the `dtype` to
-`fp32`. Make sure you normalize your data to lie between `0` and `1` if its not
+`fp32`. Make sure you normalize your data to lie between `0` and `1` if it's not
done automatically. Skipping this step could lead to `NaN` errors if you have
enabled [AMP](https://developer.nvidia.com/automatic-mixed-precision).
diff --git a/site/en/guide/ragged_tensor.ipynb b/site/en/guide/ragged_tensor.ipynb
index d36010699db..ba0be2928ce 100644
--- a/site/en/guide/ragged_tensor.ipynb
+++ b/site/en/guide/ragged_tensor.ipynb
@@ -674,14 +674,14 @@
"source": [
"### Keras\n",
"\n",
- "[tf.keras](https://www.tensorflow.org/guide/keras) is TensorFlow's high-level API for building and training deep learning models. Ragged tensors may be passed as inputs to a Keras model by setting `ragged=True` on `tf.keras.Input` or `tf.keras.layers.InputLayer`. Ragged tensors may also be passed between Keras layers, and returned by Keras models. The following example shows a toy LSTM model that is trained using ragged tensors."
+ "[tf.keras](https://www.tensorflow.org/guide/keras) is TensorFlow's high-level API for building and training deep learning models. It doesn't have ragged support. But it does support masked tensors. So the easiest way to use a ragged tensor in a Keras model is to convert the ragged tensor to a dense tensor, using `.to_tensor()` and then using Keras's builtin masking:"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
- "id": "pHls7hQVJlk5"
+ "id": "ucYf2sSzTvQo"
},
"outputs": [],
"source": [
@@ -691,26 +691,77 @@
" 'She turned me into a newt.',\n",
" 'A newt?',\n",
" 'Well, I got better.'])\n",
- "is_question = tf.constant([True, False, True, False])\n",
- "\n",
+ "is_question = tf.constant([True, False, True, False])"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "MGYKmizJTw8B"
+ },
+ "outputs": [],
+ "source": [
"# Preprocess the input strings.\n",
"hash_buckets = 1000\n",
"words = tf.strings.split(sentences, ' ')\n",
"hashed_words = tf.strings.to_hash_bucket_fast(words, hash_buckets)\n",
- "\n",
+ "hashed_words.to_list()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "7FTujwOlUT8J"
+ },
+ "outputs": [],
+ "source": [
+ "hashed_words.to_tensor()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "vzWudaESUBOZ"
+ },
+ "outputs": [],
+ "source": [
+ "tf.keras.Input?"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "pHls7hQVJlk5"
+ },
+ "outputs": [],
+ "source": [
"# Build the Keras model.\n",
"keras_model = tf.keras.Sequential([\n",
- " tf.keras.layers.Input(shape=[None], dtype=tf.int64, ragged=True),\n",
- " tf.keras.layers.Embedding(hash_buckets, 16),\n",
- " tf.keras.layers.LSTM(32, use_bias=False),\n",
+ " tf.keras.layers.Embedding(hash_buckets, 16, mask_zero=True),\n",
+ " tf.keras.layers.LSTM(32, return_sequences=True, use_bias=False),\n",
+ " tf.keras.layers.GlobalAveragePooling1D(),\n",
" tf.keras.layers.Dense(32),\n",
" tf.keras.layers.Activation(tf.nn.relu),\n",
" tf.keras.layers.Dense(1)\n",
"])\n",
"\n",
"keras_model.compile(loss='binary_crossentropy', optimizer='rmsprop')\n",
- "keras_model.fit(hashed_words, is_question, epochs=5)\n",
- "print(keras_model.predict(hashed_words))"
+ "keras_model.fit(hashed_words.to_tensor(), is_question, epochs=5)\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "1IAjjmdTU9OU"
+ },
+ "outputs": [],
+ "source": [
+ "print(keras_model.predict(hashed_words.to_tensor()))"
]
},
{
@@ -799,7 +850,7 @@
"source": [
"### Datasets\n",
"\n",
- "[tf.data](https://www.tensorflow.org/guide/data) is an API that enables you to build complex input pipelines from simple, reusable pieces. Its core data structure is `tf.data.Dataset`, which represents a sequence of elements, in which each element consists of one or more components. "
+ "[tf.data](https://www.tensorflow.org/guide/data) is an API that enables you to build complex input pipelines from simple, reusable pieces. Its core data structure is `tf.data.Dataset`, which represents a sequence of elements, in which each element consists of one or more components."
]
},
{
@@ -1078,9 +1129,11 @@
"import tempfile\n",
"\n",
"keras_module_path = tempfile.mkdtemp()\n",
- "tf.saved_model.save(keras_model, keras_module_path)\n",
- "imported_model = tf.saved_model.load(keras_module_path)\n",
- "imported_model(hashed_words)"
+ "keras_model.save(keras_module_path+\"/my_model.keras\")\n",
+ "\n",
+ "imported_model = tf.keras.models.load_model(keras_module_path+\"/my_model.keras\")\n",
+ "\n",
+ "imported_model(hashed_words.to_tensor())"
]
},
{
@@ -2125,7 +2178,6 @@
],
"metadata": {
"colab": {
- "collapsed_sections": [],
"name": "ragged_tensor.ipynb",
"toc_visible": true
},
diff --git a/site/en/guide/random_numbers.ipynb b/site/en/guide/random_numbers.ipynb
index 5212a10a49a..f8b824ad906 100644
--- a/site/en/guide/random_numbers.ipynb
+++ b/site/en/guide/random_numbers.ipynb
@@ -166,7 +166,7 @@
"source": [
"See the *Algorithms* section below for more information about it.\n",
"\n",
- "Another way to create a generator is with `Generator.from_non_deterministic_state`. A generator created this way will start from a non-deterministic state, depending on e.g. time and OS."
+ "Another way to create a generator is with `Generator.from_non_deterministic_state`. A generator created this way will start from a non-deterministic state, depending on e.g., time and OS."
]
},
{
diff --git a/site/en/guide/saved_model.ipynb b/site/en/guide/saved_model.ipynb
index a6b2c7185f0..2601e504669 100644
--- a/site/en/guide/saved_model.ipynb
+++ b/site/en/guide/saved_model.ipynb
@@ -86,8 +86,24 @@
"id": "9SuIC7FiI9g8"
},
"source": [
- "## Creating a SavedModel from Keras\n",
- "\n",
+ "## Creating a SavedModel from Keras"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "AtSmftAvhJvE"
+ },
+ "source": [
+ "Deprecated: For Keras objects, it's recommended to use the new high-level `.keras` format and `tf.keras.Model.export`, as demonstrated in the guide [here](https://www.tensorflow.org/guide/keras/save_and_serialize). The low-level SavedModel format continues to be supported for existing code."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "eLSOptpYhJvE"
+ },
+ "source": [
"For a quick introduction, this section exports a pre-trained Keras model and serves image classification requests with it. The rest of the guide will fill in details and discuss other ways to create SavedModels."
]
},
@@ -354,7 +370,9 @@
"source": [
"The `assets` directory contains files used by the TensorFlow graph, for example text files used to initialize vocabulary tables. It is unused in this example.\n",
"\n",
- "SavedModels may have an `assets.extra` directory for any files not used by the TensorFlow graph, for example information for consumers about what to do with the SavedModel. TensorFlow itself does not use this directory."
+ "SavedModels may have an `assets.extra` directory for any files not used by the TensorFlow graph, for example information for consumers about what to do with the SavedModel. TensorFlow itself does not use this directory.\n",
+ "\n",
+ "The `fingerprint.pb` file contains the [fingerprint](https://en.wikipedia.org/wiki/Fingerprint_(computing)) of the SavedModel, which is composed of several 64-bit hashes that uniquely identify the contents of the SavedModel. The fingerprinting API is currently experimental, but `tf.saved_model.experimental.read_fingerprint` can be used to read the SavedModel fingerprint into a `tf.saved_model.experimental.Fingerprint` object."
]
},
{
@@ -621,7 +639,7 @@
"outputs": [],
"source": [
"imported_with_signatures = tf.saved_model.load(module_with_signature_path)\n",
- "list(imported_with_signatures.signatures.keys())\n"
+ "list(imported_with_signatures.signatures.keys()) # [\"serving_default\"]"
]
},
{
@@ -656,8 +674,12 @@
},
"outputs": [],
"source": [
- "imported_with_multiple_signatures = tf.saved_model.load(module_multiple_signatures_path)\n",
- "list(imported_with_multiple_signatures.signatures.keys())"
+ "imported_with_multiple_signatures = tf.saved_model.load(\n",
+ " module_multiple_signatures_path\n",
+ ")\n",
+ "list(\n",
+ " imported_with_multiple_signatures.signatures.keys()\n",
+ ") # [\"serving_default\", \"array_input\"]"
]
},
{
@@ -682,7 +704,7 @@
" super(CustomModuleWithOutputName, self).__init__()\n",
" self.v = tf.Variable(1.)\n",
"\n",
- " @tf.function(input_signature=[tf.TensorSpec([], tf.float32)])\n",
+ " @tf.function(input_signature=[tf.TensorSpec(None, tf.float32)])\n",
" def __call__(self, x):\n",
" return {'custom_output_name': x * self.v}\n",
"\n",
@@ -702,7 +724,41 @@
"outputs": [],
"source": [
"imported_with_output_name = tf.saved_model.load(module_output_path)\n",
- "imported_with_output_name.signatures['serving_default'].structured_outputs"
+ "imported_with_output_name.signatures[\n",
+ " 'serving_default'\n",
+ "].structured_outputs # {'custom_output_name': TensorSpec(shape=, dtype=tf.float32, name='custom_output_name')}"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "Q4bCK55x1IBW"
+ },
+ "source": [
+ "## Proto-splitting\n",
+ "\n",
+ "Note: This feature will be part of the TensorFlow 2.15 release. It is currently available in the nightly build which you cqan install with `pip install tf-nightly`.\n",
+ "\n",
+ "Due to limits of the protobuf implementation, proto sizes cannot exceed 2GB. This can lead to the following errors when attempting to save very large models:\n",
+ "\n",
+ "```\n",
+ "ValueError: Message tensorflow.SavedModel exceeds maximum protobuf size of 2GB: ...\n",
+ "```\n",
+ "\n",
+ "```\n",
+ "google.protobuf.message.DecodeError: Error parsing message as the message exceeded the protobuf limit with type 'tensorflow.GraphDef'\n",
+ "```\n",
+ "\n",
+ "If you wish to save models that exceed the 2GB limit, then you'll need to save using the new proto-splitting option:\n",
+ "\n",
+ "```python\n",
+ "tf.saved_model.save(\n",
+ " ...,\n",
+ " options=tf.saved_model.SaveOptions(experimental_image_format=True)\n",
+ ")\n",
+ "```\n",
+ "\n",
+ "More information can be found in the [Proto Splitter / Merger Library guide](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/tools/proto_splitter/g3doc/in-depth-guide.md)."
]
},
{
@@ -976,8 +1032,8 @@
],
"metadata": {
"colab": {
- "collapsed_sections": [],
"name": "saved_model.ipynb",
+ "provenance": [],
"toc_visible": true
},
"kernelspec": {
diff --git a/site/en/guide/sparse_tensor.ipynb b/site/en/guide/sparse_tensor.ipynb
index cd38fdf55ab..3d4daca7fad 100644
--- a/site/en/guide/sparse_tensor.ipynb
+++ b/site/en/guide/sparse_tensor.ipynb
@@ -256,7 +256,7 @@
" values=[31, 2], \n",
" dense_shape=[4, 10])\n",
"\n",
- "st_b = tf.sparse.SparseTensor(indices=[[0, 2], [7, 0]],\n",
+ "st_b = tf.sparse.SparseTensor(indices=[[0, 2], [3, 0]],\n",
" values=[56, 38],\n",
" dense_shape=[4, 10])\n",
"\n",
@@ -620,7 +620,7 @@
"\n",
"However, there are a few cases where it can be useful to distinguish zero values from missing values. In particular, this allows for one way to encode missing/unknown data in your training data. For example, consider a use case where you have a tensor of scores (that can have any floating point value from -Inf to +Inf), with some missing scores. You can encode this tensor using a sparse tensor where the explicit zeros are known zero scores but the implicit zero values actually represent missing data and not zero. \n",
"\n",
- "Note: This is generally not the intended usage of `tf.sparse.SparseTensor`s; and you might want to also consier other techniques for encoding this such as for example using a separate mask tensor that identifies the locations of known/unknown values. However, exercise caution while using this approach, since most sparse operations will treat explicit and implicit zero values identically."
+ "Note: This is generally not the intended usage of `tf.sparse.SparseTensor`s; and you might want to also consider other techniques for encoding this such as for example using a separate mask tensor that identifies the locations of known/unknown values. However, exercise caution while using this approach, since most sparse operations will treat explicit and implicit zero values identically."
]
},
{
diff --git a/site/en/guide/tensor.ipynb b/site/en/guide/tensor.ipynb
index 453dde97f7e..2eb261aad75 100644
--- a/site/en/guide/tensor.ipynb
+++ b/site/en/guide/tensor.ipynb
@@ -80,9 +80,9 @@
"id": "VQ3s2J8Vgowq"
},
"source": [
- "Tensors are multi-dimensional arrays with a uniform type (called a `dtype`). You can see all supported `dtypes` at `tf.dtypes.DType`.\n",
+ "Tensors are multi-dimensional arrays with a uniform type (called a `dtype`). You can see all supported `dtypes` at `tf.dtypes`.\n",
"\n",
- "If you're familiar with [NumPy](https://numpy.org/devdocs/user/quickstart.html){:.external}, tensors are (kind of) like `np.arrays`.\n",
+ "If you're familiar with [NumPy](https://numpy.org/devdocs/user/quickstart.html), tensors are (kind of) like `np.arrays`.\n",
"\n",
"All tensors are immutable like Python numbers and strings: you can never update the contents of a tensor, only create a new one.\n"
]
@@ -326,7 +326,7 @@
"a = tf.constant([[1, 2],\n",
" [3, 4]])\n",
"b = tf.constant([[1, 1],\n",
- " [1, 1]]) # Could have also said `tf.ones([2,2])`\n",
+ " [1, 1]]) # Could have also said `tf.ones([2,2], dtype=tf.int32)`\n",
"\n",
"print(tf.add(a, b), \"\\n\")\n",
"print(tf.multiply(a, b), \"\\n\")\n",
@@ -571,7 +571,7 @@
"source": [
"### Single-axis indexing\n",
"\n",
- "TensorFlow follows standard Python indexing rules, similar to [indexing a list or a string in Python](https://docs.python.org/3/tutorial/introduction.html#strings){:.external}, and the basic rules for NumPy indexing.\n",
+ "TensorFlow follows standard Python indexing rules, similar to [indexing a list or a string in Python](https://docs.python.org/3/tutorial/introduction.html#strings), and the basic rules for NumPy indexing.\n",
"\n",
"* indexes start at `0`\n",
"* negative indices count backwards from the end\n",
@@ -1035,7 +1035,7 @@
"source": [
"## Broadcasting\n",
"\n",
- "Broadcasting is a concept borrowed from the [equivalent feature in NumPy](https://numpy.org/doc/stable/user/basics.broadcasting.html){:.external}. In short, under certain conditions, smaller tensors are \"stretched\" automatically to fit larger tensors when running combined operations on them.\n",
+ "Broadcasting is a concept borrowed from the [equivalent feature in NumPy](https://numpy.org/doc/stable/user/basics.broadcasting.html). In short, under certain conditions, smaller tensors are \"stretched\" automatically to fit larger tensors when running combined operations on them.\n",
"\n",
"The simplest and most common case is when you attempt to multiply or add a tensor to a scalar. In that case, the scalar is broadcast to be the same shape as the other argument. "
]
@@ -1161,7 +1161,7 @@
"source": [
"Unlike a mathematical op, for example, `broadcast_to` does nothing special to save memory. Here, you are materializing the tensor.\n",
"\n",
- "It can get even more complicated. [This section](https://jakevdp.github.io/PythonDataScienceHandbook/02.05-computation-on-arrays-broadcasting.html){:.external} of Jake VanderPlas's book _Python Data Science Handbook_ shows more broadcasting tricks (again in NumPy)."
+ "It can get even more complicated. [This section](https://jakevdp.github.io/PythonDataScienceHandbook/02.05-computation-on-arrays-broadcasting.html) of Jake VanderPlas's book _Python Data Science Handbook_ shows more broadcasting tricks (again in NumPy)."
]
},
{
@@ -1445,7 +1445,7 @@
"id": "st9OxrUxWSKY"
},
"source": [
- "And `tf.string.to_number`:"
+ "And `tf.strings.to_number`:"
]
},
{
@@ -1564,7 +1564,6 @@
"Tce3stUlHN0L"
],
"name": "tensor.ipynb",
- "provenance": [],
"toc_visible": true
},
"kernelspec": {
diff --git a/site/en/guide/tensor_slicing.ipynb b/site/en/guide/tensor_slicing.ipynb
index 9f58a206de6..c5cb2d71356 100644
--- a/site/en/guide/tensor_slicing.ipynb
+++ b/site/en/guide/tensor_slicing.ipynb
@@ -635,7 +635,6 @@
"colab": {
"collapsed_sections": [],
"name": "tensor_slicing.ipynb",
- "provenance": [],
"toc_visible": true
},
"kernelspec": {
diff --git a/site/en/guide/tf_numpy.ipynb b/site/en/guide/tf_numpy.ipynb
index 9b469241a3b..3083acb147d 100644
--- a/site/en/guide/tf_numpy.ipynb
+++ b/site/en/guide/tf_numpy.ipynb
@@ -70,7 +70,7 @@
"source": [
"## Overview\n",
"\n",
- "TensorFlow implements a subset of the [NumPy API](https://numpy.org/doc/1.16), available as `tf.experimental.numpy`. This allows running NumPy code, accelerated by TensorFlow, while also allowing access to all of TensorFlow's APIs."
+ "TensorFlow implements a subset of the [NumPy API](https://numpy.org/doc/stable/index.html), available as `tf.experimental.numpy`. This allows running NumPy code, accelerated by TensorFlow, while also allowing access to all of TensorFlow's APIs."
]
},
{
@@ -142,7 +142,7 @@
"\n",
"An instance of `tf.experimental.numpy.ndarray`, called **ND Array**, represents a multidimensional dense array of a given `dtype` placed on a certain device. It is an alias to `tf.Tensor`. Check out the ND array class for useful methods like `ndarray.T`, `ndarray.reshape`, `ndarray.ravel` and others.\n",
"\n",
- "First create an ND array object, and then invoke different methods. "
+ "First create an ND array object, and then invoke different methods."
]
},
{
@@ -170,11 +170,28 @@
{
"cell_type": "markdown",
"metadata": {
- "id": "Mub8-dvJMUr4"
+ "id": "-BOY8CGRKEhE"
},
"source": [
"### Type promotion\n",
"\n",
+ "There are 4 options for type promotion in TensorFlow.\n",
+ "\n",
+ "- By default, TensorFlow raises errors instead of promoting types for mixed type operations.\n",
+ "- Running `tf.numpy.experimental_enable_numpy_behavior()` switches TensorFlow to use `NumPy` type promotion rules (described below).\n",
+ "- After TensorFlow 2.15, there are two new options (refer to [TF NumPy Type Promotion](tf_numpy_type_promotion.ipynb) for details):\n",
+ " - `tf.numpy.experimental_enable_numpy_behavior(dtype_conversion_mode=\"all\")` uses Jax type promotion rules.\n",
+ " - `tf.numpy.experimental_enable_numpy_behavior(dtype_conversion_mode=\"safe\")` uses Jax type promotion rules, but disallows certain unsafe promotions."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "SXskSHrX5J45"
+ },
+ "source": [
+ "#### NumPy Type Promotion\n",
+ "\n",
"TensorFlow NumPy APIs have well-defined semantics for converting literals to ND array, as well as for performing type promotion on ND array inputs. Please see [`np.result_type`](https://numpy.org/doc/1.16/reference/generated/numpy.result_type.html) for more details."
]
},
@@ -200,7 +217,7 @@
" (tnp.int32, tnp.int64, tnp.float32, tnp.float64)]\n",
"for i, v1 in enumerate(values):\n",
" for v2 in values[i + 1:]:\n",
- " print(\"%s + %s => %s\" % \n",
+ " print(\"%s + %s => %s\" %\n",
" (v1.dtype.name, v2.dtype.name, (v1 + v2).dtype.name))"
]
},
@@ -932,8 +949,8 @@
"metadata": {
"accelerator": "GPU",
"colab": {
- "collapsed_sections": [],
"name": "tf_numpy.ipynb",
+ "provenance": [],
"toc_visible": true
},
"kernelspec": {
diff --git a/site/en/guide/tf_numpy_type_promotion.ipynb b/site/en/guide/tf_numpy_type_promotion.ipynb
new file mode 100644
index 00000000000..f984310822a
--- /dev/null
+++ b/site/en/guide/tf_numpy_type_promotion.ipynb
@@ -0,0 +1,1138 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "ZjN_IJ8mhJ-4"
+ },
+ "source": [
+ "##### Copyright 2023 The TensorFlow Authors."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "sY3Ffd83hK3b"
+ },
+ "outputs": [],
+ "source": [
+ "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+ "# you may not use this file except in compliance with the License.\n",
+ "# You may obtain a copy of the License at\n",
+ "#\n",
+ "# https://www.apache.org/licenses/LICENSE-2.0\n",
+ "#\n",
+ "# Unless required by applicable law or agreed to in writing, software\n",
+ "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+ "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+ "# See the License for the specific language governing permissions and\n",
+ "# limitations under the License."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "03Pw58e6mTHI"
+ },
+ "source": [
+ "# TF-NumPy Type Promotion"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "l9nPKvxK-_pM"
+ },
+ "source": [
+ "
+ !pip install --upgrade tensorflow_hub
+
+ import tensorflow_hub as hub
+
+ model = hub.KerasLayer("https://tfhub.dev/google/nnlm-en-dim128/2")
+ embeddings = model(["The rain in Spain.", "falls",
+ "mainly", "In the plain!"])
+
+ print(embeddings.shape) #(4,128)
+
+ - options:
+ - cards
+ - centered-header
+ heading: >
+ Models
+ description: >
+ Find trained models from the TensorFlow community on TFHub.dev
+ items:
+ - heading: BERT
+ description: Check out BERT for NLP tasks including text classification and question answering.
+ path: https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/3
+ image_path: /hub/images/bert.png
+ buttons:
+ - label: See the modelnorth_east
+ path: https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/3
+ - heading: Object detection
+ description: Use the Faster R-CNN Inception ResNet V2 640x640 model for detecting objects in images.
+ path: https://tfhub.dev/tensorflow/faster_rcnn/inception_resnet_v2_640x640/1
+ image_path: /hub/images/object_detection.png
+ buttons:
+ - label: See the modelnorth_east
+ path: https://tfhub.dev/tensorflow/faster_rcnn/inception_resnet_v2_640x640/1
+ - heading: Style transfer
+ description: Transfer the style of one image to another using the image style transfer model.
+ path: https://tfhub.dev/google/magenta/arbitrary-image-stylization-v1-256/2
+ image_path: /hub/images/style_transfer.png
+ buttons:
+ - label: See the modelnorth_east
+ path: https://tfhub.dev/google/magenta/arbitrary-image-stylization-v1-256/2
+ - heading: On-device food classifier
+ description: Use this TFLite model to classify photos of food on a mobile device.
+ path: https://tfhub.dev/google/lite-model/aiy/vision/classifier/food_V1/1
+ image_path: /hub/images/food.png
+ buttons:
+ - label: See the modelnorth_east
+ path: https://tfhub.dev/google/lite-model/aiy/vision/classifier/food_V1/1
+ - options:
+ - cards
+ - centered-header
+ heading: >
+ News & announcements
+ description: >
+ Check out our blog for more announcements and view the latest #TFHub updates on Twitter
+ items:
+ - heading: TensorFlow Hub for Real World Impact at Google I/O
+ youtube_id: BE5nkhFe3AE
+ description: >
+ Learn how you can use TensorFlow Hub to build ML solutions with real world impact.
+ buttons:
+ - label: Watch the videonorth_east
+ path: https://www.youtube.com/watch?v=BE5nkhFe3AE
+ - heading: "On-device ML solutions"
+ description: >
+ To explore ML solutions for your mobile and web apps including TensorFlow Hub, visit the Google on-device machine learning page.
+ path: https://g.co/on-device-ml
+ image_path: /hub/images/odml.png
+ buttons:
+ - label: Visit the sitenorth_east
+ path: https://g.co/on-device-ml
+ - heading: "Making BERT Easier with Preprocessing Models From TensorFlow Hub"
+ description: >
+ TensorFlow Hub makes BERT simple to use with new preprocessing models.
+ path: https://blog.tensorflow.org/2020/12/making-bert-easier-with-preprocessing-models-from-tensorflow-hub.html
+ image_path: /hub/images/bert_preprocess_wide.png
+ buttons:
+ - label: Read the blogarrow_forward
+ path: https://blog.tensorflow.org/2020/12/making-bert-easier-with-preprocessing-models-from-tensorflow-hub.html
+ - heading: "From singing to musical scores: Estimating pitch with SPICE and Tensorflow Hub"
+ description: >
+ Learn how to use the SPICE model to automatically transcribe sheet music from live audio.
+ path: https://blog.tensorflow.org/2020/06/estimating-pitch-with-spice-and-tensorflow-hub.html
+ image_path: /hub/images/spice_blog.png
+ buttons:
+ - label: Read the blogarrow_forward
+ path: https://blog.tensorflow.org/2020/06/estimating-pitch-with-spice-and-tensorflow-hub.html
+ - options:
+ - cards
+ - centered-header
+ heading: >
+ Community
+ description: Join the TensorFlow Hub community
+ items:
+ - heading: TensorFlow Hub on GitHub
+ icon:
+ path: /hub/images/github_icon.svg
+ path: https://github.com/tensorflow/hub
+ - heading: Contribute models
+ icon:
+ name: publish
+ path: /hub/publish
+ - options:
+ - cta
+ items:
+ - heading: Get started with TensorFlow Hub
+ buttons:
+ - label: Find trained models north_east
+ path: https://tfhub.dev
+ classname: button
diff --git a/site/en/hub/_redirects.yaml b/site/en/hub/_redirects.yaml
new file mode 100644
index 00000000000..bee1cbec873
--- /dev/null
+++ b/site/en/hub/_redirects.yaml
@@ -0,0 +1,7 @@
+redirects:
+- from: /hub/becoming_a_publisher
+ to: /hub/publish
+- from: /hub/writing_model_documentation
+ to: /hub/writing_documentation#model
+- from: /hub/creating_a_collection
+ to: /hub/writing_documentation#collection
diff --git a/site/en/hub/build_from_source.md b/site/en/hub/build_from_source.md
new file mode 100644
index 00000000000..42e19eb6208
--- /dev/null
+++ b/site/en/hub/build_from_source.md
@@ -0,0 +1,195 @@
+
+
+
+# Creating the TensorFlow Hub pip package using Linux
+
+Note: This document is for developers interested in modifying TensorFlow Hub
+itself. To _use_ TensorFlow Hub, see the [Install instructions](installation.md)
+
+If you make changes to TensorFlow Hub pip package, you will likely want to
+rebuild the pip package from source to try out your changes.
+
+This requires:
+
+* Python
+* TensorFlow
+* Git
+* [Bazel](https://docs.bazel.build/versions/master/install.html)
+
+Alternatively, if you install the protobuf compiler you can
+[try out your changes without using bazel](#develop).
+
+## Setup a virtualenv {:#setup}
+
+### Activate virtualenv
+
+Install virtualenv if it's not installed already:
+
+```shell
+~$ sudo apt-get install python-virtualenv
+```
+
+Create a virtual environment for the package creation:
+
+```shell
+~$ virtualenv --system-site-packages tensorflow_hub_env
+```
+
+And activate it:
+
+```shell
+~$ source ~/tensorflow_hub_env/bin/activate # bash, sh, ksh, or zsh
+~$ source ~/tensorflow_hub_env/bin/activate.csh # csh or tcsh
+```
+
+### Clone the TensorFlow Hub repository.
+
+```shell
+(tensorflow_hub_env)~/$ git clone https://github.com/tensorflow/hub
+(tensorflow_hub_env)~/$ cd hub
+```
+
+## Test your changes
+
+### Run TensorFlow Hub's tests
+
+```shell
+(tensorflow_hub_env)~/hub/$ bazel test tensorflow_hub:all
+```
+
+## Build and install the package
+
+### Build TensorFlow Hub pip packaging script
+
+To build a pip package for TensorFlow Hub:
+
+```shell
+(tensorflow_hub_env)~/hub/$ bazel build tensorflow_hub/pip_package:build_pip_package
+```
+
+### Create the TensorFlow Hub pip package
+
+```shell
+(tensorflow_hub_env)~/hub/$ bazel-bin/tensorflow_hub/pip_package/build_pip_package \
+/tmp/tensorflow_hub_pkg
+```
+
+### Install and test the pip package (optional)
+
+Run the following commands to install the pip package.
+
+```shell
+(tensorflow_hub_env)~/hub/$ pip install /tmp/tensorflow_hub_pkg/*.whl
+```
+
+Test import TensorFlow Hub:
+
+```shell
+(tensorflow_hub_env)~/hub/$ cd .. # exit the directory to avoid confusion
+(tensorflow_hub_env)~/$ python -c "import tensorflow_hub as hub"
+```
+
+## "Developer" install (experimental)
+
+
+
+Warning: This approach to running TensorFlow is experimental, and not officially
+supported by the TensorFlow Hub team.
+
+Building the package with bazel is the only officially supported method. However
+if you are unfamiliar with bazel simpler to work with open source tools. For
+that you can do a "developer install" of the package.
+
+This installation method allows you to install the working directory into your
+python environment, so that ongoing changes are reflected when you import the
+package.
+
+### Setup the repository
+
+First setup the virtualenv and repository, as described [above](#setup).
+
+### Install `protoc`
+
+Because TensorFlow Hub uses protobufs you will need the protobuf compiler to
+create the necessary python `_pb2.py` files from the `.proto` files.
+
+#### On a Mac:
+
+```
+(tensorflow_hub_env)~/hub/$ brew install protobuf
+```
+
+#### On Linux
+
+```
+(tensorflow_hub_env)~/hub/$ sudo apt install protobuf-compiler
+```
+
+### Compile the `.proto` files
+
+Initially there are no `_pb2.py` files in the directory:
+
+```
+(tensorflow_hub_env)~/hub/$ ls -1 tensorflow_hub/*_pb2.py
+```
+
+Run `protoc` to create them:
+
+```
+(tensorflow_hub_env)~/hub/$ protoc -I=tensorflow_hub --python_out=tensorflow_hub tensorflow_hub/*.proto
+(tensorflow_hub_env)~/hub/$ ls -1 tensorflow_hub/*_pb2.py
+```
+
+
+
+Note: Don't forget to recompile the `_pb2.py` files if you make changes to the
+`.proto` definitions.
+
+### Import directly from the repository
+
+With the `_pb2.py` files in place, you can use try out your modifications
+directly from the TensorFlow Hub directory:
+
+```
+(tensorflow_hub_env)~/$ python -c "import tensorflow_hub as hub"
+```
+
+### Install in "developer" mode
+
+Or to use this from outside the repository root, you can use the `setup.py
+develop` installation:
+
+```
+(tensorflow_hub_env)~/hub/$ python tensorflow_hub/pip_package/setup.py develop
+```
+
+Now you can use your local changes in a regular python virtualenv, without the
+need to rebuild and install the pip package for each new change:
+
+```shell
+(tensorflow_hub_env)~/hub/$ cd .. # exit the directory to avoid confusion
+(tensorflow_hub_env)~/$ python -c "import tensorflow_hub as hub"
+```
+
+## De-activate the virtualenv
+
+```shell
+(tensorflow_hub_env)~/hub/$ deactivate
+```
diff --git a/site/en/hub/caching.md b/site/en/hub/caching.md
new file mode 100644
index 00000000000..678b2c22af0
--- /dev/null
+++ b/site/en/hub/caching.md
@@ -0,0 +1,86 @@
+
+# Caching model downloads from TF Hub
+
+## Overview
+
+The `tensorflow_hub` library currently supports two modes for downloading
+models. By default, a model is downloaded as a compressed archive and cached on
+disk. Secondly, models can directly be read from remote storage into TensorFlow.
+Either way, the calls to `tensorflow_hub` functions in the actual Python code
+can and should continue to use the canonical tfhub.dev URLs of models, which are
+portable across systems and navigable for documentation. In the rare case that
+user code needs the actual filesystem location (after downloading and
+decompressing, or after resolving a model handle into a filesystem path),
+it can be obtained by the function `hub.resolve(handle)`.
+
+### Caching of compressed downloads
+
+The `tensorflow_hub` library by default caches models on the filesystem when
+they have been downloaded from tfhub.dev (or other [hosting sites](hosting.md))
+and decompressed. This mode is recommended for most environments, except if disk
+space is scarce but network bandwidth and latency are superb.
+
+The download location defaults to a local temporary directory but can be
+customized by setting the environment variable `TFHUB_CACHE_DIR` (recommended)
+or by passing the command-line flag `--tfhub_cache_dir`. The default cache
+location `/tmp/tfhub_modules` (or whatever `os.path.join(tempfile.gettempdir(),
+"tfhub_modules")` is evaluated to) should work in most cases.
+
+Users who prefer persistent caching across system reboots can instead set
+`TFHUB_CACHE_DIR` to a location in their home directory. For example, a user of
+the bash shell on a Linux system can add a line like the following to
+`~/.bashrc`:
+
+```bash
+export TFHUB_CACHE_DIR=$HOME/.cache/tfhub_modules
+```
+
+...restart the shell, and then this location will be used. When using a
+persistent location, be aware that there is no automatic cleanup.
+
+### Reading from remote storage
+
+Users can instruct the `tensorflow_hub` library to directly read models from
+remote storage (GCS) instead of downloading the models locally with:
+
+```shell
+os.environ["TFHUB_MODEL_LOAD_FORMAT"] = "UNCOMPRESSED"
+```
+
+or by setting the command-line flag `--tfhub_model_load_format` to
+`UNCOMPRESSED`. This way, no caching directory is needed, which is especially
+helpful in environments that provide little disk space but a fast internet
+connection.
+
+### Running on TPU in Colab notebooks
+
+On [colab.research.google.com](https://colab.research.google.com), downloading
+compressed models will conflict with the TPU runtime since the computation
+workload is delegated to another machine that does not have access to the cache
+location by default. There are two workarounds for this situation:
+
+#### 1) Use a GCS bucket that the TPU worker can access
+
+The easiest solution is to instruct the `tensorflow_hub` library to read the
+models from TF Hub's GCS bucket as explained above. Users with their own GCS
+bucket can instead specify a directory in their bucket as the cache location
+with code like:
+
+```python
+import os
+os.environ["TFHUB_CACHE_DIR"] = "gs://my-bucket/tfhub-modules-cache"
+```
+
+...before calling the `tensorflow_hub` library.
+
+#### 2) Redirect all reads through the Colab host
+
+Another workaround is to redirect all reads (even of large variables) through
+the Colab host:
+
+```python
+load_options =
+tf.saved_model.LoadOptions(experimental_io_device='/job:localhost')
+reloaded_model = hub.load("https://tfhub.dev/...", options=load_options)
+```
+**Note:** See more information regarding valid handles [here](tf2_saved_model.md#model_handles).
diff --git a/site/en/hub/common_issues.md b/site/en/hub/common_issues.md
new file mode 100644
index 00000000000..03ba4a62a8e
--- /dev/null
+++ b/site/en/hub/common_issues.md
@@ -0,0 +1,148 @@
+
+# Common issues
+
+If your issue is not listed here, please search the
+[github issues](https://github.com/tensorflow/hub/issues) before filling a new
+one.
+
+**Note:** This documentation uses TFhub.dev URL handles in examples. See more
+information regarding other valid handle types [here](tf2_saved_model.md#model_handles).
+
+## TypeError: 'AutoTrackable' object is not callable
+
+```python
+# BAD: Raises error
+embed = hub.load('https://tfhub.dev/google/nnlm-en-dim128/1')
+embed(['my text', 'batch'])
+```
+
+This error frequently arises when loading models in TF1 Hub format with the
+`hub.load()` API in TF2. Adding the correct signature should fix this problem.
+See the [TF-Hub migration guide for TF2](migration_tf2.md) for more details on
+moving to TF2 and the use of models in TF1 Hub format in TF2.
+
+```python
+
+embed = hub.load('https://tfhub.dev/google/nnlm-en-dim128/1')
+embed.signatures['default'](['my text', 'batch'])
+```
+
+## Cannot download a module
+
+In the process of using a module from an URL there are many errors that can show
+up due to the network stack. Often this is a problem specific to the machine
+running the code and not an issue with the library. Here is a list of the common
+ones:
+
+* **"EOF occurred in violation of protocol"** - This issue is likely to be
+ generated if the installed python version does not support the TLS
+ requirements of the server hosting the module. Notably, python 2.7.5 is
+ known to fail resolving modules from tfhub.dev domain. **FIX**: Please
+ update to a newer python version.
+
+* **"cannot verify tfhub.dev's certificate"** - This issue is likely to be
+ generated if something on the network is trying to act as the dev gTLD.
+ Before .dev was used as a gTLD, developers and frameworks would sometimes
+ use .dev names to help testing code. **FIX:** Identify and reconfigure the
+ software that intercepts name resolution in the ".dev" domain.
+
+* Failures to write to the cache directory `/tmp/tfhub_modules` (or similar):
+ see [Caching](caching.md) for what that is and how to change its location.
+
+If the above errors and fixes do not work, one can try to manually download a
+module by simulating the protocol of attaching `?tf-hub-format=compressed` to
+the URL to download a tar compressed file that has to be manually decompressed
+into a local file. The path to the local file can then be used instead of the
+URL. Here is a quick example:
+
+```bash
+# Create a folder for the TF hub module.
+$ mkdir /tmp/moduleA
+# Download the module, and uncompress it to the destination folder. You might want to do this manually.
+$ curl -L "https://tfhub.dev/google/universal-sentence-encoder/2?tf-hub-format=compressed" | tar -zxvC /tmp/moduleA
+# Test to make sure it works.
+$ python
+> import tensorflow_hub as hub
+> hub.Module("/tmp/moduleA")
+```
+
+## Running inference on a pre-initialized module
+
+If you are writing a Python program that applies a module many times on input
+data, you can apply the following recipes. (Note: For serving requests in
+production services, consider
+[TensorFlow Serving](https://www.tensorflow.org/tfx/guide/serving) or other
+scalable, Python-free solutions.)
+
+Assuming your use-case model is **initialization** and subsequent **requests**
+(for example Django, Flask, custom HTTP server, etc.), you can set-up the
+serving as follows:
+
+### TF2 SavedModels
+
+* In the initialization part:
+ * Load the TF2.0 model.
+
+```python
+import tensorflow_hub as hub
+
+embedding_fn = hub.load("https://tfhub.dev/google/universal-sentence-encoder/4")
+```
+
+* In the request part:
+ * Use the embedding function to run inference.
+
+```python
+embedding_fn(["Hello world"])
+```
+
+This call of a tf.function is optimized for performance, see
+[tf.function guide](https://www.tensorflow.org/guide/function).
+
+### TF1 Hub modules
+
+* In the initialization part:
+ * Build the graph with a **placeholder** - entry point into the graph.
+ * Initialize the session.
+
+```python
+import tensorflow as tf
+import tensorflow_hub as hub
+
+# Create graph and finalize (finalizing optional but recommended).
+g = tf.Graph()
+with g.as_default():
+ # We will be feeding 1D tensors of text into the graph.
+ text_input = tf.placeholder(dtype=tf.string, shape=[None])
+ embed = hub.Module("https://tfhub.dev/google/universal-sentence-encoder/2")
+ embedded_text = embed(text_input)
+ init_op = tf.group([tf.global_variables_initializer(), tf.tables_initializer()])
+g.finalize()
+
+# Create session and initialize.
+session = tf.Session(graph=g)
+session.run(init_op)
+```
+
+* In the request part:
+ * Use the session to feed data into the graph through the placeholder.
+
+```python
+result = session.run(embedded_text, feed_dict={text_input: ["Hello world"]})
+```
+
+## Cannot change a model's dtype (e.g., float32 to bfloat16)
+
+TensorFlow's SavedModels (shared on TF Hub or otherwise) contain operations that
+work on fixed data types (often, float32 for the weights and intermediate
+activations of neural networks). These cannot be changed after the fact when
+loading the SavedModel (but model publishers can choose to publish different
+models with different data types).
+
+## Update a model version
+
+The documentation metadata of model versions can be updated. However, the
+version's assets (model files) are immutable. If you want to change the model
+assets, you can publish a newer version of the model. It's a good practice to
+extend the documentation with a change log that describes what changed between
+versions.
diff --git a/site/en/hub/common_saved_model_apis/images.md b/site/en/hub/common_saved_model_apis/images.md
new file mode 100644
index 00000000000..5413f0adc07
--- /dev/null
+++ b/site/en/hub/common_saved_model_apis/images.md
@@ -0,0 +1,155 @@
+
+# Common SavedModel APIs for Image Tasks
+
+This page describes how [TF2 SavedModels](../tf2_saved_model.md) for
+image-related tasks should implement the
+[Reusable SavedModel API](../reusable_saved_models.md). (This replaces the
+[Common Signatures for Images](../common_signatures/images.md) for the
+now-deprecated [TF1 Hub format](../tf1_hub_module).)
+
+
+
+## Image Feature Vector
+
+### Usage summary
+
+An **image feature vector** is a dense 1-D tensor that represents a whole image,
+typically for use by a simple feed-forward classifier in the consumer model. (In
+terms of classic CNNs, this is the bottleneck value after the spatial extent has
+been pooled or flattened away, but before classification is done; for that, see
+[image classification](#classification) below.)
+
+A Reusable SavedModel for image feature extraction has a `__call__` method on
+the root object that maps a batch of images to a batch of feature vectors. It
+can be used like so:
+
+```python
+obj = hub.load("path/to/model") # That's tf.saved_model.load() after download.
+images = ... # A batch of images with shape [batch_size, height, width, 3].
+features = obj(images) # A batch with shape [batch_size, num_features].
+```
+
+In Keras, the equivalent is
+
+```python
+features = hub.KerasLayer("path/to/model")(images)
+```
+
+The input follows the general convention for [input of images](#input). The
+model documentation specifies the permissible range for `height` and `width` of
+the input.
+
+The output is a single tensor of dtype `float32` and shape `[batch_size,
+num_features]`. The `batch_size` is the same as in the input. `num_features` is
+a module-specific constant independent of input size.
+
+### API details
+
+The [Reusable SavedModel API](../reusable_saved_models.md) also provides a list
+of `obj.variables` (e.g., for initialization when not loading eagerly).
+
+A model that supports fine-tuning provides a list of `obj.trainable_variables`.
+It may require you to pass `training=True` to execute in training mode (e.g.,
+for dropout). Some models allow optional arguments to override hyperparameters
+(e.g., dropout rate; to be described in model documentation). The model may also
+provide a list of `obj.regularization_losses`. For details, see the
+[Reusable SavedModel API](../reusable_saved_models.md).
+
+In Keras, this is taken care of by `hub.KerasLayer`: initialize it with
+`trainable=True` to enable fine-tuning, and (in the rare case that hparam
+overrides apply) with `arguments=dict(some_hparam=some_value, ...))`.
+
+### Notes
+
+Applying dropout to the output features (or not) should be left to the model
+consumer. The SavedModel itself should not perform dropout on the actual outputs
+(even if it uses dropout internally in other places).
+
+### Examples
+
+Reusable SavedModels for image feature vectors are used in
+
+* the Colab tutorial
+ [Retraining an Image Classifier](https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/hub/tutorials/tf2_image_retraining.ipynb),
+
+
+
+## Image Classification
+
+### Usage summary
+
+**Image classification** maps the pixels of an image to linear scores (logits)
+for membership in the classes of a taxonomy _selected by the module publisher_.
+This allows model consumers to to draw conclusions from the particular
+classification learned by the publisher module. (For image classification with
+a new set of classes, it is common to reuse an
+[Image Feature Vector](#feature-vector) model with a new classifier instead.)
+
+A Reusable SavedModel for image classification has a `__call__` method on the
+root object that maps a batch of images to a batch of logits. It can be used
+like so:
+
+```python
+obj = hub.load("path/to/model") # That's tf.saved_model.load() after download.
+images = ... # A batch of images with shape [batch_size, height, width, 3].
+logits = obj(images) # A batch with shape [batch_size, num_classes].
+```
+
+In Keras, the equivalent is
+
+```python
+logits = hub.KerasLayer("path/to/model")(images)
+```
+
+The input follows the general convention for [input of images](#input). The
+model documentation specifies the permissible range for `height` and `width` of
+the input.
+
+The output `logits` is a single tensor of dtype `float32` and shape
+`[batch_size, num_classes]`. The `batch_size` is the same as in the input.
+`num_classes` is the number of classes in the classification, which is a
+model-specific constant.
+
+The value `logits[i, c]` is a score predicting the membership of example `i` in
+the class with index `c`.
+
+It depends on the underlying classification whether these scores are meant to be
+used with softmax (for mutually exclusive classes), sigmoid (for orthogonal
+classes), or something else. The module documentation should describe this, and
+refer to a definition of the class indices.
+
+### API details
+
+The [Reusable SavedModel API](../reusable_saved_models.md) also provides a list
+of `obj.variables` (e.g., for initialization when not loading eagerly).
+
+A model that supports fine-tuning provides a list of `obj.trainable_variables`.
+It may require you to pass `training=True` to execute in training mode (e.g.,
+for dropout). Some models allow optional arguments to override hyperparameters
+(e.g., dropout rate; to be described in model documentation). The model may also
+provide a list of `obj.regularization_losses`. For details, see the
+[Reusable SavedModel API](../reusable_saved_models.md).
+
+In Keras, this is taken care of by `hub.KerasLayer`: initialize it with
+`trainable=True` to enable fine-tuning, and (in the rare case that hparam
+overrides apply) with `arguments=dict(some_hparam=some_value, ...))`.
+
+
+
+## Image input
+
+This is common to all types of image models.
+
+A model that takes a batch of images as input accepts them as a dense 4-D tensor
+of dtype `float32` and shape `[batch_size, height, width, 3]` whose elements are
+RGB color values of pixels normalized to the range [0, 1]. This is what you get
+from `tf.image.decode_*()` followed by `tf.image.convert_image_dtype(...,
+tf.float32)`.
+
+The model accepts any `batch_size`. The model documentation specifies the
+permissible range for `height` and `width`. The last dimension is fixed to 3 RGB
+channels.
+
+It is recommended that models use the `channels_last` (or `NHWC`) layout of
+Tensors throughout, and leave it to TensorFlow's graph optimizer to rewrite to
+`channels_first` (or `NCHW`) if needed.
diff --git a/site/en/hub/common_saved_model_apis/index.md b/site/en/hub/common_saved_model_apis/index.md
new file mode 100644
index 00000000000..356505f9952
--- /dev/null
+++ b/site/en/hub/common_saved_model_apis/index.md
@@ -0,0 +1,46 @@
+
+# Common SavedModel APIs for TF Hub
+
+## Introduction
+
+[TensorFlow Hub](https://tfhub.dev) hosts models for a variety of tasks. Models
+for the same task are encouraged to implement a common API so that model
+consumers can easily exchange them without modifying the code that uses them,
+even if they come from different publishers.
+
+The goal is to make exchanging different models for the same task as simple as
+switching a string-valued hyperparameter. With that, model consumers can easily
+find the best one for their problem.
+
+This directory collects specifications of common APIs for models in the
+[TF2 SavedModel format](../tf2_saved_model.md). (It replaces the
+[Common Signatures](../common_signatures/index.md) for the now-deprecated
+[TF1 Hub format](../tf1_hub_module.md).)
+
+## Reusable SavedModel: the common foundation
+
+The [Reusable SavedModel API](../reusable_saved_models.md) defines general
+conventions how to load a SavedModel back into a Python program and reuse it as
+part of a bigger TensorFlow model.
+
+Basic usage:
+
+```python
+obj = hub.load("path/to/model") # That's tf.saved_model.load() after download.
+outputs = obj(inputs, training=False) # Invokes the tf.function obj.__call__.
+```
+
+Key point: This uses the object-based interface to restored SavedModels that was
+added in TensorFlow 2, not the SavedModel signatures for serving.
+
+For Keras users, the `hub.KerasLayer` class relies on this API to wrap the
+Reusable SavedModel as a Keras Layer (shielding Keras users from its details),
+with inputs and outputs according to the task-specific APIs listed below.
+
+## Task-specific APIs
+
+These refine the [Reusable SavedModel API](../reusable_saved_models.md) with
+conventions for particular ML tasks and types of data.
+
+* [Image tasks](images.md)
+* [Text tasks](text.md)
diff --git a/site/en/hub/common_saved_model_apis/text.md b/site/en/hub/common_saved_model_apis/text.md
new file mode 100644
index 00000000000..209319f27a9
--- /dev/null
+++ b/site/en/hub/common_saved_model_apis/text.md
@@ -0,0 +1,361 @@
+
+# Common SavedModel APIs for Text Tasks
+
+This page describes how [TF2 SavedModels](../tf2_saved_model.md) for
+text-related tasks should implement the
+[Reusable SavedModel API](../reusable_saved_models.md). (This replaces and
+extends the [Common Signatures for Text](../common_signatures/text.md) for the
+now-deprecated [TF1 Hub format](../tf1_hub_module).)
+
+## Overview
+
+There are several APIs to compute **text embeddings** (also known as dense
+representations of text, or text feature vectors).
+
+* The API for *text embeddings from text inputs* is implemented by a
+ SavedModel that maps a batch of strings to a batch of embedding vectors.
+ This is very easy to use, and many models on TF Hub have implemented it.
+ However, this does not allow fine-tuning the model on TPU.
+
+* The API for *text embeddings with preprocessed inputs* solves the same task,
+ but is implemented by two separate SavedModels:
+
+ * a *preprocessor* that can run inside a tf.data input pipeline and
+ converts strings and other variable-length data into numeric Tensors,
+ * an *encoder* that accepts the results of the preprocessor and performs
+ the trainable part of the embedding computation.
+
+ This split allows inputs to be preprocessed asynchronously before being fed
+ into the training loop. In particular, it allows building encoders that can
+ be run and fine-tuned on [TPU](https://www.tensorflow.org/guide/tpu).
+
+* The API for *text embeddings with Transformer encoders* extends the API for
+ text embeddings from preprocessed inputs to the particular case of BERT and
+ other Transformer encoders.
+
+ * The *preprocessor* is extended to build encoder inputs from more than
+ one segment of input text.
+ * The *Transformer encoder* exposes the context-aware embeddings of
+ individual tokens.
+
+In each case, the text inputs are UTF-8 encoded strings, typically of plain
+text, unless the model documentation provides otherwise.
+
+Regardless of API, different models have been pre-trained on text from different
+languages and domains, and with different tasks in mind. Therefore, not every
+text embedding model is suitable for every problem.
+
+
+
+
+## Text Embedding from Text Inputs
+
+A SavedModel for **text embeddings from text inputs** accepts a batch of inputs
+in a string Tensor of shape `[batch_size]` and maps them to a float32 Tensor of
+shape `[batch_size, dim]` with dense representations (feature vectors) of the
+inputs.
+
+### Usage synopsis
+
+```python
+obj = hub.load("path/to/model")
+text_input = ["A long sentence.",
+ "single-word",
+ "http://example.com"]
+embeddings = obj(text_input)
+```
+
+Recall from the [Reusable SavedModel API](../reusable_saved_models.md) that
+running the model in training mode (e.g., for dropout) may require a keyword
+argument `obj(..., training=True)`, and that `obj` provides attributes
+`.variables`, `.trainable_variables` and `.regularization_losses` as applicable.
+
+In Keras, all this is taken care of by
+
+```python
+embeddings = hub.KerasLayer("path/to/model", trainable=...)(text_input)
+```
+
+### Distributed training
+
+If the text embedding is used as part of a model that gets trained with a
+distribution strategy, the call to `hub.load("path/to/model")` or
+`hub.KerasLayer("path/to/model", ...)`, resp., must happen inside the
+DistributionStrategy scope in order to create the model's variables in the
+distributed way. For example
+
+```python
+ with strategy.scope():
+ ...
+ model = hub.load("path/to/model")
+ ...
+```
+
+### Examples
+
+* Colab tutorial
+ [Text Classification with Movie Reviews](https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/hub/tutorials/tf2_text_classification.ipynb).
+
+
+
+## Text Embeddings with Preprocessed Inputs
+
+A **text embedding with preprocessed inputs** is implemented by two separate
+SavedModels:
+
+* a **preprocessor** that maps a string Tensor of shape `[batch_size]` to a
+ dict of numeric Tensors,
+* an **encoder** that accepts a dict of Tensors as returned by the
+ preprocessor, performs the trainable part of the embedding computation, and
+ returns a dict of outputs. The output under key `"default"` is a float32
+ Tensor of shape `[batch_size, dim]`.
+
+This allows to run the preprocessor in an input pipeline but fine-tune the
+embeddings computed by the encoder as part of a larger model. In particular, it
+allows to build encoders that can be run and fine-tuned on
+[TPU](https://www.tensorflow.org/guide/tpu).
+
+It is an implementation detail which Tensors are contained in the preprocessor's
+output, and which (if any) additional Tensors besides `"default"` are contained
+in the encoder's output.
+
+The documentation of the encoder must specify which preprocessor to use with it.
+Typically, there is exactly one correct choice.
+
+### Usage synopsis
+
+```python
+text_input = tf.constant(["A long sentence.",
+ "single-word",
+ "http://example.com"])
+preprocessor = hub.load("path/to/preprocessor") # Must match `encoder`.
+encoder_inputs = preprocessor(text_input)
+
+encoder = hub.load("path/to/encoder")
+encoder_outputs = encoder(encoder_inputs)
+embeddings = encoder_outputs["default"]
+```
+
+Recall from the [Reusable SavedModel API](../reusable_saved_models.md) that
+running the encoder in training mode (e.g., for dropout) may require a keyword
+argument `encoder(..., training=True)`, and that `encoder` provides attributes
+`.variables`, `.trainable_variables` and `.regularization_losses` as applicable.
+
+The `preprocessor` model may have `.variables` but is not meant to be trained
+further. Preprocessing is not mode-dependent: if `preprocessor()` has a
+`training=...` argument at all, it has no effect.
+
+In Keras, all this is taken care of by
+
+```python
+encoder_inputs = hub.KerasLayer("path/to/preprocessor")(text_input)
+encoder_outputs = hub.KerasLayer("path/to/encoder", trainable=True)(encoder_inputs)
+embeddings = encoder_outputs["default"]
+```
+
+### Distributed training
+
+If the encoder is used as part of a model that gets trained with a distribution
+strategy, the call to `hub.load("path/to/encoder")` or
+`hub.KerasLayer("path/to/encoder", ...)`, resp., must happen inside
+
+```python
+ with strategy.scope():
+ ...
+```
+
+in order to re-create the encoder variables in the distributed way.
+
+Likewise, if the preprocessor is part of the trained model (as in the simple
+example above), it also needs to be loaded under the distribution strategy
+scope. If, however, the preprocessor is used in an input pipeline (e.g., in a
+callable passed to `tf.data.Dataset.map()`), its loading must happen *outside*
+the distribution strategy scope, in order to place its variables (if any) on the
+host CPU.
+
+### Examples
+
+* Colab tutorial
+ [Classify text with BERT](https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/tutorials/text/classify_text_with_bert.ipynb).
+
+
+
+## Text embeddings with Transformer Encoders
+
+Transformer encoders for text operate on a batch of input sequences, each
+sequence comprising *n* ≥ 1 segments of tokenized text, within some
+model-specific bound on *n*. For BERT and many of its extensions, that bound is
+2, so they accept single segments and segment pairs.
+
+The API for **text embeddings with Transformer encoders** extends the API for
+text embeddings with preprocessed inputs to this setting.
+
+### Preprocessor
+
+A preprocessor SavedModel for text embeddings with Transformer encoders
+implements the API of a preprocessor SavedModel for text embeddings with
+preprocessed inputs (see above), which provides a way to map single-segment text
+inputs directly to encoder inputs.
+
+In addition, the preprocessor SavedModel provides callable subobjects `tokenize`
+for tokenization (separately per segment) and `bert_pack_inputs` for packing *n*
+tokenized segments into one input sequence for the encoder. Each subobject
+follows the [Reusable SavedModel API](../reusable_saved_models.md).
+
+#### Usage synopsis
+
+As a concrete example for two segments of text, let us look at a sentence
+entailment task that asks whether a premise (first segment) does or does not
+imply a hypothesis (second segment).
+
+```python
+preprocessor = hub.load("path/to/preprocessor")
+
+# Tokenize batches of both text inputs.
+text_premises = tf.constant(["The quick brown fox jumped over the lazy dog.",
+ "Good day."])
+tokenized_premises = preprocessor.tokenize(text_premises)
+text_hypotheses = tf.constant(["The dog was lazy.", # Implied.
+ "Axe handle!"]) # Not implied.
+tokenized_hypotheses = preprocessor.tokenize(text_hypotheses)
+
+# Pack input sequences for the Transformer encoder.
+seq_length = 128
+encoder_inputs = preprocessor.bert_pack_inputs(
+ [tokenized_premises, tokenized_hypotheses],
+ seq_length=seq_length) # Optional argument.
+```
+
+In Keras, this computation can be expressed as
+
+```python
+tokenize = hub.KerasLayer(preprocessor.tokenize)
+tokenized_hypotheses = tokenize(text_hypotheses)
+tokenized_premises = tokenize(text_premises)
+
+bert_pack_inputs = hub.KerasLayer(
+ preprocessor.bert_pack_inputs,
+ arguments=dict(seq_length=seq_length)) # Optional argument.
+encoder_inputs = bert_pack_inputs([tokenized_premises, tokenized_hypotheses])
+```
+
+#### Details of `tokenize`
+
+A call to `preprocessor.tokenize()` accepts a string Tensor of shape
+`[batch_size]` and returns a
+[RaggedTensor](https://www.tensorflow.org/guide/ragged_tensor) of shape
+`[batch_size, ...]` whose values are int32 token ids representing the input
+strings. There can be *r* ≥ 1 ragged dimensions after `batch_size` but no other
+uniform dimension.
+
+* If *r*=1, the shape is `[batch_size, (tokens)]`, and each input is simply
+ tokenized into a flat sequence of tokens.
+* If *r*>1, there are *r*-1 additional levels of grouping. For example,
+ [tensorflow_text.BertTokenizer](https://github.com/tensorflow/text/blob/v2.3.0/tensorflow_text/python/ops/bert_tokenizer.py#L138)
+ uses *r*=2 to group tokens by words and yields shape `[batch_size, (words),
+ (tokens_per_word)]`. It is up to the model at hand how many of these extra
+ level(s) exist, if any, and what groupings they represent.
+
+The user can (but need not) modify tokenized inputs, e.g., to accommodate the
+seq_length limit that will be enforced in packing encoder inputs. Extra
+dimensions in the tokenizer output can help here (e.g., to respect word
+boundaries) but become meaningless in the next step.
+
+In terms of the [Reusable SavedModel API](../reusable_saved_models.md), the
+`preprocessor.tokenize` object may have `.variables` but is not meant to be
+trained further. Tokenization is not mode-dependent: if
+`preprocessor.tokenize()` has a `training=...` argument at all, it has no
+effect.
+
+#### Details of `bert_pack_inputs`
+
+A call to `preprocessor.bert_pack_inputs()` accepts a Python list of tokenized
+inputs (batched separately for each input segment) and returns a dict of Tensors
+representing a batch of fixed-length input sequences for the Transformer encoder
+model.
+
+Each tokenized input is an int32 RaggedTensor of shape `[batch_size, ...]`,
+where the number *r* of ragged dimensions after batch_size is either 1 or the
+same as in the output of `preprocessor.tokenize().` (The latter is for
+convenience only; the extra dimensions are flattened out before packing.)
+
+Packing adds special tokens around the input segments as expected by the
+encoder. The `bert_pack_inputs()` call implements exactly the packing scheme
+used by the original BERT models and many of their extensions: the packed
+sequence starts with one start-of-sequence token, followed by the tokenized
+segments, each terminated by one end-of-segment token. Remaining positions up to
+seq_length, if any, are filled up with padding tokens.
+
+If a packed sequence would exceed seq_length, `bert_pack_inputs()` truncates its
+segments to prefixes of approximately equal sizes so that the packed sequence
+fits exactly within seq_length.
+
+Packing is not mode-dependent: if `preprocessor.bert_pack_inputs()` has a
+`training=...` argument at all, it has no effect. Also,
+`preprocessor.bert_pack_inputs` is not expected to have variables, or support
+fine-tuning.
+
+### Encoder
+
+The encoder is called on the dict of `encoder_inputs` in the same way as in the
+API for text embeddings with preprocessed inputs (see above), including the
+provisions from the [Reusable SavedModel API](../reusable_saved_models.md).
+
+#### Usage synopsis
+
+```python
+encoder = hub.load("path/to/encoder")
+encoder_outputs = encoder(encoder_inputs)
+```
+
+or equivalently in Keras:
+
+```python
+encoder = hub.KerasLayer("path/to/encoder", trainable=True)
+encoder_outputs = encoder(encoder_inputs)
+```
+
+#### Details
+
+The `encoder_outputs` are a dict of Tensors with the following keys.
+
+
+* `"sequence_output"`: a float32 Tensor of shape `[batch_size, seq_length,
+ dim]` with the context-aware embedding of each token of every packed input
+ sequence.
+* `"pooled_output"`: a float32 Tensor of shape `[batch_size, dim]` with the
+ embedding of each input sequence as a whole, derived from sequence_output in
+ some trainable manner.
+* `"default"`, as required by the API for text embeddings with preprocessed
+ inputs: a float32 Tensor of shape `[batch_size, dim]` with the embedding of
+ each input sequence. (This might be just an alias of pooled_output.)
+
+The contents of the `encoder_inputs` are not strictly required by this API
+definition. However, for encoders that use BERT-style inputs, it is recommended
+to use the following names (from the
+[NLP Modeling Toolkit of TensorFlow Model Garden](https://github.com/tensorflow/models/tree/master/official/nlp))
+to minimize friction in interchanging encoders and reusing preprocessor models:
+
+* `"input_word_ids"`: an int32 Tensor of shape `[batch_size, seq_length]` with
+ the token ids of the packed input sequence (that is, including a
+ start-of-sequence token, end-of-segment tokens, and padding).
+* `"input_mask"`: an int32 Tensor of shape `[batch_size, seq_length]` with
+ value 1 at the position of all input tokens present before padding and value
+ 0 for the padding tokens.
+* `"input_type_ids"`: an int32 Tensor of shape `[batch_size, seq_length]` with
+ the index of the input segment that gave rise to the input token at the
+ respective position. The first input segment (index 0) includes the
+ start-of-sequence token and its end-of-segment token. The second and later
+ segments (if present) include their respective end-of-segment token. Padding
+ tokens get index 0 again.
+
+### Distributed training
+
+For loading the preprocessor and encoder objects inside or outside a
+distribution strategy scope, the same rules apply as in the API for text
+embeddings with preprocessed inputs (see above).
+
+### Examples
+
+* Colab tutorial
+ [Solve GLUE tasks using BERT on TPU](https://colab.research.google.com/github/tensorflow/text/blob/master/docs/tutorials/bert_glue.ipynb).
diff --git a/site/en/hub/common_signatures/images.md b/site/en/hub/common_signatures/images.md
new file mode 100644
index 00000000000..5e41c3e2960
--- /dev/null
+++ b/site/en/hub/common_signatures/images.md
@@ -0,0 +1,155 @@
+
+# Common Signatures for Images
+
+This page describes common signatures that should be implemented by modules in
+the [TF1 Hub format](../tf1_hub_module.md) for image-related tasks. (For the
+[TF2 SavedModel format](../tf2_saved_model.md), see the analogous
+[SavedModel API](../common_saved_model_apis/images.md).)
+
+Some modules can be used for more than one task (e.g., image classification
+modules tend to do some feature extraction on the way). Therefore, each module
+provides (1) named signatures for all the tasks anticipated by the publisher,
+and (2) a default signature `output = m(images)` for its designated primary
+task.
+
+
+## Image Feature Vector
+
+### Usage summary
+
+An **image feature vector** is a dense 1-D tensor that represents a whole image,
+typically for classification by the consumer model. (Unlike the intermediate
+activations of CNNs, it does not offer a spatial breakdown. Unlike [image
+classification](#classification), it discards the classification learned
+by the publisher model.)
+
+A module for image feature extraction has a default signature that maps a batch
+of images to a batch of feature vectors. It can be used like so:
+
+```python
+ module_spec = hub.load_module_spec("path/to/module")
+ height, width = hub.get_expected_image_size(module_spec)
+ images = ... # A batch of images with shape [batch_size, height, width, 3].
+ module = hub.Module(module_spec)
+ features = module(images) # A batch with shape [batch_size, num_features].
+```
+
+It also defines the corresponding named signature.
+
+### Signature specification
+
+The named signature for extracting image feature vectors is invoked as
+
+```python
+ outputs = module(dict(images=images), signature="image_feature_vector",
+ as_dict=True)
+ features = outputs["default"]
+```
+
+The input follows the general convention for
+[input of images](#input).
+
+The outputs dictionary contains a `"default"` output of dtype `float32` and
+shape `[batch_size, num_features]`. The `batch_size` is the same as in the
+input, but not known at graph construction time. `num_features` is a known,
+module-specific constant independent of input size.
+
+These feature vectors are meant to be usable for classification with a simple
+feed-forward classifier (like the pooled features from the topmost convolutional
+layer in a typical CNN for image classification).
+
+Applying dropout to the output features (or not) should be left to the module
+consumer. The module itself should not perform dropout on the actual outputs
+(even if it uses dropout internally in other places).
+
+The outputs dictionary may provide further outputs, for example, the activations
+of hidden layers inside the module. Their keys and values are module-dependent.
+It is recommended to prefix architecture-dependent keys with an architecture
+name (e.g., to avoid confusing the intermediate layer `"InceptionV3/Mixed_5c"`
+with the topmost convolutional layer `"InceptionV2/Mixed_5c"`).
+
+
+## Image Classification
+
+### Usage summary
+
+**Image classification** maps the pixels of an image to linear scores (logits)
+for membership in the classes of a taxonomy _selected by the module publisher_.
+This allows consumers to draw conclusions from the particular classification
+learned by the publisher module, and not just its underlying features (cf.
+[Image Feature Vector](#feature-vector)).
+
+A module for image feature extraction has a default signature that maps a batch
+of images to a batch of logits. It can be used like so:
+
+```python
+ module_spec = hub.load_module_spec("path/to/module")
+ height, width = hub.get_expected_image_size(module_spec)
+ images = ... # A batch of images with shape [batch_size, height, width, 3].
+ module = hub.Module(module_spec)
+ logits = module(images) # A batch with shape [batch_size, num_classes].
+```
+
+It also defines the corresponding named signature.
+
+### Signature specification
+
+The named signature for extracting image feature vectors is invoked as
+
+```python
+ outputs = module(dict(images=images), signature="image_classification",
+ as_dict=True)
+ logits = outputs["default"]
+```
+
+The input follows the general convention for
+[input of images](#input).
+
+The outputs dictionary contains a `"default"` output of dtype `float32` and
+shape `[batch_size, num_classes]`. The `batch_size` is the same as in the input,
+but not known at graph construction time. `num_classes` is the number of classes
+in the classification, which is a known constant independent of input size.
+
+Evaluating `outputs["default"][i, c]` yields a score predicting the membership
+of example `i` in the class with index `c`.
+
+It depends on the underlying classification whether these scores are meant to be
+used with softmax (for mutually exclusive classes), sigmoid (for orthogonal
+classes), or something else. The module documentation should describe this,
+and refer to a definition of the class indices.
+
+The outputs dictionary may provide further outputs, for example, the activations
+of hidden layers inside the module. Their keys and values are module-dependent.
+It is recommended to prefix architecture-dependent keys with an architecture
+name (e.g., to avoid confusing the intermediate layer `"InceptionV3/Mixed_5c"`
+with the topmost convolutional layer `"InceptionV2/Mixed_5c"`).
+
+
+## Image input
+
+This is common to all types of image modules and image signatures.
+
+A signature that takes a batch of images as input accepts them as a dense 4-D
+tensor of dtype `float32` and shape `[batch_size, height, width, 3]` whose
+elements are RGB color values of pixels normalized to the range [0, 1]. This is
+what you get from `tf.image.decode_*()` followed by
+`tf.image.convert_image_dtype(..., tf.float32)`.
+
+A module with exactly one (or one principal) input of images uses the name
+`"images"` for this input.
+
+The module accepts any `batch_size`, and correspondingly sets the first
+dimension of TensorInfo.tensor_shape to "unknown". The last dimension is fixed
+to the number `3` of RGB channels. The `height` and `width` dimensions are
+fixed to the expected size of input images. (Future work may remove that
+restriction for fully convolutional modules.)
+
+Consumers of the module should not inspect the shape directly, but obtain
+the size information by calling hub.get_expected_image_size()
+on the module or module spec, and are expected to resize input images
+accordingly (typically before/during batching).
+
+For simplicity, TF-Hub modules use the `channels_last`
+(or `NHWC`) layout of Tensors, and leave it to TensorFlow's graph optimizer
+to rewrite to `channels_first` (or `NCHW`) if needed. It has been doing that
+by default since TensorFlow version 1.7.
diff --git a/site/en/hub/common_signatures/index.md b/site/en/hub/common_signatures/index.md
new file mode 100644
index 00000000000..05eacc8b37f
--- /dev/null
+++ b/site/en/hub/common_signatures/index.md
@@ -0,0 +1,25 @@
+
+# Common Signatures for TF Hub Modules
+
+## Introduction
+
+[TensorFlow Hub](https://tfhub.dev) hosts models for a variety of tasks. Models
+for the same task are encouraged to implement a common API so that model
+consumers can easily exchange them without modifying the code that uses them,
+even if they come from different publishers.
+
+The goal is to make exchanging different models for the same task as simple as
+switching a string-valued hyperparameter. With that, model consumers can easily
+find the best one for their problem.
+
+This directory collects specifications of common signatures for modules in the
+[TF1 Hub format](../tf1_hub_module.md).
+
+Note that the TF1 Hub format has been **deprecated** in favor of the
+[TF2 SavedModel format](../tf2_saved_model.md) and its
+[Common SavedModel APIs](../common_saved_model_apis/index.md).
+
+## Signatures
+
+* [Image Signatures](images.md)
+* [Text Signatures](text.md)
diff --git a/site/en/hub/common_signatures/text.md b/site/en/hub/common_signatures/text.md
new file mode 100644
index 00000000000..3ea8f27c91d
--- /dev/null
+++ b/site/en/hub/common_signatures/text.md
@@ -0,0 +1,46 @@
+
+# Common Signatures for Text
+
+This page describes common signatures that should be implemented by modules in
+the [TF1 Hub format](../tf1_hub_module.md) for tasks that accept text inputs.
+(For the [TF2 SavedModel format](../tf2_saved_model.md), see the analogous
+[SavedModel API](../common_saved_model_apis/text.md).)
+
+## Text feature vector
+
+A **text feature vector** module creates a dense vector representation
+from text features.
+It accepts a batch of strings of shape `[batch_size]` and maps them to
+a `float32` tensor of shape `[batch_size, N]`. This is often called
+**text embedding** in dimension `N`.
+
+### Basic usage
+
+```python
+ embed = hub.Module("path/to/module")
+ representations = embed([
+ "A long sentence.",
+ "single-word",
+ "http://example.com"])
+```
+
+### Feature column usage
+
+```python
+ feature_columns = [
+ hub.text_embedding_column("comment", "path/to/module", trainable=False),
+ ]
+ input_fn = tf.estimator.inputs.numpy_input_fn(features, labels, shuffle=True)
+ estimator = tf.estimator.DNNClassifier(hidden_units, feature_columns)
+ estimator.train(input_fn, max_steps=100)
+```
+
+## Notes
+
+Modules have been pre-trained on different domains and/or tasks,
+and therefore not every text feature vector module would be suitable for
+your problem. E.g.: some modules could have been trained on a single language.
+
+This interface does not allow fine-tuning of the text representation on TPUs,
+because it requires the module to instantiate both string processing and the
+trainable variables at the same time.
diff --git a/site/en/hub/community.md b/site/en/hub/community.md
new file mode 100644
index 00000000000..a7a4c2bf0ec
--- /dev/null
+++ b/site/en/hub/community.md
@@ -0,0 +1,6 @@
+
+# Community and support
+
+* The source code is available on [GitHub](https://github.com/tensorflow/hub).
+ We use [GitHub issues](https://github.com/tensorflow/hub/issues) for
+ tracking feature requests and bugs.
\ No newline at end of file
diff --git a/site/en/hub/contribute.md b/site/en/hub/contribute.md
new file mode 100644
index 00000000000..e537f79f766
--- /dev/null
+++ b/site/en/hub/contribute.md
@@ -0,0 +1,16 @@
+
+# Contribute
+
+To learn more about how to publish a model or model collection on
+[tfhub.dev](https://tfhub.dev/), see the [becoming_a_publisher](publish.md)
+guide.
+
+You can find more information of how to contribute to the
+[TensorFlow Hub library](https://github.com/tensorflow/hub) in our
+[GitHub contributing guide](https://github.com/tensorflow/hub/blob/master/CONTRIBUTING.md).
+
+Content published to tfhub.dev can be automatically mirrored to other model
+hubs, provided it follows a specified format and is permitted by our Terms
+(https://tfhub.dev/terms). See [our publishing documentation](publish.md) for
+more details, and [our contribution documentation](contribute_a_model.md) if
+you'd like to opt-out of mirroring.
diff --git a/site/en/hub/hosting.md b/site/en/hub/hosting.md
new file mode 100644
index 00000000000..ce2ce76b0a6
--- /dev/null
+++ b/site/en/hub/hosting.md
@@ -0,0 +1,175 @@
+
+# Model hosting protocol
+
+This document describes the URL conventions used when hosting all model types on
+[tfhub.dev](https://tfhub.dev) - TFJS, TF Lite and TensorFlow models. It also
+describes the HTTP(S)-based protocol implemented by the `tensorflow_hub` library
+in order to load TensorFlow models from [tfhub.dev](https://tfhub.dev) and
+compatible services into TensorFlow programs.
+
+Its key feature is to use the same URL in code to load a model and in a browser
+to view the model documentation.
+
+## General URL conventions
+
+[tfhub.dev](https://tfhub.dev) supports the following URL formats:
+
+* TF Hub publishers follow `https://tfhub.dev/`
+* TF Hub collections follow
+ `https://tfhub.dev//collection/`
+* TF Hub models have versioned url
+ `https://tfhub.dev///` and unversioned url
+ `https://tfhub.dev//` that resolves to the latest
+ version of the model.
+
+TF Hub models can be downloaded as compressed assets by appending URL parameters
+to the [tfhub.dev](https://tfhub.dev) model URL. However, the URL parameters
+required to achieve that depend on the model type:
+
+* TensorFlow models (both SavedModel and TF1 Hub formats): append
+ `?tf-hub-format=compressed` to the TensorFlow model url.
+* TFJS models: append `?tfjs-format=compressed` to the TFJS model url to
+ download the compressed or `/model.json?tfjs-format=file` to read if from
+ remote storage.
+* TF lite models: append `?lite-format=tflite` to the TF Lite model url.
+
+For example:
+
+
+
+Additionally, some models also are hosted in a format that can be read directly
+from remote storage without being downloaded. This is especially useful if there
+is no local storage available, such as running a TF.js model in the browser or
+loading a SavedModel on [Colab](https://colab.research.google.com/). Be
+conscious that reading models that are hosted remotely without being downloaded
+locally may increase latency.
+
+
+
+
+
+
+
+
+
Type
+
Model URL
+
Response type
+
URL param
+
Request URL
+
+
+
TensorFlow (SavedModel, TF1 Hub format)
+
https://tfhub.dev/google/spice/2
+
String (Path to GCS folder where the uncompressed model is stored)
+
+## tensorflow_hub library protocol
+
+This section describes how we host models on [tfhub.dev](https://tfhub.dev) for
+use with the tensorflow_hub library. If you want to host your own model
+repository to work with the tensorflow_hub library, your HTTP(s) distribution
+service should provide an implementation of this protocol.
+
+Note that this section does not address hosting TF Lite and TFJS models since
+they are not downloaded via the `tensorflow_hub` library. For more information
+on hosting these model types, please check [above](#general-url-conventions).
+
+### Compressed Hosting
+
+Models are stored on [tfhub.dev](https://tfhub.dev) as compressed tar.gz files.
+By default, the tensorflow_hub library automatically downloads the compressed
+model. They can also be manually downloaded by appending the
+`?tf-hub-format=compressed` to the model url, for example:
+
+```shell
+wget https://tfhub.dev/tensorflow/albert_en_xxlarge/1?tf-hub-format=compressed
+```
+
+The root of the archive is the root of the model directory and should contain a
+SavedModel, as in this example:
+
+```shell
+# Create a compressed model from a SavedModel directory.
+$ tar -cz -f model.tar.gz --owner=0 --group=0 -C /tmp/export-model/ .
+
+# Inspect files inside a compressed model
+$ tar -tf model.tar.gz
+./
+./variables/
+./variables/variables.data-00000-of-00001
+./variables/variables.index
+./assets/
+./saved_model.pb
+```
+
+Tarballs for use with the legacy
+[TF1 Hub format](https://www.tensorflow.org/hub/tf1_hub_module) will also
+contain a `./tfhub_module.pb` file.
+
+When one of `tensorflow_hub` library model loading APIs is invoked
+([hub.KerasLayer](https://www.tensorflow.org/hub/api_docs/python/hub/KerasLayer),
+[hub.load](https://www.tensorflow.org/hub/api_docs/python/hub/load), etc) the
+library downloads the model, uncompresses the model and caches it locally. The
+`tensorflow_hub` library expects that model URLs are versioned and that the
+model content of a given version is immutable, so that it can be cached
+indefinitely. Learn more about [caching models](caching.md).
+
+
+
+### Uncompressed Hosting
+
+When the environment variable `TFHUB_MODEL_LOAD_FORMAT` or the command-line flag
+`--tfhub_model_load_format` is set to `UNCOMPRESSED`, the model is read directly
+from remote storage (GCS) instead of being downloaded and uncompressed locally.
+When this behavior is enabled the library appends `?tf-hub-format=uncompressed`
+to the model URL. That request returns the path to the folder on GCS that
+contains the uncompressed model files. As an example, \
+`https://tfhub.dev/google/spice/2?tf-hub-format=uncompressed` \
+returns \
+`gs://kaggle-tfhub-models-uncompressed/tfhub-modules/google/spice/2/uncompressed`
+in the body of the 303 response. The library then reads the model from that GCS
+destination.
diff --git a/site/en/hub/images/action_recognition.gif b/site/en/hub/images/action_recognition.gif
new file mode 100644
index 00000000000..a58c22ac8c3
Binary files /dev/null and b/site/en/hub/images/action_recognition.gif differ
diff --git a/site/en/hub/images/bert.png b/site/en/hub/images/bert.png
new file mode 100644
index 00000000000..e36f69c9c7b
Binary files /dev/null and b/site/en/hub/images/bert.png differ
diff --git a/site/en/hub/images/bert_preprocess.png b/site/en/hub/images/bert_preprocess.png
new file mode 100644
index 00000000000..18b3b435d1b
Binary files /dev/null and b/site/en/hub/images/bert_preprocess.png differ
diff --git a/site/en/hub/images/bert_preprocess_wide.png b/site/en/hub/images/bert_preprocess_wide.png
new file mode 100644
index 00000000000..b414196724e
Binary files /dev/null and b/site/en/hub/images/bert_preprocess_wide.png differ
diff --git a/site/en/hub/images/bit_blog.jpg b/site/en/hub/images/bit_blog.jpg
new file mode 100644
index 00000000000..260415bf0b1
Binary files /dev/null and b/site/en/hub/images/bit_blog.jpg differ
diff --git a/site/en/hub/images/boundless.png b/site/en/hub/images/boundless.png
new file mode 100644
index 00000000000..ccc52d17f84
Binary files /dev/null and b/site/en/hub/images/boundless.png differ
diff --git a/site/en/hub/images/colab_logo.svg b/site/en/hub/images/colab_logo.svg
new file mode 100644
index 00000000000..d03f1106221
--- /dev/null
+++ b/site/en/hub/images/colab_logo.svg
@@ -0,0 +1 @@
+
diff --git a/site/en/hub/images/food.png b/site/en/hub/images/food.png
new file mode 100644
index 00000000000..41865be3984
Binary files /dev/null and b/site/en/hub/images/food.png differ
diff --git a/site/en/hub/images/gan_faces.gif b/site/en/hub/images/gan_faces.gif
new file mode 100644
index 00000000000..a34b8d517f4
Binary files /dev/null and b/site/en/hub/images/gan_faces.gif differ
diff --git a/site/en/hub/images/github_icon.svg b/site/en/hub/images/github_icon.svg
new file mode 100644
index 00000000000..0a607bb98b3
--- /dev/null
+++ b/site/en/hub/images/github_icon.svg
@@ -0,0 +1 @@
+
diff --git a/site/en/hub/images/guide_basics.png b/site/en/hub/images/guide_basics.png
new file mode 100644
index 00000000000..e6aee34f516
Binary files /dev/null and b/site/en/hub/images/guide_basics.png differ
diff --git a/site/en/hub/images/image_classification.png b/site/en/hub/images/image_classification.png
new file mode 100644
index 00000000000..a3840e3482c
Binary files /dev/null and b/site/en/hub/images/image_classification.png differ
diff --git a/site/en/hub/images/interpolation.png b/site/en/hub/images/interpolation.png
new file mode 100644
index 00000000000..d2f062da7c1
Binary files /dev/null and b/site/en/hub/images/interpolation.png differ
diff --git a/site/en/hub/images/library_download_cache.png b/site/en/hub/images/library_download_cache.png
new file mode 100644
index 00000000000..1b581a4a819
Binary files /dev/null and b/site/en/hub/images/library_download_cache.png differ
diff --git a/site/en/hub/images/object_detection.png b/site/en/hub/images/object_detection.png
new file mode 100644
index 00000000000..57b327099ae
Binary files /dev/null and b/site/en/hub/images/object_detection.png differ
diff --git a/site/en/hub/images/odml.png b/site/en/hub/images/odml.png
new file mode 100644
index 00000000000..29bf3bcc61b
Binary files /dev/null and b/site/en/hub/images/odml.png differ
diff --git a/site/en/hub/images/similarity.png b/site/en/hub/images/similarity.png
new file mode 100644
index 00000000000..3155e8706e1
Binary files /dev/null and b/site/en/hub/images/similarity.png differ
diff --git a/site/en/hub/images/spice_blog.png b/site/en/hub/images/spice_blog.png
new file mode 100644
index 00000000000..cf19769e6d8
Binary files /dev/null and b/site/en/hub/images/spice_blog.png differ
diff --git a/site/en/hub/images/spice_color.png b/site/en/hub/images/spice_color.png
new file mode 100644
index 00000000000..35b68d7c444
Binary files /dev/null and b/site/en/hub/images/spice_color.png differ
diff --git a/site/en/hub/images/stackoverflow_icon.svg b/site/en/hub/images/stackoverflow_icon.svg
new file mode 100644
index 00000000000..491a75e464d
--- /dev/null
+++ b/site/en/hub/images/stackoverflow_icon.svg
@@ -0,0 +1 @@
+
diff --git a/site/en/hub/images/style_transfer.png b/site/en/hub/images/style_transfer.png
new file mode 100644
index 00000000000..d0427408830
Binary files /dev/null and b/site/en/hub/images/style_transfer.png differ
diff --git a/site/en/hub/images/super_resolution.png b/site/en/hub/images/super_resolution.png
new file mode 100644
index 00000000000..7d3f3741077
Binary files /dev/null and b/site/en/hub/images/super_resolution.png differ
diff --git a/site/en/hub/images/text_video.gif b/site/en/hub/images/text_video.gif
new file mode 100644
index 00000000000..5fe639b1eea
Binary files /dev/null and b/site/en/hub/images/text_video.gif differ
diff --git a/site/en/hub/images/yamnet.png b/site/en/hub/images/yamnet.png
new file mode 100644
index 00000000000..416956ad6fb
Binary files /dev/null and b/site/en/hub/images/yamnet.png differ
diff --git a/site/en/hub/installation.md b/site/en/hub/installation.md
new file mode 100644
index 00000000000..2381fbea614
--- /dev/null
+++ b/site/en/hub/installation.md
@@ -0,0 +1,57 @@
+
+# Installation
+
+## Installing tensorflow_hub
+
+The `tensorflow_hub` library can be installed alongside TensorFlow 1 and
+TensorFlow 2. We recommend that new users start with TensorFlow 2 right away,
+and current users upgrade to it.
+
+### Use with TensorFlow 2
+
+Use [pip](https://pip.pypa.io/) to
+[install TensorFlow 2](https://www.tensorflow.org/install) as usual. (See there
+for extra instructions about GPU support.) Then install a current version of
+[`tensorflow-hub`](https://pypi.org/project/tensorflow-hub/) next to it (must be
+0.5.0 or newer).
+
+```bash
+$ pip install "tensorflow>=2.0.0"
+$ pip install --upgrade tensorflow-hub
+```
+
+The TF1-style API of TensorFlow Hub works with the v1 compatibility mode of
+TensorFlow 2.
+
+### Legacy use with TensorFlow 1
+
+TensorFlow 1.15 is the only version of TensorFlow 1.x still supported by the
+`tensorflow_hub` library (as of release 0.11.0). TensorFlow 1.15 defaults to
+TF1-compatible behavior but contains many TF2 features under the hood to allow
+some use of TensorFlow Hub's TF2-style APIs.
+
+```bash
+$ pip install "tensorflow>=1.15,<2.0"
+$ pip install --upgrade tensorflow-hub
+```
+
+### Use of pre-release versions
+
+The pip packages `tf-nightly` and `tf-hub-nightly` are built automatically from
+the source code on github, with no release testing. This lets developers try out
+the latest code without [building from source](build_from_source.md).
+
+```bash
+$ pip install tf-nightly
+$ pip install --upgrade tf-hub-nightly
+```
+
+## Next Steps
+
+- [Library overview](lib_overview.md)
+- Tutorials:
+ - [Text classification](https://github.com/tensorflow/docs/blob/master/site/en/hub/tutorials/tf2_text_classification.ipynb)
+ - [Image classification](https://github.com/tensorflow/docs/blob/master/site/en/hub/tutorials/tf2_image_retraining.ipynb)
+ - Additional examples
+ [on GitHub](https://github.com/tensorflow/hub/blob/master/examples/README.md)
+- Find models on [tfhub.dev](https://tfhub.dev).
\ No newline at end of file
diff --git a/site/en/hub/lib_overview.md b/site/en/hub/lib_overview.md
new file mode 100644
index 00000000000..c480ad2fbdf
--- /dev/null
+++ b/site/en/hub/lib_overview.md
@@ -0,0 +1,50 @@
+
+# TensorFlow Hub Library Overview
+
+The [`tensorflow_hub`](https://github.com/tensorflow/hub) library lets you
+download and reuse trained models in your TensorFlow program with a minimum
+amount of code. The main way to load a trained model is using the
+`hub.KerasLayer` API.
+
+```python
+import tensorflow_hub as hub
+
+embed = hub.KerasLayer("https://tfhub.dev/google/nnlm-en-dim128/2")
+embeddings = embed(["A long sentence.", "single-word", "http://example.com"])
+print(embeddings.shape, embeddings.dtype)
+```
+**Note:** This documentation uses TFhub.dev URL handles in examples. See more
+information regarding other valid handle types [here](tf2_saved_model.md#model_handles).
+
+## Setting the cache location for downloads.
+
+By default, `tensorflow_hub` uses a system-wide, temporary directory to cache
+downloaded and uncompressed models. See [Caching](caching.md) for options to use
+other, possibly more persistent locations.
+
+## API stability
+
+Although we hope to prevent breaking changes, this project is still under active
+development and is not yet guaranteed to have a stable API or model format.
+
+## Fairness
+
+As in all of machine learning, [fairness](http://ml-fairness.com) is an
+[important](https://research.googleblog.com/2016/10/equality-of-opportunity-in-machine.html)
+consideration. Many pre-trained models are trained on large datasets. When
+reusing any model, it’s important to be mindful of what data the model was
+trained on (and whether there are any existing biases there), and how these
+might impact your use of it.
+
+## Security
+
+Since they contain arbitrary TensorFlow graphs, models can be thought of as
+programs.
+[Using TensorFlow Securely](https://github.com/tensorflow/tensorflow/blob/master/SECURITY.md)
+describes the security implications of referencing a model from an untrusted
+source.
+
+## Next Steps
+
+- [Use the library](tf2_saved_model.md)
+- [Reusable SavedModels](reusable_saved_models.md)
diff --git a/site/en/hub/migration_tf2.md b/site/en/hub/migration_tf2.md
new file mode 100644
index 00000000000..c2cc4b50759
--- /dev/null
+++ b/site/en/hub/migration_tf2.md
@@ -0,0 +1,114 @@
+
+# Migrating from TF1 to TF2 with TensorFlow Hub
+
+This page explains how to keep using TensorFlow Hub while migrating your
+TensorFlow code from TensorFlow 1 to TensorFlow 2. It complements TensorFlow's
+general [migration guide](https://www.tensorflow.org/guide/migrate).
+
+For TF2, TF Hub has switched away from the legacy `hub.Module` API for building
+a `tf.compat.v1.Graph` like `tf.contrib.v1.layers` do. Instead, there is now a
+`hub.KerasLayer` for use alongside other Keras layers for building a
+`tf.keras.Model` (typically in TF2's new
+[eager execution environment](https://www.tensorflow.org/api_docs/python/tf/executing_eagerly))
+and its underlying `hub.load()` method for low-level TensorFlow code.
+
+The `hub.Module` API remains available in the `tensorflow_hub` library for use
+in TF1 and in the TF1 compatibility mode of TF2. It can only load models in the
+[TF1 Hub format](tf1_hub_module.md).
+
+The new API of `hub.load()` and `hub.KerasLayer` works for TensorFlow 1.15 (in
+eager and graph mode) and in TensorFlow 2. This new API can load the new
+[TF2 SavedModel](tf2_saved_model.md) assets, and, with the restrictions laid out
+in the [model compatibility guide](model_compatibility.md), the legacy models in
+TF1 Hub format.
+
+In general, it is recommended to use new API wherever possible.
+
+## Summary of the new API
+
+`hub.load()` is the new low-level function to load a SavedModel from TensorFlow
+Hub (or compatible services). It wraps TF2's `tf.saved_model.load()`;
+TensorFlow's [SavedModel Guide](https://www.tensorflow.org/guide/saved_model)
+describes what you can do with the result.
+
+```python
+m = hub.load(handle)
+outputs = m(inputs)
+```
+
+The `hub.KerasLayer` class calls `hub.load()` and adapts the result for use in
+Keras alongside other Keras layers. (It may even be a convenient wrapper for
+loaded SavedModels used in other ways.)
+
+```python
+model = tf.keras.Sequential([
+ hub.KerasLayer(handle),
+ ...])
+```
+
+Many tutorials show these APIs in action. Here are some examples:
+
+* [Text classification example notebook](https://github.com/tensorflow/docs/blob/master/site/en/hub/tutorials/tf2_text_classification.ipynb)
+* [Image classification example notebook](https://github.com/tensorflow/docs/blob/master/site/en/hub/tutorials/tf2_image_retraining.ipynb)
+
+### Using the new API in Estimator training
+
+If you use a TF2 SavedModel in an Estimator for training with parameter servers
+(or otherwise in a TF1 Session with variables placed on remote devices), you
+need to set `experimental.share_cluster_devices_in_session` in the tf.Session's
+ConfigProto, or else you will get an error like "Assigned device
+'/job:ps/replica:0/task:0/device:CPU:0' does not match any device."
+
+The necessary option can be set like
+
+```python
+session_config = tf.compat.v1.ConfigProto()
+session_config.experimental.share_cluster_devices_in_session = True
+run_config = tf.estimator.RunConfig(..., session_config=session_config)
+estimator = tf.estimator.Estimator(..., config=run_config)
+```
+
+Starting with TF2.2, this option is no longer experimental, and the
+`.experimental` piece can be dropped.
+
+## Loading legacy models in TF1 Hub format
+
+It can happen that a new TF2 SavedModel is not yet available for your use-case
+and you need to load an legacy model in TF1 Hub format. Starting in
+`tensorflow_hub` release 0.7, you can use legacy model in TF1 Hub format
+together with `hub.KerasLayer` as shown below:
+
+```python
+m = hub.KerasLayer(handle)
+tensor_out = m(tensor_in)
+```
+
+Additionally `KerasLayer` exposes the ability to specify `tags`, `signature`,
+`output_key` and `signature_outputs_as_dict` for more specific usages of legacy
+models in TF1 Hub format and legacy SavedModels.
+
+For more information on TF1 Hub format compatibility see the
+[model compatibility guide](model_compatibility.md).
+
+## Using lower level APIs
+
+Legacy TF1 Hub format models can be loaded via `tf.saved_model.load`. Instead of
+
+```python
+# DEPRECATED: TensorFlow 1
+m = hub.Module(handle, tags={"foo", "bar"})
+tensors_out_dict = m(dict(x1=..., x2=...), signature="sig", as_dict=True)
+```
+
+it is recommended to use:
+
+```python
+# TensorFlow 2
+m = hub.load(path, tags={"foo", "bar"})
+tensors_out_dict = m.signatures["sig"](x1=..., x2=...)
+```
+
+In these examples `m.signatures` is a dict of TensorFlow
+[concrete functions](https://www.tensorflow.org/tutorials/customization/performance#tracing)
+keyed by signature names. Calling such a function computes all its outputs, even
+if unused. (This is different from the lazy evaluation of TF1's graph mode.)
diff --git a/site/en/hub/model_compatibility.md b/site/en/hub/model_compatibility.md
new file mode 100644
index 00000000000..e37ed717c3b
--- /dev/null
+++ b/site/en/hub/model_compatibility.md
@@ -0,0 +1,144 @@
+
+# Model compatibility for TF1/TF2
+
+## TF Hub model formats
+
+TF Hub offers reusable model pieces that can be loaded back, built upon, and
+possibly be retrained in a TensorFlow program. These come in two different
+formats:
+
+* The custom [TF1 Hub format](https://www.tensorflow.org/hub/tf1_hub_module) .
+ Its main intended use is in TF1 (or TF1 compatibility mode in TF2) via its
+ [hub.Module API](https://www.tensorflow.org/hub/api_docs/python/hub/Module).
+ Full compatibility details [below](#compatibility_of_hubmodule).
+* The native [TF2 SavedModel](https://www.tensorflow.org/hub/tf2_saved_model)
+ format. Its main intended use is in TF2 via the
+ [hub.load](https://www.tensorflow.org/hub/api_docs/python/hub/load) and
+ [hub.KerasLayer](https://www.tensorflow.org/hub/api_docs/python/hub/KerasLayer)
+ APIs. Full compatibility details [below](#compatibility_of_tf2_savedmodel).
+
+The model format can be found on the model page on
+[tfhub.dev](https://tfhub.dev). Model **loading/inference**, **fine-tuning** or
+**creation** might not be supported in TF1/2 based on the model formats.
+
+## Compatibility of the TF1 Hub format {:#compatibility_of_hubmodule}
+
+
+ Supported for a hub.KerasLayer used in tf.keras.Model when trained with
+ Model.fit() or trained in an Estimator whose model_fn wraps the Model per the custom model_fn guide.
+
+ Note: hub.KerasLayer does not
+ fill in graph collections like the old tf.compat.v1.layers or hub.Module
+ APIs did.
+
[1] "TF1 compat mode in TF2" refers to the combined
+ effect of importing TF2 with
+ import tensorflow.compat.v1 as tf
+ and running
+ tf.disable_v2_behavior()
+ as described in the
+ TensorFlow Migration guide
+ .
diff --git a/site/en/hub/model_formats.md b/site/en/hub/model_formats.md
new file mode 100644
index 00000000000..73ae7c247a1
--- /dev/null
+++ b/site/en/hub/model_formats.md
@@ -0,0 +1,79 @@
+
+# Model formats
+
+[tfhub.dev](https://tfhub.dev) hosts the following model
+formats: TF2 SavedModel, TF1 Hub format, TF.js and TFLite. This page provides an
+overview of each model format.
+
+Content published to tfhub.dev can be automatically mirrored to other model
+hubs, provided it follows a specified format and is permitted by our Terms
+(https://tfhub.dev/terms). See [our publishing documentation](publish.md) for
+more details, and [our contribution documentation](contribute_a_model.md) if
+you'd like to opt-out of mirroring.
+
+## TensorFlow formats
+
+[tfhub.dev](https://tfhub.dev) hosts TensorFlow models in the TF2 SavedModel
+format and TF1 Hub format. We recommend using models in the standardized TF2
+SavedModel format instead of the deprecated TF1 Hub format when possible.
+
+### SavedModel
+
+TF2 SavedModel is the recommended format for sharing TensorFlow models. You can
+learn more about the SavedModel format in the
+[TensorFlow SavedModel](https://www.tensorflow.org/guide/saved_model) guide.
+
+You can browse SavedModels on tfhub.dev by using the TF2 version filter on the
+[tfhub.dev browse page](https://tfhub.dev/s?subtype=module,placeholder) or by
+following
+[this link](https://tfhub.dev/s?subtype=module,placeholder&tf-version=tf2).
+
+You can use SavedModels from tfhub.dev without depending on the `tensorflow_hub`
+library, since this format is a part of core TensorFlow.
+
+Learn more about SavedModels on TF Hub:
+
+* [Using TF2 SavedModels](tf2_saved_model.md)
+* [Exporting a TF2 SavedModel](exporting_tf2_saved_model.md)
+* [TF1/TF2 compatibility of TF2 SavedModels](model_compatibility.md)
+
+### TF1 Hub format
+
+The TF1 Hub format is a custom serialization format used in by TF Hub library.
+The TF1 Hub format is similar to the SavedModel format of TensorFlow 1 on a
+syntactic level (same file names and protocol messages) but semantically
+different to allow for module reuse, composition and re-training (e.g.,
+different storage of resource initializers, different tagging conventions for
+metagraphs). The easiest way to tell them apart on disk is the presence or
+absence of the `tfhub_module.pb` file.
+
+You can browse models in the TF1 Hub format on tfhub.dev by using the TF1
+version filter on the
+[tfhub.dev browse page](https://tfhub.dev/s?subtype=module,placeholder) or by
+following
+[this link](https://tfhub.dev/s?subtype=module,placeholder&tf-version=tf1).
+
+Learn more about models in TF1 Hub format on TF Hub:
+
+* [Using TF1 Hub format models](tf1_hub_module.md)
+* [Exporting a model in the TF1 Hub format](exporting_hub_format.md)
+* [TF1/TF2 compatibility of TF1 Hub format](model_compatibility.md)
+
+## TFLite format
+
+The TFLite format is used for on-device inference. You can learn more at the
+[TFLite documentation](https://www.tensorflow.org/lite).
+
+You can browse TF Lite models on tfhub.dev by using the TF Lite model format
+filter on the
+[tfhub.dev browse page](https://tfhub.dev/s?subtype=module,placeholder) or by
+following [this link](https://tfhub.dev/lite).
+
+## TFJS format
+
+The TF.js format is used for in-browser ML. You can learn more at the
+[TF.js documentation](https://www.tensorflow.org/js).
+
+You can browse TF.js models on tfhub.dev by using the TF.js model format filter
+on the [tfhub.dev browse page](https://tfhub.dev/s?subtype=module,placeholder)
+or by following [this link](https://tfhub.dev/js).
diff --git a/site/en/hub/overview.md b/site/en/hub/overview.md
new file mode 100644
index 00000000000..b6d814eba73
--- /dev/null
+++ b/site/en/hub/overview.md
@@ -0,0 +1,31 @@
+
+# TensorFlow Hub
+
+TensorFlow Hub is an open repository and library for reusable machine learning.
+The [tfhub.dev](https://tfhub.dev) repository provides many pre-trained models:
+text embeddings, image classification models, TF.js/TFLite models and much more.
+The repository is open to
+[community contributors](https://tfhub.dev/s?subtype=publisher).
+
+The [`tensorflow_hub`](https://github.com/tensorflow/hub) library lets you
+download and reuse them in your TensorFlow program with a minimum amount of
+code.
+
+```python
+import tensorflow_hub as hub
+
+model = hub.KerasLayer("https://tfhub.dev/google/nnlm-en-dim128/2")
+embeddings = model(["The rain in Spain.", "falls",
+ "mainly", "In the plain!"])
+
+print(embeddings.shape) #(4,128)
+```
+
+## Next Steps
+
+- [Find models on tfhub.dev](https://tfhub.dev)
+- [Publish models on tfhub.dev](publish.md)
+- TensorFlow Hub library
+ - [Install TensorFlow Hub](installation.md)
+ - [Library overview](lib_overview.md)
+- [Follow tutorials](tutorials)
diff --git a/site/en/hub/portability_and_deletion.md b/site/en/hub/portability_and_deletion.md
new file mode 100644
index 00000000000..67fa401d161
--- /dev/null
+++ b/site/en/hub/portability_and_deletion.md
@@ -0,0 +1,18 @@
+
+## I want to see what I’ve uploaded to TensorFlow Hub. Can I get a copy of my data?
+
+Yes. If you’d like the Kaggle Team to **send you a copy** of all of the
+data you have uploaded, please send us an email at [support@kaggle.com](mailto:support@kaggle.com)
+and we’ll respond as soon as possible.
+
+## How do I delete what I’ve uploaded to TensorFlow Hub?
+
+Similarly, if you’d like us to **delete or remove content**, please send us an
+email at [support@kaggle.com](mailto:support@kaggle.com) and we’ll delete
+all copies that we have and stop serving it on tfhub.dev. Please note:
+
+* Because TensorFlow Hub is an open-source platform, copies of your assets may
+still be retained by members of the public.
+* Deletion is permanent and cannot be undone.
+* Deletion can cause downstream breakages if users are not caching your model
+locally and/or are not properly warned prior to deletion.
diff --git a/site/en/hub/publish.md b/site/en/hub/publish.md
new file mode 100644
index 00000000000..7fc5e7c1751
--- /dev/null
+++ b/site/en/hub/publish.md
@@ -0,0 +1,19 @@
+
+# Publishing Process
+
+Thank you for considering to publish your models!
+
+**Please join the Early Access Model Publishing (EAP) on
+[Kaggle Models](https://www.kaggle.com/models):**
+
+- Email [kaggle-models@google.com](mailto:kaggle-models@google.com) and
+ provide the following to get access to EAP:
+ - (1) Your Kaggle username
+ - (2) Your desired organization slug
+ - (3) A URL to a square-shaped profile image (which is needed for the
+ organization creation)
+- Follow the
+ [documentation instructions](https://www.kaggle.com/model-publishing-instructions)
+ to create and publish your model
+- Feel free to raise any questions and get support from
+ [Kaggle Discord channel](https://discord.gg/rKEyxj9WF)
diff --git a/site/en/hub/reusable_saved_models.md b/site/en/hub/reusable_saved_models.md
new file mode 100644
index 00000000000..b2114135d77
--- /dev/null
+++ b/site/en/hub/reusable_saved_models.md
@@ -0,0 +1,208 @@
+
+# Reusable SavedModels
+
+## Introduction
+
+TensorFlow Hub hosts SavedModels for TensorFlow 2, among other assets.
+They can be loaded back into a Python program with `obj = hub.load(url)`
+[[learn more](tf2_saved_model)]. The returned `obj` is the result
+of `tf.saved_model.load()` (see TensorFlow's
+[SavedModel guide](https://www.tensorflow.org/guide/saved_model)).
+This object can have arbitrary attributes that are tf.functions,
+tf.Variables (initialized from their pre-trained values), other resources
+and, recursively, more such objects.
+
+This page describes an interface to be implemented by the loaded `obj`
+in order to be *reused* in a TensorFlow Python program.
+SavedModels conforming to this interface are called *Reusable SavedModels*.
+
+Reusing means building a larger model around `obj`, including the ability
+to fine-tune it. Fine-tuning means further training of the weights in the loaded
+`obj` as part of the surrounding model. The loss function and the
+optimizer are determined by the surrounding model; `obj` only defines
+the mapping of input to output activations (the "forward pass"), possibly
+including techniques such as dropout or batch normalization.
+
+**The TensorFlow Hub team recommends implementing the Reusable SavedModel
+interface** in all SavedModels that are meant to be reused in the above sense.
+Many utilities from the `tensorflow_hub` library, notably `hub.KerasLayer`,
+require SavedModels to implement it.
+
+### Relation to SignatureDefs
+
+This interface in terms of tf.functions and other TF2 features
+is separate from the SavedModel's signatures, which have been
+available since TF1 and continue to be used in TF2 for inference
+(such as deploying SavedModels to TF Serving or TF Lite).
+Signatures for inference are not expressive enough to support fine-tuning,
+and [`tf.function`](https://www.tensorflow.org/api_docs/python/tf/function)
+provides a more natural and expressive
+[Python API](https://www.tensorflow.org/tutorials/customization/performance)
+for the reused model.
+
+### Relation to model-building libraries
+
+A Reusable SavedModel uses only TensorFlow 2 primitives, independent of any
+particular model-building library like Keras or Sonnet. This facilitates reuse
+across model-building libraries, free from dependencies on the original
+model-building code.
+
+Some amount of adaptation will be needed load Reusable SavedModels into or save
+them from any given model-building library. For Keras,
+[hub.KerasLayer](https://www.tensorflow.org/hub/api_docs/python/hub/KerasLayer)
+provides the loading, and Keras's built-in saving in the SavedModel format has
+been redesigned for TF2 with the goal of providing a superset of this interface
+(see the
+[RFC](https://github.com/tensorflow/community/blob/master/rfcs/20190509-keras-saved-model.md)
+from May 2019).
+
+### Relation to task-specific "Common SavedModel APIs"
+
+The interface definition on this page allows for any number and type of inputs
+and outputs. The
+[Common SavedModel APIs for TF Hub](common_saved_model_apis/index.md) refine
+this general interface with usage conventions for specific tasks to make models
+easily interchangeable.
+
+## Interface definition
+
+### Attributes
+
+A Reusable SavedModel is a TensorFlow 2 SavedModel such that
+`obj = tf.saved_model.load(...)` returns an object that has the following
+attributes
+
+ * `__call__`. Required. A tf.function implementing the model's computation
+ (the "forward pass") subject to the specification below.
+
+ * `variables`: A list of tf.Variable objects, listing all the variables
+ used by any possible invocation of `__call__`, including both
+ trainable and non-trainable ones.
+
+ This list can be omitted if empty.
+
+ Note: Conveniently, this name coincides with the attribute synthesized by
+ `tf.saved_model.load(...)` when loading a TF1 SavedModel to represent
+ its `GLOBAL_VARIABLES` collection.
+
+ * `trainable_variables`: A list of tf.Variable objects such that
+ `v.trainable` is true for all elements.
+ These variables must be a subset of `variables`.
+ These are the variables to be trained when fine-tuning the object.
+ The SavedModel creator may choose to omit some variables here that were
+ originally trainable to indicate that these should not be modified during
+ fine-tuning.
+
+ This list can be omitted if empty, in particular, if the SavedModel does not
+ support fine-tuning.
+
+ * `regularization_losses`: A list of tf.functions, each taking zero inputs
+ and returning a single scalar float tensor. For fine-tuning, the
+ SavedModel user is advised to include these as additional regularization
+ terms into the loss (in the simplest case without further scaling).
+ Typically, these are used to represent weight regularizers.
+ (For lack of inputs, these tf.functions cannot express
+ activity regularizers.)
+
+ This list can be omitted if empty, in particular, if the SavedModel does not
+ support fine-tuning or does not wish to prescribe weight regularization.
+
+### The `__call__` function
+
+A Restored SavedModel `obj` has an `obj.__call__` attribute that is
+a restored tf.function and allows `obj` to be called as follows.
+
+Synopsis (pseudo-code):
+
+```python
+outputs = obj(inputs, trainable=..., **kwargs)
+```
+
+#### Arguments
+
+The arguments are as follows.
+
+ * There is one positional, required argument with a batch of input activations
+ of the SavedModel. Its type is one of
+
+ * a single Tensor for a single input,
+ * a list of Tensors for an ordered sequence of unnamed inputs,
+ * a dict of Tensors keyed by a particular set of input names.
+
+ (Future revisions of this interface may allow more general nests.)
+ The SavedModel creator chooses one of those and the tensor shapes
+ and dtypes. Where useful, some dimensions of the shape should be
+ undefined (notably batch size).
+
+ * There may be an optional keyword argument `training` that accepts a Python
+ boolean, `True` or `False`. The default is `False`.
+ If the model supports fine-tuning, and if its computation differs between
+ the two (e.g., as in dropout and batch normalization), that distinction
+ is implemented with this argument. Otherwise, this argument may be absent.
+
+ It is not required that `__call__` accept a Tensor-valued `training`
+ argument. It falls on the caller to use `tf.cond()` if necessary
+ to dispatch between them.
+
+ * The SavedModel creator may choose to accept more optional `kwargs`
+ of particular names.
+
+ * For Tensor-valued arguments, the SavedModel creator defines their
+ permissible dtypes and shapes. `tf.function` accepts a Python default
+ value on an argument that is traced with a tf.TensorSpec input.
+ Such arguments can be used to allow customization of numeric
+ hyperparameters involved in `__call__` (e.g., dropout rate).
+
+ * For Python-valued arguments, the SavedModel creator defines their
+ permissible values. Such arguments can be used as flags to make
+ discrete choices in the traced function (but mind the combinatorial
+ explosion of traces).
+
+The restored `__call__` function must provide traces for all permissible
+combinations of arguments. Flipping `training` between `True` and `False`
+must not change the permissibility of arguments.
+
+#### Result
+
+The `outputs` from calling `obj` can be
+
+ * a single Tensor for a single output,
+ * a list of Tensors for an ordered sequence of unnamed outputs,
+ * a dict of Tensors keyed by a particular set of output names.
+
+(Future revisions of this interface may allow more general nests.)
+The return type may vary depending on the Python-valued kwargs.
+This allows for flags producing extra outputs.
+The SavedModel creator defines the output dtypes and shapes and their
+dependency on inputs.
+
+
+### Named callables
+
+A Reusable SavedModel can provide multiple model pieces in the way
+described above by putting them into named subobjects, for example,
+`obj.foo`, `obj.bar` and so on.
+Each subobject provides a `__call__` method and supporting attributes
+about the variables etc. specific to that model piece.
+For the example above, there would be `obj.foo.__call__`,
+`obj.foo.variables` and so on.
+
+Note that this interface does *not* cover the approach of adding
+a bare tf.function directly as `tf.foo`.
+
+Users of Reusable SavedModels are only expected to handle one level of nesting
+(`obj.bar` but not `obj.bar.baz`). (Future revisions of this interface may allow
+deeper nesting, and may waive the requirement that the top-level object be
+callable itself.)
+
+## Closing remarks
+
+### Relation to in-process APIs
+
+This document describes an interface of a Python class which consists
+of primitives like tf.function and tf.Variable that survive a
+round-trip through serialization via `tf.saved_model.save()`
+and `tf.saved_model.load()`. However, the interface was already present
+on the original object that was passed to `tf.saved_model.save()`.
+Adaptation to that interface enables the exchange of model pieces
+across model-building APIs within a single TensorFlow program.
diff --git a/site/en/hub/tf1_hub_module.md b/site/en/hub/tf1_hub_module.md
new file mode 100644
index 00000000000..7601878dc1b
--- /dev/null
+++ b/site/en/hub/tf1_hub_module.md
@@ -0,0 +1,198 @@
+
+# TF1 Hub format
+
+At its launch in 2018, TensorFlow Hub offered a single type of asset: TF1 Hub
+format for import into TensorFlow 1 programs.
+
+This page explains how to use TF1 Hub format in TF1 (or the TF1 compatibility
+mode of TF2) with the `hub.Module` class and associated APIs. (The typical use
+is to build a `tf.Graph`, possibly inside a TF1 `Estimator`, by combining one or
+more models in TF1 Hub format with `tf.compat.layers` or `tf.layers`).
+
+Users of TensorFlow 2 (outside TF1 compatibility mode) must use
+[the new API with `hub.load()` or `hub.KerasLayer`](tf2_saved_model.md). The new
+API loads the new TF2 SavedModel asset type, but also has limited
+[support for loading TF1 Hub format into TF2](migration_tf2.md).
+
+## Using a model in TF1 Hub format
+
+### Instantiating a model in TF1 Hub format
+
+A model in TF1 Hub format is imported into a TensorFlow program by creating a
+`hub.Module` object from a string with its URL or filesystem path, such as:
+
+```python
+m = hub.Module("path/to/a/module_dir")
+```
+**Note:** See more information regarding other valid handle types [here](tf2_saved_model.md#model_handles).
+
+This adds the module's variables to the current TensorFlow graph.
+Running their initializers will read their pre-trained values from disk.
+Likewise, tables and other state is added to the graph.
+
+### Caching Modules
+
+When creating a module from a URL, the module content is downloaded and cached
+in the local system temporary directory. The location where modules are cached
+can be overridden using `TFHUB_CACHE_DIR` environment variable. For details, see
+[Caching](caching.md).
+
+### Applying a Module
+
+Once instantiated, a module `m` can be called zero or more times like a Python
+function from tensor inputs to tensor outputs:
+
+```python
+y = m(x)
+```
+
+Each such call adds operations to the current TensorFlow graph to compute
+`y` from `x`. If this involves variables with trained weights, these are
+shared between all applications.
+
+Modules can define multiple named *signatures* in order to allow being applied
+in more than one way (similar to how Python objects have *methods*).
+A module's documentation should describe the available
+signatures. The call above applies the signature named `"default"`. Any
+signature can be selected by passing its name to the optional `signature=`
+argument.
+
+If a signature has multiple inputs, they must be passed as a dict, with the keys
+defined by the signature. Likewise, if a signature has multiple outputs, these
+can be retrieved as a dict by passing `as_dict=True`, under the keys defined by
+the signature (the key `"default"` is for the single output returned if
+`as_dict=False`). So the most general form of applying a Module looks like:
+
+```python
+outputs = m(dict(apples=x1, oranges=x2), signature="fruit_to_pet", as_dict=True)
+y1 = outputs["cats"]
+y2 = outputs["dogs"]
+```
+
+A caller must supply all inputs defined by a signature, but there is no
+requirement to use all of a module's outputs.
+TensorFlow will run only those parts of the module that end up
+as dependencies of a target in `tf.Session.run()`. Indeed, module publishers may
+choose to provide various outputs for advanced uses (like activations of
+intermediate layers) along with the main outputs. Module consumers should
+handle additional outputs gracefully.
+
+### Trying out alternative modules
+
+Whenever there are multiple modules for the same task, TensorFlow Hub
+encourages to equip them with compatible signatures (interfaces)
+such that trying different ones is as easy as varying the module handle
+as a string-valued hyperparameter.
+
+To this end, we maintain a collection of recommended
+[Common Signatures](common_signatures/index.md) for popular tasks.
+
+
+## Creating a New Module
+
+### Compatibility note
+
+The TF1 Hub format is geared towards TensorFlow 1. It is only partially
+supported by TF Hub in TensorFlow 2. Please do consider publishing in the new
+[TF2 SavedModel](tf2_saved_model.md) format instead.
+
+The TF1 Hub format is similar to the SavedModel format of TensorFlow 1 on a
+syntactic level (same file names and protocol messages) but semantically
+different to allow for module reuse, composition and re-training (e.g.,
+different storage of resource initializers, different tagging conventions for
+metagraphs). The easiest way to tell them apart on disk is the presence or
+absence of the `tfhub_module.pb` file.
+
+### General approach
+
+To define a new module, a publisher calls `hub.create_module_spec()` with a
+function `module_fn`. This function constructs a graph representing the module's
+internal structure, using `tf.placeholder()` for inputs to be supplied by
+the caller. Then it defines signatures by calling
+`hub.add_signature(name, inputs, outputs)` one or more times.
+
+For example:
+
+```python
+def module_fn():
+ inputs = tf.placeholder(dtype=tf.float32, shape=[None, 50])
+ layer1 = tf.layers.dense(inputs, 200)
+ layer2 = tf.layers.dense(layer1, 100)
+ outputs = dict(default=layer2, hidden_activations=layer1)
+ # Add default signature.
+ hub.add_signature(inputs=inputs, outputs=outputs)
+
+...
+spec = hub.create_module_spec(module_fn)
+```
+
+The result of `hub.create_module_spec()` can be used, instead of a path,
+to instantiate a module object within a particular TensorFlow graph. In
+such case, there is no checkpoint, and the module instance will use the
+variable initializers instead.
+
+Any module instance can be serialized to disk via its `export(path, session)`
+method. Exporting a module serializes its definition together with the current
+state of its variables in `session` into the passed path. This can be used
+when exporting a module for the first time, as well as when exporting a fine
+tuned module.
+
+For compatibility with TensorFlow Estimators, `hub.LatestModuleExporter` exports
+modules from the latest checkpoint, just like `tf.estimator.LatestExporter`
+exports the entire model from the latest checkpoint.
+
+Module publishers should implement a [common
+signature](common_signatures/index.md) when possible, so that consumers can
+easily exchange modules and find the best one for their problem.
+
+### Real example
+
+Take a look at our [text embedding module exporter](https://github.com/tensorflow/hub/blob/master/examples/text_embeddings/export.py)
+for a real-world example of how to create a module from a common text embedding
+format.
+
+
+## Fine-Tuning
+
+Training the variables of an imported module together with those of the model
+around it is called *fine-tuning*. Fine-tuning can result in better quality, but
+adds new complications. We advise consumers to look into fine-tuning only after
+exploring simpler quality tweaks, and only if the module publisher recommends
+it.
+
+### For Consumers
+
+To enable fine-tuning, instantiate the module with
+`hub.Module(..., trainable=True)` to make its variables trainable and
+import TensorFlow's `REGULARIZATION_LOSSES`. If the module has multiple
+graph variants, make sure to pick the one appropriate for training.
+Usually, that's the one with tags `{"train"}`.
+
+Choose a training regime that does not ruin the pre-trained weights,
+for example, a lower learning rate than for training from scratch.
+
+### For Publishers
+
+To make fine-tuning easier for consumers, please be mindful of the following:
+
+* Fine-tuning needs regularization. Your module is exported with the
+ `REGULARIZATION_LOSSES` collection, which is what puts your choice of
+ `tf.layers.dense(..., kernel_regularizer=...)` etc. into what the consumer
+ gets from `tf.losses.get_regularization_losses()`. Prefer this way of
+ defining L1/L2 regularization losses.
+
+* In the publisher model, avoid defining L1/L2 regularization via the `l1_`
+ and `l2_regularization_strength` parameters of `tf.train.FtrlOptimizer`,
+ `tf.train.ProximalGradientDescentOptimizer`, and other proximal optimizers.
+ These are not exported alongside the module, and setting regularization
+ strengths globally may not be appropriate for the consumer. Except for L1
+ regularization in wide (i.e. sparse linear) or wide & deep models, it should
+ be possible to use individual regularization losses instead.
+
+* If you use dropout, batch normalization, or similar training techniques, set
+ their hyperparameters to values that make sense across many expected uses.
+ The dropout rate may have to be adjusted to the target problem's propensity
+ to overfitting. In batch normalization, the momentum (a.k.a. decay
+ coefficient) should be small enough to enable fine-tuning with small
+ datasets and/or large batches. For advanced consumers, consider adding a
+ signature that exposes control over critical hyperparameters.
diff --git a/site/en/hub/tf2_saved_model.md b/site/en/hub/tf2_saved_model.md
new file mode 100644
index 00000000000..e41337b2548
--- /dev/null
+++ b/site/en/hub/tf2_saved_model.md
@@ -0,0 +1,289 @@
+
+# SavedModels from TF Hub in TensorFlow 2
+
+The
+[SavedModel format of TensorFlow 2](https://www.tensorflow.org/guide/saved_model)
+is the recommended way to share pre-trained models and model pieces on
+TensorFlow Hub. It replaces the older [TF1 Hub format](tf1_hub_module.md) and
+comes with a new set of APIs.
+
+This page explains how to reuse TF2 SavedModels in a TensorFlow 2 program with
+the low-level `hub.load()` API and its `hub.KerasLayer` wrapper. (Typically,
+`hub.KerasLayer` is combined with other `tf.keras.layers` to build a Keras model
+or the `model_fn` of a TF2 Estimator.) These APIs can also load the legacy
+models in TF1 Hub format, within limits, see the
+[compatibility guide](model_compatibility.md).
+
+Users of TensorFlow 1 can update to TF 1.15 and then use the same APIs.
+Older versions of TF1 do not work.
+
+## Using SavedModels from TF Hub
+
+### Using a SavedModel in Keras
+
+[Keras](https://www.tensorflow.org/guide/keras/) is TensorFlow's high-level API
+for building deep learning models by composing Keras Layer objects.
+The `tensorflow_hub` library provides the class `hub.KerasLayer` that gets
+initialized with the URL (or filesystem path) of a SavedModel and then
+provides the computation from the SavedModel, including its pre-trained
+weights.
+
+Here is an example of using a pre-trained text embedding:
+
+```python
+import tensorflow as tf
+import tensorflow_hub as hub
+
+hub_url = "https://tfhub.dev/google/nnlm-en-dim128/2"
+embed = hub.KerasLayer(hub_url)
+embeddings = embed(["A long sentence.", "single-word", "http://example.com"])
+print(embeddings.shape, embeddings.dtype)
+```
+
+From this, a text classifier can be built in the usual Keras way:
+
+```python
+model = tf.keras.Sequential([
+ embed,
+ tf.keras.layers.Dense(16, activation="relu"),
+ tf.keras.layers.Dense(1, activation="sigmoid"),
+])
+```
+
+The [Text classification
+colab](https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/hub/tutorials/tf2_text_classification.ipynb)
+is a complete example how to train and evaluate such a classifier.
+
+The model weights in a `hub.KerasLayer` are set to non-trainable by default.
+See the section on fine-tuning below for how to change that. Weights are
+shared between all applications of the same layer object, as usual in Keras.
+
+
+### Using a SavedModel in an Estimator
+
+Users of TensorFlow's
+[Estimator](https://www.tensorflow.org/tutorials/distribute/multi_worker_with_estimator)
+API for distributed training can use SavedModels from TF Hub by
+writing their `model_fn` in terms of `hub.KerasLayer` among other
+`tf.keras.layers`.
+
+
+### Behind the scenes: SavedModel downloading and caching
+
+Using a SavedModel from TensorFlow Hub (or other HTTPS servers that implement
+its [hosting](hosting.md) protocol) downloads and decompresses it to the local
+filesystem if not already present. The environment variable `TFHUB_CACHE_DIR`
+can be set to override the default temporary location for caching the downloaded
+and uncompressed SavedModels. For details, see [Caching](caching.md).
+
+### Using a SavedModel in low-level TensorFlow
+#### Model Handles
+
+SavedModels can be loaded from a specified `handle`, where the `handle` is a
+filesystem path, valid TFhub.dev model URL (e.g. "https://tfhub.dev/...").
+Kaggle Models URLs mirror TFhub.dev handles in accordance with our Terms and the
+license associated with the model assets, e.g., "https://www.kaggle.com/...".
+Handles from Kaggle Models are equivalent to their corresponding TFhub.dev
+handle.
+
+The function `hub.load(handle)` downloads and decompresses a SavedModel
+(unless `handle` is already a filesystem path) and then returns the result
+of loading it with TensorFlow's built-in function `tf.saved_model.load()`.
+Therefore, `hub.load()` can handle any valid SavedModel (unlike its
+predecessor `hub.Module` for TF1).
+
+#### Advanced topic: what to expect from the SavedModel after loading
+
+Depending on the contents of the SavedModel, the result of
+`obj = hub.load(...)` can be invoked in various ways (as explained in
+much greater detail in TensorFlow's [SavedModel
+Guide](https://www.tensorflow.org/guide/saved_model):
+
+ * The serving signatures of the SavedModel (if any) are represented as a
+ dictionary of concrete functions and can be called like
+ `tensors_out = obj.signatures["serving_default"](**tensors_in)`,
+ with dictionaries of tensors keyed by the respective input and output
+ names and subject to the signature's shape and dtype constraints.
+
+ * The
+ [`@tf.function`](https://www.tensorflow.org/api_docs/python/tf/function)-decorated
+ methods of the saved object (if any) are restored as tf.function objects
+ that can be called by all combinations of Tensor and non-Tensor arguments
+ for which the tf.function had been
+ [traced](https://www.tensorflow.org/tutorials/customization/performance#tracing)
+ prior to saving. In particular, if there is an `obj.__call__` method
+ with suitable traces, `obj` itself can be called like a Python function.
+ A simple example could look like
+ `output_tensor = obj(input_tensor, training=False)`.
+
+This leaves enormous liberty in the interfaces that SavedModels can
+implement. The [Reusable SavedModels interface](reusable_saved_models.md)
+for `obj` establishes conventions such that client code, including adapters
+like `hub.KerasLayer`, know how to use the SavedModel.
+
+Some SavedModels may not follow that convention, especially whole models
+not meant to be reused in larger models, and just provide serving signatures.
+
+The trainable variables in a SavedModel are reloaded as trainable,
+and `tf.GradientTape` will watch them by default. See the section on
+fine-tuning below for some caveats, and consider avoiding this for starters.
+Even if you want to fine-tune, you may want to see if `obj.trainable_variables`
+advises to re-train only a subset of the originally trainable variables.
+
+
+## Creating SavedModels for TF Hub
+
+### Overview
+
+SavedModel is TensorFlow's standard serialization format for trained models
+or model pieces.
+It stores the model's trained weights together with the exact TensorFlow
+operations to perform its computation. It can be used independently from
+the code that created it. In particular, it can be reused across different
+high-level model-building APIs like Keras, because TensorFlow operations
+are their common basic language.
+
+### Saving from Keras
+
+Starting with TensorFlow 2, `tf.keras.Model.save()` and
+`tf.keras.models.save_model()` default to the SavedModel format (not HDF5).
+The resulting SavedModels that can be used with `hub.load()`,
+`hub.KerasLayer` and similar adapters for other high-level APIs
+as they become available.
+
+To share a complete Keras Model, just save it with `include_optimizer=False`.
+
+To share a piece of a Keras Model, make the piece a Model in itself and then
+save that. You can either lay out the code like that from the start....
+
+```python
+piece_to_share = tf.keras.Model(...)
+full_model = tf.keras.Sequential([piece_to_share, ...])
+full_model.fit(...)
+piece_to_share.save(...)
+```
+
+...or cut out the piece to share after the fact (if it aligns with the
+layering of your full model):
+
+```python
+full_model = tf.keras.Model(...)
+sharing_input = full_model.get_layer(...).get_output_at(0)
+sharing_output = full_model.get_layer(...).get_output_at(0)
+piece_to_share = tf.keras.Model(sharing_input, sharing_output)
+piece_to_share.save(..., include_optimizer=False)
+```
+
+[TensorFlow Models](https://github.com/tensorflow/models) on GitHub uses the
+former approach for BERT (see
+[nlp/tools/export_tfhub_lib.py](https://github.com/tensorflow/models/blob/master/official/nlp/tools/export_tfhub_lib.py),
+note the split between `core_model` for export and the `pretrainer` for
+restoring the checkpoint) and the latter approach for ResNet (see
+[legacy/image_classification/tfhub_export.py](https://github.com/tensorflow/models/blob/master/official/legacy/image_classification/resnet/tfhub_export.py)).
+
+### Saving from low-level TensorFlow
+
+This requires good familiarity with TensorFlow's [SavedModel
+Guide](https://www.tensorflow.org/guide/saved_model).
+
+If you want to provide more than just a serving signature, you should
+implement the [Reusable SavedModel interface](reusable_saved_models.md).
+Conceptually, this looks like
+
+```python
+class MyMulModel(tf.train.Checkpoint):
+ def __init__(self, v_init):
+ super().__init__()
+ self.v = tf.Variable(v_init)
+ self.variables = [self.v]
+ self.trainable_variables = [self.v]
+ self.regularization_losses = [
+ tf.function(input_signature=[])(lambda: 0.001 * self.v**2),
+ ]
+
+ @tf.function(input_signature=[tf.TensorSpec(shape=None, dtype=tf.float32)])
+ def __call__(self, inputs):
+ return tf.multiply(inputs, self.v)
+
+tf.saved_model.save(MyMulModel(2.0), "/tmp/my_mul")
+
+layer = hub.KerasLayer("/tmp/my_mul")
+print(layer([10., 20.])) # [20., 40.]
+layer.trainable = True
+print(layer.trainable_weights) # [2.]
+print(layer.losses) # 0.004
+```
+
+
+## Fine-Tuning
+
+Training the already-trained variables of an imported SavedModel together with
+those of the model around it is called *fine-tuning* the SavedModel.
+This can result in better quality, but often makes the training more
+demanding (may take more time, depend more on the optimizer and its
+hyperparameters, increase the risk of overfitting and require dataset
+augmentation, esp. for CNNs). We advise SavedModel consumers to look into
+fine-tuning only after having established a good training regime,
+and only if the SavedModel publisher recommends it.
+
+Fine-tuning changes the "continuous" model parameters that are trained.
+It does not change hard-coded transformations, such as tokenizing text
+input and mapping tokens to their corresponding entries in an embedding matrix.
+
+### For SavedModel consumers
+
+Creating a `hub.KerasLayer` like
+
+```python
+layer = hub.KerasLayer(..., trainable=True)
+```
+
+enables fine-tuning of the SavedModel loaded by the layer. It adds the
+trainable weights and weight regularizers declared in the SavedModel
+to the Keras model, and runs the SavedModel's computation in training
+mode (think of dropout etc.).
+
+The [image classification
+colab](https://github.com/tensorflow/docs/blob/master/site/en/hub/tutorials/tf2_image_retraining.ipynb)
+contains an end-to-end example with optional fine-tuning.
+
+#### Re-exporting the fine-tuning result
+
+Advanced users may want to save the results of fine-tuning back into
+a SavedModel that can be used instead of the originally loaded one.
+This can be done with code like
+
+```python
+loaded_obj = hub.load("https://tfhub.dev/...")
+hub_layer = hub.KerasLayer(loaded_obj, trainable=True, ...)
+
+model = keras.Sequential([..., hub_layer, ...])
+model.compile(...)
+model.fit(...)
+
+export_module_dir = os.path.join(os.getcwd(), "finetuned_model_export")
+tf.saved_model.save(loaded_obj, export_module_dir)
+```
+
+### For SavedModel creators
+
+When creating a SavedModel for sharing on TensorFlow Hub,
+think ahead if and how its consumers should fine-tune it,
+and provide guidance in the documentation.
+
+Saving from a Keras Model should make all the mechanics of fine-tuning work
+(saving weight regularization losses, declaring trainable variables, tracing
+`__call__` for both `training=True` and `training=False`, etc.)
+
+Choose a model interface that plays well with gradient flow,
+e.g., output logits instead of softmax probabilities or top-k predictions.
+
+If the model use dropout, batch normalization, or similar training techniques
+that involve hyperparameters, set them to values that make sense across many
+expected target problems and batch sizes. (As of this writing, saving from
+Keras does not make it easy to let consumers adjust them.)
+
+Weight regularizers on individual layers are saved (with their regularization
+strength coefficients), but weight regularization from within the optimizer
+(like `tf.keras.optimizers.Ftrl.l1_regularization_strength=...)`)
+is lost. Advise consumers of your SavedModel accordingly.
diff --git a/site/en/hub/tutorials/_index.yaml b/site/en/hub/tutorials/_index.yaml
new file mode 100644
index 00000000000..deb98108393
--- /dev/null
+++ b/site/en/hub/tutorials/_index.yaml
@@ -0,0 +1,174 @@
+book_path: /hub/_book.yaml
+project_path: /hub/_project.yaml
+title: Tutorials
+landing_page:
+ custom_css_path: /site-assets/css/style.css
+ nav: left
+ meta_tags:
+ - name: description
+ content: >
+ TensorFlow Hub tutorials to help you get started with using and adapting pre-trained
+ machine learning models to your needs.
+ rows:
+ # Getting started
+ - classname: devsite-landing-row-100
+ items:
+ - description: >
+
Getting started
+
TensorFlow Hub is a comprehensive repository of pre-trained
+ models ready for fine-tuning and deployable anywhere. Download the latest trained models
+ with a minimal amount of code with the tensorflow_hub library.
+
The following tutorials should help you getting
+ started with using and applying models from TF Hub for your needs. Interactive tutorials let you
+ modify them and execute them with your changes. Click the Run in Google Colab
+ button at the top of an interactive tutorial to tinker with it.
If you are unfamiliar with machine learning and TensorFlow, you can start by getting
+ an overview of how to classify images and text, detecting objects in images, or by stylizing your own pictures like famous artwork:
+ Build a Keras model on top of a pre-trained image classifier to distinguish flowers.
+ path: /hub/tutorials/tf2_image_retraining
+ image_path: /hub/images/image_classification.png
+ - classname: tfo-landing-page-card
+ description: >
+
+ Use BERT to build a Keras model to solve a text classificaton sentiment analysis task.
+ path: /tutorials/text/classify_text_with_bert
+ image_path: /hub/images/bert_preprocess.png
+ - classname: tfo-landing-page-card
+ description: >
+
+
+
+ Let a neural network redraw an image in the style of Picasso, van Gogh or like your own style image.
+ path: /hub/tutorials/tf2_arbitrary_image_stylization
+ image_path: /hub/images/style_transfer.png
+ - classname: tfo-landing-page-card
+ description: >
+
+ Record yourself singing and detect the pitch of your voice using the SPICE model.
+ path: /hub/tutorials/spice
+ image_path: /hub/images/spice_color.png
+ - classname: tfo-landing-page-card
+ description: >
+
+ Use the YAMNet model to classify sounds as 521 audio event classes from the AudioSet-YouTube corpus.
+ path: /hub/tutorials/yamnet
+ image_path: /hub/images/yamnet.png
+
+ # Video tutorials
+ - classname: devsite-landing-row-100
+ items:
+ - description: >
+
Video tutorials
+
Try out trained ML models for video data for action recognition, video interpolation, and more.
+ Detect one of 400 actions in a video using the Inflated 3D ConvNet model.
+ path: /hub/tutorials/action_recognition_with_tf_hub
+ image_path: /hub/images/action_recognition.gif
+ - classname: tfo-landing-page-card
+ description: >
+
+ Interpolate between video frames using Inbetweening with 3D Convolutions.
+ path: /hub/tutorials/tweening_conv3d
+ image_path: /hub/images/interpolation.png
+ - classname: tfo-landing-page-card
+ description: >
+
"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "GhN2WtIrBQ4y"
+ },
+ "source": [
+ "Caution: In addition to installing Python packages with pip, this notebook uses\n",
+ "`sudo apt install` to install system packages: `unzip`.\n",
+ "\n",
+ "This Colab is a demonstration of using [Tensorflow Hub](https://www.tensorflow.org/hub/) for text classification in non-English/local languages. Here we choose [Bangla](https://en.wikipedia.org/wiki/Bengali_language) as the local language and use pretrained word embeddings to solve a multiclass classification task where we classify Bangla news articles in 5 categories. The pretrained embeddings for Bangla comes from [fastText](https://fasttext.cc/docs/en/crawl-vectors.html) which is a library by Facebook with released pretrained word vectors for 157 languages. \n",
+ "\n",
+ "We'll use TF-Hub's pretrained embedding exporter for converting the word embeddings to a text embedding module first and then use the module to train a classifier with [tf.keras](https://www.tensorflow.org/api_docs/python/tf/keras), Tensorflow's high level user friendly API to build deep learning models. Even if we are using fastText embeddings here, it's possible to export any other embeddings pretrained from other tasks and quickly get results with Tensorflow hub. "
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "Q4DN769E2O_R"
+ },
+ "source": [
+ "## Setup"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "9Vt-StAAZguA"
+ },
+ "outputs": [],
+ "source": [
+ "%%bash\n",
+ "# https://github.com/pypa/setuptools/issues/1694#issuecomment-466010982\n",
+ "pip install gdown --no-use-pep517"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "WcBA19FlDPZO"
+ },
+ "outputs": [],
+ "source": [
+ "%%bash\n",
+ "sudo apt-get install -y unzip"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "zSeyZMq-BYsu"
+ },
+ "outputs": [],
+ "source": [
+ "import os\n",
+ "\n",
+ "import tensorflow as tf\n",
+ "import tensorflow_hub as hub\n",
+ "\n",
+ "import gdown\n",
+ "import numpy as np\n",
+ "from sklearn.metrics import classification_report\n",
+ "import matplotlib.pyplot as plt\n",
+ "import seaborn as sns"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "9FB7gLU4F54l"
+ },
+ "source": [
+ "# Dataset\n",
+ "\n",
+ "We will use [BARD](https://www.researchgate.net/publication/328214545_BARD_Bangla_Article_Classification_Using_a_New_Comprehensive_Dataset) (Bangla Article Dataset) which has around 376,226 articles collected from different Bangla news portals and labelled with 5 categories: economy, state, international, sports, and entertainment. We download the file from Google Drive this ([bit.ly/BARD_DATASET](https://bit.ly/BARD_DATASET)) link is referring to from [this](https://github.com/tanvirfahim15/BARD-Bangla-Article-Classifier) GitHub repository.\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "zdQrL_rwa-1K"
+ },
+ "outputs": [],
+ "source": [
+ "gdown.download(\n",
+ " url='https://drive.google.com/uc?id=1Ag0jd21oRwJhVFIBohmX_ogeojVtapLy',\n",
+ " output='bard.zip',\n",
+ " quiet=True\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "P2YW4GGa9Y5o"
+ },
+ "outputs": [],
+ "source": [
+ "%%bash\n",
+ "unzip -qo bard.zip"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "js75OARBF_B8"
+ },
+ "source": [
+ "# Export pretrained word vectors to TF-Hub module"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "-uAicYA6vLsf"
+ },
+ "source": [
+ "TF-Hub provides some useful scripts for converting word embeddings to TF-hub text embedding modules [here](https://github.com/tensorflow/hub/tree/master/examples/text_embeddings_v2). To make the module for Bangla or any other languages, we simply have to download the word embedding `.txt` or `.vec` file to the same directory as `export_v2.py` and run the script.\n",
+ "\n",
+ "\n",
+ "The exporter reads the embedding vectors and exports it to a Tensorflow [SavedModel](https://www.tensorflow.org/beta/guide/saved_model). A SavedModel contains a complete TensorFlow program including weights and graph. TF-Hub can load the SavedModel as a [module](https://www.tensorflow.org/hub/api_docs/python/hub/Module), which we will use to build the model for text classification. Since we are using `tf.keras` to build the model, we will use [hub.KerasLayer](https://www.tensorflow.org/hub/api_docs/python/hub/KerasLayer), which provides a wrapper for a TF-Hub module to use as a Keras Layer.\n",
+ "\n",
+ "First we will get our word embeddings from fastText and embedding exporter from TF-Hub [repo](https://github.com/tensorflow/hub).\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "5DY5Ze6pO1G5"
+ },
+ "outputs": [],
+ "source": [
+ "%%bash\n",
+ "curl -O https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.bn.300.vec.gz\n",
+ "curl -O https://raw.githubusercontent.com/tensorflow/hub/master/examples/text_embeddings_v2/export_v2.py\n",
+ "gunzip -qf cc.bn.300.vec.gz --k"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "PAzdNZaHmdl1"
+ },
+ "source": [
+ "Then, we will run the exporter script on our embedding file. Since fastText embeddings have a header line and are pretty large (around 3.3 GB for Bangla after converting to a module) we ignore the first line and export only the first 100, 000 tokens to the text embedding module."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "Tkv5acr_Q9UU"
+ },
+ "outputs": [],
+ "source": [
+ "%%bash\n",
+ "python export_v2.py --embedding_file=cc.bn.300.vec --export_path=text_module --num_lines_to_ignore=1 --num_lines_to_use=100000"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "k9WEpmedF_3_"
+ },
+ "outputs": [],
+ "source": [
+ "module_path = \"text_module\"\n",
+ "embedding_layer = hub.KerasLayer(module_path, trainable=False)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "fQHbmS_D4YIo"
+ },
+ "source": [
+ "The text embedding module takes a batch of sentences in a 1D tensor of strings as input and outputs the embedding vectors of shape (batch_size, embedding_dim) corresponding to the sentences. It preprocesses the input by splitting on spaces. Word embeddings are combined to sentence embeddings with the `sqrtn` combiner(See [here](https://www.tensorflow.org/api_docs/python/tf/nn/embedding_lookup_sparse)). For demonstration we pass a list of Bangla words as input and get the corresponding embedding vectors."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "Z1MBnaBUihWn"
+ },
+ "outputs": [],
+ "source": [
+ "embedding_layer(['বাস', 'বসবাস', 'ট্রেন', 'যাত্রী', 'ট্রাক']) "
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "4KY8LiFOHmcd"
+ },
+ "source": [
+ "# Convert to Tensorflow Dataset \n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "pNguCDNe6bvz"
+ },
+ "source": [
+ "Since the dataset is really large instead of loading the entire dataset in memory we will use a generator to yield samples in run-time in batches using [Tensorflow Dataset](https://www.tensorflow.org/api_docs/python/tf/data/Dataset) functions. The dataset is also very imbalanced, so, before using the generator, we will shuffle the dataset. \n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "bYv6LqlEChO1"
+ },
+ "outputs": [],
+ "source": [
+ "dir_names = ['economy', 'sports', 'entertainment', 'state', 'international']\n",
+ "\n",
+ "file_paths = []\n",
+ "labels = []\n",
+ "for i, dir in enumerate(dir_names):\n",
+ " file_names = [\"/\".join([dir, name]) for name in os.listdir(dir)]\n",
+ " file_paths += file_names\n",
+ " labels += [i] * len(os.listdir(dir))\n",
+ " \n",
+ "np.random.seed(42)\n",
+ "permutation = np.random.permutation(len(file_paths))\n",
+ "\n",
+ "file_paths = np.array(file_paths)[permutation]\n",
+ "labels = np.array(labels)[permutation]"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "8b-UtAP5TL-W"
+ },
+ "source": [
+ "We can check the distribution of labels in the training and validation examples after shuffling."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "mimhWVSzzAmS"
+ },
+ "outputs": [],
+ "source": [
+ "train_frac = 0.8\n",
+ "train_size = int(len(file_paths) * train_frac)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "4BNXFrkotAYu"
+ },
+ "outputs": [],
+ "source": [
+ "# plot training vs validation distribution\n",
+ "plt.subplot(1, 2, 1)\n",
+ "plt.hist(labels[0:train_size])\n",
+ "plt.title(\"Train labels\")\n",
+ "plt.subplot(1, 2, 2)\n",
+ "plt.hist(labels[train_size:])\n",
+ "plt.title(\"Validation labels\")\n",
+ "plt.tight_layout()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "RVbHb2I3TUNA"
+ },
+ "source": [
+ "To create a [Dataset](https://www.tensorflow.org/api_docs/python/tf/data/Dataset) using a generator, we first write a generator function which reads each of the articles from `file_paths` and the labels from the label array, and yields one training example at each step. We pass this generator function to the [`tf.data.Dataset.from_generator`](https://www.tensorflow.org/api_docs/python/tf/data/Dataset#from_generator) method and specify the output types. Each training example is a tuple containing an article of `tf.string` data type and one-hot encoded label. We split the dataset with a train-validation split of 80-20 using [`tf.data.Dataset.skip`](https://www.tensorflow.org/api_docs/python/tf/data/Dataset#skip) and [`tf.data.Dataset.take`](https://www.tensorflow.org/api_docs/python/tf/data/Dataset#take) methods."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "eZRGTzEhUi7Q"
+ },
+ "outputs": [],
+ "source": [
+ "def load_file(path, label):\n",
+ " return tf.io.read_file(path), label"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "2g4nRflB7fbF"
+ },
+ "outputs": [],
+ "source": [
+ "def make_datasets(train_size):\n",
+ " batch_size = 256\n",
+ "\n",
+ " train_files = file_paths[:train_size]\n",
+ " train_labels = labels[:train_size]\n",
+ " train_ds = tf.data.Dataset.from_tensor_slices((train_files, train_labels))\n",
+ " train_ds = train_ds.map(load_file).shuffle(5000)\n",
+ " train_ds = train_ds.batch(batch_size).prefetch(tf.data.AUTOTUNE)\n",
+ "\n",
+ " test_files = file_paths[train_size:]\n",
+ " test_labels = labels[train_size:]\n",
+ " test_ds = tf.data.Dataset.from_tensor_slices((test_files, test_labels))\n",
+ " test_ds = test_ds.map(load_file)\n",
+ " test_ds = test_ds.batch(batch_size).prefetch(tf.data.AUTOTUNE)\n",
+ "\n",
+ "\n",
+ " return train_ds, test_ds"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "8PuuN6el8tv9"
+ },
+ "outputs": [],
+ "source": [
+ "train_data, validation_data = make_datasets(train_size)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "MrdZI6FqPJNP"
+ },
+ "source": [
+ "# Model Training and Evaluation"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "jgr7YScGVS58"
+ },
+ "source": [
+ "Since we have already added a wrapper around our module to use it as any other layer in Keras, we can create a small [Sequential](https://www.tensorflow.org/api_docs/python/tf/keras/Sequential) model which is a linear stack of layers. We can add our text embedding module with `model.add` just like any other layer. We compile the model by specifying the loss and optimizer and train it for 10 epochs. The `tf.keras` API can handle Tensorflow Datasets as input, so we can pass a Dataset instance to the fit method for model training. Since we are using the generator function, `tf.data` will handle generating the samples, batching them and feeding them to the model."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "WhCqbDK2uUV5"
+ },
+ "source": [
+ "## Model"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "nHUw807XPPM9"
+ },
+ "outputs": [],
+ "source": [
+ "def create_model():\n",
+ " model = tf.keras.Sequential([\n",
+ " tf.keras.layers.Input(shape=[], dtype=tf.string),\n",
+ " embedding_layer,\n",
+ " tf.keras.layers.Dense(64, activation=\"relu\"),\n",
+ " tf.keras.layers.Dense(16, activation=\"relu\"),\n",
+ " tf.keras.layers.Dense(5),\n",
+ " ])\n",
+ " model.compile(loss=tf.losses.SparseCategoricalCrossentropy(from_logits=True),\n",
+ " optimizer=\"adam\", metrics=['accuracy'])\n",
+ " return model"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "5J4EXJUmPVNG"
+ },
+ "outputs": [],
+ "source": [
+ "model = create_model()\n",
+ "# Create earlystopping callback\n",
+ "early_stopping_callback = tf.keras.callbacks.EarlyStopping(monitor='val_loss', min_delta=0, patience=3)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "ZZ7XJLg2u2No"
+ },
+ "source": [
+ "## Training"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "OoBkN2tAaXWD"
+ },
+ "outputs": [],
+ "source": [
+ "history = model.fit(train_data, \n",
+ " validation_data=validation_data, \n",
+ " epochs=5, \n",
+ " callbacks=[early_stopping_callback])"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "XoDk8otmMoT7"
+ },
+ "source": [
+ "## Evaluation"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "G5ZRKGOsXEh4"
+ },
+ "source": [
+ "We can visualize the accuracy and loss curves for training and validation data using the `tf.keras.callbacks.History` object returned by the `tf.keras.Model.fit` method, which contains the loss and accuracy value for each epoch."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "V6tOnByIOeGn"
+ },
+ "outputs": [],
+ "source": [
+ "# Plot training & validation accuracy values\n",
+ "plt.plot(history.history['accuracy'])\n",
+ "plt.plot(history.history['val_accuracy'])\n",
+ "plt.title('Model accuracy')\n",
+ "plt.ylabel('Accuracy')\n",
+ "plt.xlabel('Epoch')\n",
+ "plt.legend(['Train', 'Test'], loc='upper left')\n",
+ "plt.show()\n",
+ "\n",
+ "# Plot training & validation loss values\n",
+ "plt.plot(history.history['loss'])\n",
+ "plt.plot(history.history['val_loss'])\n",
+ "plt.title('Model loss')\n",
+ "plt.ylabel('Loss')\n",
+ "plt.xlabel('Epoch')\n",
+ "plt.legend(['Train', 'Test'], loc='upper left')\n",
+ "plt.show()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "D54IXLqcG8Cq"
+ },
+ "source": [
+ "## Prediction\n",
+ "\n",
+ "We can get the predictions for the validation data and check the confusion matrix to see the model's performance for each of the 5 classes. Because `tf.keras.Model.predict` method returns an n-d array for probabilities for each class, they can be converted to class labels using `np.argmax`."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "dptEywzZJk4l"
+ },
+ "outputs": [],
+ "source": [
+ "y_pred = model.predict(validation_data)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "7Dzeml6Pk0ub"
+ },
+ "outputs": [],
+ "source": [
+ "y_pred = np.argmax(y_pred, axis=1)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "T4M3Lzg8jHcB"
+ },
+ "outputs": [],
+ "source": [
+ "samples = file_paths[0:3]\n",
+ "for i, sample in enumerate(samples):\n",
+ " f = open(sample)\n",
+ " text = f.read()\n",
+ " print(text[0:100])\n",
+ " print(\"True Class: \", sample.split(\"/\")[0])\n",
+ " print(\"Predicted Class: \", dir_names[y_pred[i]])\n",
+ " f.close()\n",
+ " "
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "PlDTIpMBu6h-"
+ },
+ "source": [
+ "## Compare Performance\n",
+ "\n",
+ "Now we can take the correct labels for the validation data from `labels` and compare them with our predictions to get a [classification_report](http://scikit-learn.org/stable/modules/generated/sklearn.metrics.classification_report.html). "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "mqrERUCS1Xn7"
+ },
+ "outputs": [],
+ "source": [
+ "y_true = np.array(labels[train_size:])"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "NX5w-NuTKuVP"
+ },
+ "outputs": [],
+ "source": [
+ "print(classification_report(y_true, y_pred, target_names=dir_names))"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "p5e9m3bV6oXK"
+ },
+ "source": [
+ "We can also compare our model's performance with the published results obtained in the original [paper](https://www.researchgate.net/publication/328214545_BARD_Bangla_Article_Classification_Using_a_New_Comprehensive_Dataset), which had a 0.96 precision .The original authors described many preprocessing steps performed on the dataset, such as dropping punctuations and digits, removing top 25 most frequest stop words. As we can see in the `classification_report`, we also manage to obtain a 0.96 precision and accuracy after training for only 5 epochs without any preprocessing! \n",
+ "\n",
+ "In this example, when we created the Keras layer from our embedding module, we set the parameter`trainable=False`, which means the embedding weights will not be updated during training. Try setting it to `True` to reach around 97% accuracy using this dataset after only 2 epochs. "
+ ]
+ }
+ ],
+ "metadata": {
+ "colab": {
+ "collapsed_sections": [
+ "IDdZSPcLtKx4"
+ ],
+ "name": "bangla_article_classifier.ipynb",
+ "toc_visible": true
+ },
+ "kernelspec": {
+ "display_name": "Python 3",
+ "name": "python3"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}
diff --git a/site/en/hub/tutorials/bert_experts.ipynb b/site/en/hub/tutorials/bert_experts.ipynb
new file mode 100644
index 00000000000..5440909f7cb
--- /dev/null
+++ b/site/en/hub/tutorials/bert_experts.ipynb
@@ -0,0 +1,286 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "-1vOMEXIhMQt"
+ },
+ "source": [
+ "##### Copyright 2020 The TensorFlow Hub Authors.\n",
+ "\n",
+ "Licensed under the Apache License, Version 2.0 (the \"License\");"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "cellView": "form",
+ "id": "pRfq9ZU5hQhg"
+ },
+ "outputs": [],
+ "source": [
+ "#@title Copyright 2020 The TensorFlow Hub Authors. All Rights Reserved.\n",
+ "#\n",
+ "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+ "# you may not use this file except in compliance with the License.\n",
+ "# You may obtain a copy of the License at\n",
+ "#\n",
+ "# http://www.apache.org/licenses/LICENSE-2.0\n",
+ "#\n",
+ "# Unless required by applicable law or agreed to in writing, software\n",
+ "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+ "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+ "# See the License for the specific language governing permissions and\n",
+ "# limitations under the License.\n",
+ "# =============================================================================="
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "mTL0TERThT6z"
+ },
+ "source": [
+ "
"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "FkthMlVk8bHp"
+ },
+ "source": [
+ "# BERT Experts from TF-Hub\n",
+ "\n",
+ "This colab demonstrates how to:\n",
+ "* Load BERT models from [TensorFlow Hub](https://tfhub.dev) that have been trained on different tasks including MNLI, SQuAD, and PubMed\n",
+ "* Use a matching preprocessing model to tokenize raw text and convert it to ids\n",
+ "* Generate the pooled and sequence output from the token input ids using the loaded model\n",
+ "* Look at the semantic similarity of the pooled outputs of different sentences\n",
+ "\n",
+ "#### Note: This colab should be run with a GPU runtime"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "jspO02jDPfPG"
+ },
+ "source": [
+ "## Set up and imports"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "r-ed8zj-dbwm"
+ },
+ "outputs": [],
+ "source": [
+ "!pip install --quiet \"tensorflow-text==2.11.*\""
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "czDmtrGKYw_5"
+ },
+ "outputs": [],
+ "source": [
+ "import seaborn as sns\n",
+ "from sklearn.metrics import pairwise\n",
+ "\n",
+ "import tensorflow as tf\n",
+ "import tensorflow_hub as hub\n",
+ "import tensorflow_text as text # Imports TF ops for preprocessing."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "cellView": "form",
+ "id": "GSuDcPSaY5aB"
+ },
+ "outputs": [],
+ "source": [
+ "#@title Configure the model { run: \"auto\" }\n",
+ "BERT_MODEL = \"https://tfhub.dev/google/experts/bert/wiki_books/2\" # @param {type: \"string\"} [\"https://tfhub.dev/google/experts/bert/wiki_books/2\", \"https://tfhub.dev/google/experts/bert/wiki_books/mnli/2\", \"https://tfhub.dev/google/experts/bert/wiki_books/qnli/2\", \"https://tfhub.dev/google/experts/bert/wiki_books/qqp/2\", \"https://tfhub.dev/google/experts/bert/wiki_books/squad2/2\", \"https://tfhub.dev/google/experts/bert/wiki_books/sst2/2\", \"https://tfhub.dev/google/experts/bert/pubmed/2\", \"https://tfhub.dev/google/experts/bert/pubmed/squad2/2\"]\n",
+ "# Preprocessing must match the model, but all the above use the same.\n",
+ "PREPROCESS_MODEL = \"https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3\""
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "pvaZiGVgwtqw"
+ },
+ "source": [
+ "## Sentences\n",
+ "\n",
+ "Let's take some sentences from Wikipedia to run through the model"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "tytu-rSpeDNG"
+ },
+ "outputs": [],
+ "source": [
+ "sentences = [\n",
+ " \"Here We Go Then, You And I is a 1999 album by Norwegian pop artist Morten Abel. It was Abel's second CD as a solo artist.\",\n",
+ " \"The album went straight to number one on the Norwegian album chart, and sold to double platinum.\",\n",
+ " \"Among the singles released from the album were the songs \\\"Be My Lover\\\" and \\\"Hard To Stay Awake\\\".\",\n",
+ " \"Riccardo Zegna is an Italian jazz musician.\",\n",
+ " \"Rajko Maksimović is a composer, writer, and music pedagogue.\",\n",
+ " \"One of the most significant Serbian composers of our time, Maksimović has been and remains active in creating works for different ensembles.\",\n",
+ " \"Ceylon spinach is a common name for several plants and may refer to: Basella alba Talinum fruticosum\",\n",
+ " \"A solar eclipse occurs when the Moon passes between Earth and the Sun, thereby totally or partly obscuring the image of the Sun for a viewer on Earth.\",\n",
+ " \"A partial solar eclipse occurs in the polar regions of the Earth when the center of the Moon's shadow misses the Earth.\",\n",
+ "]"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "zI39475kxCKh"
+ },
+ "source": [
+ "## Run the model\n",
+ "\n",
+ "We'll load the BERT model from TF-Hub, tokenize our sentences using the matching preprocessing model from TF-Hub, then feed in the tokenized sentences to the model. To keep this colab fast and simple, we recommend running on GPU.\n",
+ "\n",
+ "Go to **Runtime** → **Change runtime type** to make sure that **GPU** is selected"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "x4t6r22ErQg0"
+ },
+ "outputs": [],
+ "source": [
+ "preprocess = hub.load(PREPROCESS_MODEL)\n",
+ "bert = hub.load(BERT_MODEL)\n",
+ "inputs = preprocess(sentences)\n",
+ "outputs = bert(inputs)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "gItjCg4315Cv"
+ },
+ "outputs": [],
+ "source": [
+ "print(\"Sentences:\")\n",
+ "print(sentences)\n",
+ "\n",
+ "print(\"\\nBERT inputs:\")\n",
+ "print(inputs)\n",
+ "\n",
+ "print(\"\\nPooled embeddings:\")\n",
+ "print(outputs[\"pooled_output\"])\n",
+ "\n",
+ "print(\"\\nPer token embeddings:\")\n",
+ "print(outputs[\"sequence_output\"])"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "ptiW2mgw6x-l"
+ },
+ "source": [
+ "## Semantic similarity\n",
+ "\n",
+ "Now let's take a look at the `pooled_output` embeddings of our sentences and compare how similar they are across sentences."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "cellView": "form",
+ "id": "GXrSO2Vc1Qtr"
+ },
+ "outputs": [],
+ "source": [
+ "#@title Helper functions\n",
+ "\n",
+ "def plot_similarity(features, labels):\n",
+ " \"\"\"Plot a similarity matrix of the embeddings.\"\"\"\n",
+ " cos_sim = pairwise.cosine_similarity(features)\n",
+ " sns.set(font_scale=1.2)\n",
+ " cbar_kws=dict(use_gridspec=False, location=\"left\")\n",
+ " g = sns.heatmap(\n",
+ " cos_sim, xticklabels=labels, yticklabels=labels,\n",
+ " vmin=0, vmax=1, cmap=\"Blues\", cbar_kws=cbar_kws)\n",
+ " g.tick_params(labelright=True, labelleft=False)\n",
+ " g.set_yticklabels(labels, rotation=0)\n",
+ " g.set_title(\"Semantic Textual Similarity\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "td6jcT0pJMZ5"
+ },
+ "outputs": [],
+ "source": [
+ "plot_similarity(outputs[\"pooled_output\"], sentences)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "tJ4QCyzhSL7B"
+ },
+ "source": [
+ "## Learn more\n",
+ "\n",
+ "* Find more BERT models on [TensorFlow Hub](https://tfhub.dev)\n",
+ "* This notebook demonstrates simple inference with BERT, you can find a more advanced tutorial about fine-tuning BERT at [tensorflow.org/official_models/fine_tuning_bert](https://www.tensorflow.org/official_models/fine_tuning_bert)\n",
+ "* We used just one GPU chip to run the model, you can learn more about how to load models using tf.distribute at [tensorflow.org/tutorials/distribute/save_and_load](https://www.tensorflow.org/tutorials/distribute/save_and_load)"
+ ]
+ }
+ ],
+ "metadata": {
+ "accelerator": "GPU",
+ "colab": {
+ "collapsed_sections": [],
+ "name": "bert_experts.ipynb",
+ "toc_visible": true
+ },
+ "kernelspec": {
+ "display_name": "Python 3",
+ "name": "python3"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}
diff --git a/site/en/hub/tutorials/bigbigan_with_tf_hub.ipynb b/site/en/hub/tutorials/bigbigan_with_tf_hub.ipynb
new file mode 100644
index 00000000000..919abc7e354
--- /dev/null
+++ b/site/en/hub/tutorials/bigbigan_with_tf_hub.ipynb
@@ -0,0 +1,713 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "pLOYL1PJAAtK"
+ },
+ "source": [
+ "##### Copyright 2019 The TensorFlow Hub Authors.\n",
+ "\n",
+ "Licensed under the Apache License, Version 2.0 (the \"License\");"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "3fJWQ8WSAFhh"
+ },
+ "outputs": [],
+ "source": [
+ "# Copyright 2019 The TensorFlow Hub Authors. All Rights Reserved.\n",
+ "#\n",
+ "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+ "# you may not use this file except in compliance with the License.\n",
+ "# You may obtain a copy of the License at\n",
+ "#\n",
+ "# http://www.apache.org/licenses/LICENSE-2.0\n",
+ "#\n",
+ "# Unless required by applicable law or agreed to in writing, software\n",
+ "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+ "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+ "# See the License for the specific language governing permissions and\n",
+ "# limitations under the License.\n",
+ "# =============================================================================="
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "-1NTVIH6ABK-"
+ },
+ "source": [
+ "# Generating Images with BigBiGAN\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "MfBg1C5NB3X0"
+ },
+ "source": [
+ "
"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "eAVQGidpL8v5"
+ },
+ "source": [
+ "This notebook illustrates how to access the Multilingual Universal Sentence Encoder module and use it for sentence similarity across multiple languages. This module is an extension of the [original Universal Encoder module](https://tfhub.dev/google/universal-sentence-encoder/2).\n",
+ "\n",
+ "The notebook is divided as follows:\n",
+ "\n",
+ "* The first section shows a visualization of sentences between pair of languages. This is a more academic exercise. \n",
+ "* In the second section, we show how to build a semantic search engine from a sample of a Wikipedia corpus in multiple languages."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "UvNRbHGarYeR"
+ },
+ "source": [
+ "## Citation\n",
+ "\n",
+ "*Research papers that make use of the models explored in this colab should cite:*\n",
+ "\n",
+ "### [Multilingual universal sentence encoder for semantic retrieval](https://arxiv.org/abs/1907.04307)\n",
+ "Yinfei Yang, Daniel Cer, Amin Ahmad, Mandy Guo, Jax Law, Noah Constant, Gustavo Hernandez Abrego, Steve Yuan, Chris Tar, Yun-Hsuan Sung, Brian Strope, and Ray Kurzweil. 2019.\n",
+ " arXiv preprint arXiv:1907.04307"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "pOTzp8O36CyQ"
+ },
+ "source": [
+ "## Setup\n",
+ "\n",
+ "This section sets up the environment for access to the Multilingual Universal Sentence Encoder Module and also prepares a set of English sentences and their translations. In the following sections, the multilingual module will be used to compute similarity *across languages*."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "cellView": "both",
+ "id": "lVjNK8shFKOC"
+ },
+ "outputs": [],
+ "source": [
+ "%%capture\n",
+ "#@title Setup Environment\n",
+ "# Install the latest Tensorflow version.\n",
+ "!pip install \"tensorflow-text==2.11.*\"\n",
+ "!pip install bokeh\n",
+ "!pip install simpleneighbors[annoy]\n",
+ "!pip install tqdm"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "cellView": "both",
+ "id": "MSeY-MUQo2Ha"
+ },
+ "outputs": [],
+ "source": [
+ "#@title Setup common imports and functions\n",
+ "import bokeh\n",
+ "import bokeh.models\n",
+ "import bokeh.plotting\n",
+ "import numpy as np\n",
+ "import os\n",
+ "import pandas as pd\n",
+ "import tensorflow.compat.v2 as tf\n",
+ "import tensorflow_hub as hub\n",
+ "from tensorflow_text import SentencepieceTokenizer\n",
+ "import sklearn.metrics.pairwise\n",
+ "\n",
+ "from simpleneighbors import SimpleNeighbors\n",
+ "from tqdm import tqdm\n",
+ "from tqdm import trange\n",
+ "\n",
+ "def visualize_similarity(embeddings_1, embeddings_2, labels_1, labels_2,\n",
+ " plot_title,\n",
+ " plot_width=1200, plot_height=600,\n",
+ " xaxis_font_size='12pt', yaxis_font_size='12pt'):\n",
+ "\n",
+ " assert len(embeddings_1) == len(labels_1)\n",
+ " assert len(embeddings_2) == len(labels_2)\n",
+ "\n",
+ " # arccos based text similarity (Yang et al. 2019; Cer et al. 2019)\n",
+ " sim = 1 - np.arccos(\n",
+ " sklearn.metrics.pairwise.cosine_similarity(embeddings_1,\n",
+ " embeddings_2))/np.pi\n",
+ "\n",
+ " embeddings_1_col, embeddings_2_col, sim_col = [], [], []\n",
+ " for i in range(len(embeddings_1)):\n",
+ " for j in range(len(embeddings_2)):\n",
+ " embeddings_1_col.append(labels_1[i])\n",
+ " embeddings_2_col.append(labels_2[j])\n",
+ " sim_col.append(sim[i][j])\n",
+ " df = pd.DataFrame(zip(embeddings_1_col, embeddings_2_col, sim_col),\n",
+ " columns=['embeddings_1', 'embeddings_2', 'sim'])\n",
+ "\n",
+ " mapper = bokeh.models.LinearColorMapper(\n",
+ " palette=[*reversed(bokeh.palettes.YlOrRd[9])], low=df.sim.min(),\n",
+ " high=df.sim.max())\n",
+ "\n",
+ " p = bokeh.plotting.figure(title=plot_title, x_range=labels_1,\n",
+ " x_axis_location=\"above\",\n",
+ " y_range=[*reversed(labels_2)],\n",
+ " plot_width=plot_width, plot_height=plot_height,\n",
+ " tools=\"save\",toolbar_location='below', tooltips=[\n",
+ " ('pair', '@embeddings_1 ||| @embeddings_2'),\n",
+ " ('sim', '@sim')])\n",
+ " p.rect(x=\"embeddings_1\", y=\"embeddings_2\", width=1, height=1, source=df,\n",
+ " fill_color={'field': 'sim', 'transform': mapper}, line_color=None)\n",
+ "\n",
+ " p.title.text_font_size = '12pt'\n",
+ " p.axis.axis_line_color = None\n",
+ " p.axis.major_tick_line_color = None\n",
+ " p.axis.major_label_standoff = 16\n",
+ " p.xaxis.major_label_text_font_size = xaxis_font_size\n",
+ " p.xaxis.major_label_orientation = 0.25 * np.pi\n",
+ " p.yaxis.major_label_text_font_size = yaxis_font_size\n",
+ " p.min_border_right = 300\n",
+ "\n",
+ " bokeh.io.output_notebook()\n",
+ " bokeh.io.show(p)\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "gk2IRjZFGDsK"
+ },
+ "source": [
+ "This is additional boilerplate code where we import the pre-trained ML model we will use to encode text throughout this notebook."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "mkmF3w8WGLcM"
+ },
+ "outputs": [],
+ "source": [
+ "# The 16-language multilingual module is the default but feel free\n",
+ "# to pick others from the list and compare the results.\n",
+ "module_url = 'https://tfhub.dev/google/universal-sentence-encoder-multilingual/3' #@param ['https://tfhub.dev/google/universal-sentence-encoder-multilingual/3', 'https://tfhub.dev/google/universal-sentence-encoder-multilingual-large/3']\n",
+ "\n",
+ "model = hub.load(module_url)\n",
+ "\n",
+ "def embed_text(input):\n",
+ " return model(input)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "jhLPq6AROyFk"
+ },
+ "source": [
+ "# Visualize Text Similarity Between Languages\n",
+ "With the sentence embeddings now in hand, we can visualize semantic similarity across different languages."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "8xdAogbxJDTD"
+ },
+ "source": [
+ "## Computing Text Embeddings\n",
+ "\n",
+ "We first define a set of sentences translated to various languages in parallel. Then, we precompute the embeddings for all of our sentences."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "Q8F4LNGFqOiq"
+ },
+ "outputs": [],
+ "source": [
+ "# Some texts of different lengths in different languages.\n",
+ "arabic_sentences = ['كلب', 'الجراء لطيفة.', 'أستمتع بالمشي لمسافات طويلة على طول الشاطئ مع كلبي.']\n",
+ "chinese_sentences = ['狗', '小狗很好。', '我喜欢和我的狗一起沿着海滩散步。']\n",
+ "english_sentences = ['dog', 'Puppies are nice.', 'I enjoy taking long walks along the beach with my dog.']\n",
+ "french_sentences = ['chien', 'Les chiots sont gentils.', 'J\\'aime faire de longues promenades sur la plage avec mon chien.']\n",
+ "german_sentences = ['Hund', 'Welpen sind nett.', 'Ich genieße lange Spaziergänge am Strand entlang mit meinem Hund.']\n",
+ "italian_sentences = ['cane', 'I cuccioli sono carini.', 'Mi piace fare lunghe passeggiate lungo la spiaggia con il mio cane.']\n",
+ "japanese_sentences = ['犬', '子犬はいいです', '私は犬と一緒にビーチを散歩するのが好きです']\n",
+ "korean_sentences = ['개', '강아지가 좋다.', '나는 나의 개와 해변을 따라 길게 산책하는 것을 즐긴다.']\n",
+ "russian_sentences = ['собака', 'Милые щенки.', 'Мне нравится подолгу гулять по пляжу со своей собакой.']\n",
+ "spanish_sentences = ['perro', 'Los cachorros son agradables.', 'Disfruto de dar largos paseos por la playa con mi perro.']\n",
+ "\n",
+ "# Multilingual example\n",
+ "multilingual_example = [\"Willkommen zu einfachen, aber\", \"verrassend krachtige\", \"multilingüe\", \"compréhension du language naturel\", \"модели.\", \"大家是什么意思\" , \"보다 중요한\", \".اللغة التي يتحدثونها\"]\n",
+ "multilingual_example_in_en = [\"Welcome to simple yet\", \"surprisingly powerful\", \"multilingual\", \"natural language understanding\", \"models.\", \"What people mean\", \"matters more than\", \"the language they speak.\"]\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "weXZqLtTJY9b"
+ },
+ "outputs": [],
+ "source": [
+ "# Compute embeddings.\n",
+ "ar_result = embed_text(arabic_sentences)\n",
+ "en_result = embed_text(english_sentences)\n",
+ "es_result = embed_text(spanish_sentences)\n",
+ "de_result = embed_text(german_sentences)\n",
+ "fr_result = embed_text(french_sentences)\n",
+ "it_result = embed_text(italian_sentences)\n",
+ "ja_result = embed_text(japanese_sentences)\n",
+ "ko_result = embed_text(korean_sentences)\n",
+ "ru_result = embed_text(russian_sentences)\n",
+ "zh_result = embed_text(chinese_sentences)\n",
+ "\n",
+ "multilingual_result = embed_text(multilingual_example)\n",
+ "multilingual_in_en_result = embed_text(multilingual_example_in_en)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "_3zGWuF-GhUm"
+ },
+ "source": [
+ "## Visualizing Similarity\n",
+ "\n",
+ "With text embeddings in hand, we can take their dot-product to visualize how similar sentences are between languages. A darker color indicates the embeddings are semantically similar."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "WOEIJA0mh70g"
+ },
+ "source": [
+ "### Multilingual Similarity"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "R2hbCMhmiDWR"
+ },
+ "outputs": [
+ {
+ "data": {
+ "application/javascript": [
+ "\n",
+ "(function(root) {\n",
+ " function now() {\n",
+ " return new Date();\n",
+ " }\n",
+ "\n",
+ " var force = true;\n",
+ "\n",
+ " if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n",
+ " root._bokeh_onload_callbacks = [];\n",
+ " root._bokeh_is_loading = undefined;\n",
+ " }\n",
+ "\n",
+ " var JS_MIME_TYPE = 'application/javascript';\n",
+ " var HTML_MIME_TYPE = 'text/html';\n",
+ " var EXEC_MIME_TYPE = 'application/vnd.bokehjs_exec.v0+json';\n",
+ " var CLASS_NAME = 'output_bokeh rendered_html';\n",
+ "\n",
+ " /**\n",
+ " * Render data to the DOM node\n",
+ " */\n",
+ " function render(props, node) {\n",
+ " var script = document.createElement(\"script\");\n",
+ " node.appendChild(script);\n",
+ " }\n",
+ "\n",
+ " /**\n",
+ " * Handle when an output is cleared or removed\n",
+ " */\n",
+ " function handleClearOutput(event, handle) {\n",
+ " var cell = handle.cell;\n",
+ "\n",
+ " var id = cell.output_area._bokeh_element_id;\n",
+ " var server_id = cell.output_area._bokeh_server_id;\n",
+ " // Clean up Bokeh references\n",
+ " if (id != null && id in Bokeh.index) {\n",
+ " Bokeh.index[id].model.document.clear();\n",
+ " delete Bokeh.index[id];\n",
+ " }\n",
+ "\n",
+ " if (server_id !== undefined) {\n",
+ " // Clean up Bokeh references\n",
+ " var cmd = \"from bokeh.io.state import curstate; print(curstate().uuid_to_server['\" + server_id + \"'].get_sessions()[0].document.roots[0]._id)\";\n",
+ " cell.notebook.kernel.execute(cmd, {\n",
+ " iopub: {\n",
+ " output: function(msg) {\n",
+ " var id = msg.content.text.trim();\n",
+ " if (id in Bokeh.index) {\n",
+ " Bokeh.index[id].model.document.clear();\n",
+ " delete Bokeh.index[id];\n",
+ " }\n",
+ " }\n",
+ " }\n",
+ " });\n",
+ " // Destroy server and session\n",
+ " var cmd = \"import bokeh.io.notebook as ion; ion.destroy_server('\" + server_id + \"')\";\n",
+ " cell.notebook.kernel.execute(cmd);\n",
+ " }\n",
+ " }\n",
+ "\n",
+ " /**\n",
+ " * Handle when a new output is added\n",
+ " */\n",
+ " function handleAddOutput(event, handle) {\n",
+ " var output_area = handle.output_area;\n",
+ " var output = handle.output;\n",
+ "\n",
+ " // limit handleAddOutput to display_data with EXEC_MIME_TYPE content only\n",
+ " if ((output.output_type != \"display_data\") || (!output.data.hasOwnProperty(EXEC_MIME_TYPE))) {\n",
+ " return\n",
+ " }\n",
+ "\n",
+ " var toinsert = output_area.element.find(\".\" + CLASS_NAME.split(' ')[0]);\n",
+ "\n",
+ " if (output.metadata[EXEC_MIME_TYPE][\"id\"] !== undefined) {\n",
+ " toinsert[toinsert.length - 1].firstChild.textContent = output.data[JS_MIME_TYPE];\n",
+ " // store reference to embed id on output_area\n",
+ " output_area._bokeh_element_id = output.metadata[EXEC_MIME_TYPE][\"id\"];\n",
+ " }\n",
+ " if (output.metadata[EXEC_MIME_TYPE][\"server_id\"] !== undefined) {\n",
+ " var bk_div = document.createElement(\"div\");\n",
+ " bk_div.innerHTML = output.data[HTML_MIME_TYPE];\n",
+ " var script_attrs = bk_div.children[0].attributes;\n",
+ " for (var i = 0; i < script_attrs.length; i++) {\n",
+ " toinsert[toinsert.length - 1].firstChild.setAttribute(script_attrs[i].name, script_attrs[i].value);\n",
+ " }\n",
+ " // store reference to server id on output_area\n",
+ " output_area._bokeh_server_id = output.metadata[EXEC_MIME_TYPE][\"server_id\"];\n",
+ " }\n",
+ " }\n",
+ "\n",
+ " function register_renderer(events, OutputArea) {\n",
+ "\n",
+ " function append_mime(data, metadata, element) {\n",
+ " // create a DOM node to render to\n",
+ " var toinsert = this.create_output_subarea(\n",
+ " metadata,\n",
+ " CLASS_NAME,\n",
+ " EXEC_MIME_TYPE\n",
+ " );\n",
+ " this.keyboard_manager.register_events(toinsert);\n",
+ " // Render to node\n",
+ " var props = {data: data, metadata: metadata[EXEC_MIME_TYPE]};\n",
+ " render(props, toinsert[toinsert.length - 1]);\n",
+ " element.append(toinsert);\n",
+ " return toinsert\n",
+ " }\n",
+ "\n",
+ " /* Handle when an output is cleared or removed */\n",
+ " events.on('clear_output.CodeCell', handleClearOutput);\n",
+ " events.on('delete.Cell', handleClearOutput);\n",
+ "\n",
+ " /* Handle when a new output is added */\n",
+ " events.on('output_added.OutputArea', handleAddOutput);\n",
+ "\n",
+ " /**\n",
+ " * Register the mime type and append_mime function with output_area\n",
+ " */\n",
+ " OutputArea.prototype.register_mime_type(EXEC_MIME_TYPE, append_mime, {\n",
+ " /* Is output safe? */\n",
+ " safe: true,\n",
+ " /* Index of renderer in `output_area.display_order` */\n",
+ " index: 0\n",
+ " });\n",
+ " }\n",
+ "\n",
+ " // register the mime type if in Jupyter Notebook environment and previously unregistered\n",
+ " if (root.Jupyter !== undefined) {\n",
+ " var events = require('base/js/events');\n",
+ " var OutputArea = require('notebook/js/outputarea').OutputArea;\n",
+ "\n",
+ " if (OutputArea.prototype.mime_types().indexOf(EXEC_MIME_TYPE) == -1) {\n",
+ " register_renderer(events, OutputArea);\n",
+ " }\n",
+ " }\n",
+ "\n",
+ " \n",
+ " if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n",
+ " root._bokeh_timeout = Date.now() + 5000;\n",
+ " root._bokeh_failed_load = false;\n",
+ " }\n",
+ "\n",
+ " var NB_LOAD_WARNING = {'data': {'text/html':\n",
+ " \"
\\n\"+\n",
+ " \"
\\n\"+\n",
+ " \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n",
+ " \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n",
+ " \"
\\n\"+\n",
+ " \"
\\n\"+\n",
+ " \"
re-rerun `output_notebook()` to attempt to load from CDN again, or
\"}};\n",
+ "\n",
+ " function display_loaded() {\n",
+ " var el = document.getElementById(null);\n",
+ " if (el != null) {\n",
+ " el.textContent = \"BokehJS is loading...\";\n",
+ " }\n",
+ " if (root.Bokeh !== undefined) {\n",
+ " if (el != null) {\n",
+ " el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n",
+ " }\n",
+ " } else if (Date.now() < root._bokeh_timeout) {\n",
+ " setTimeout(display_loaded, 100)\n",
+ " }\n",
+ " }\n",
+ "\n",
+ "\n",
+ " function run_callbacks() {\n",
+ " try {\n",
+ " root._bokeh_onload_callbacks.forEach(function(callback) {\n",
+ " if (callback != null)\n",
+ " callback();\n",
+ " });\n",
+ " } finally {\n",
+ " delete root._bokeh_onload_callbacks\n",
+ " }\n",
+ " console.debug(\"Bokeh: all callbacks have finished\");\n",
+ " }\n",
+ "\n",
+ " function load_libs(css_urls, js_urls, callback) {\n",
+ " if (css_urls == null) css_urls = [];\n",
+ " if (js_urls == null) js_urls = [];\n",
+ "\n",
+ " root._bokeh_onload_callbacks.push(callback);\n",
+ " if (root._bokeh_is_loading > 0) {\n",
+ " console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n",
+ " return null;\n",
+ " }\n",
+ " if (js_urls == null || js_urls.length === 0) {\n",
+ " run_callbacks();\n",
+ " return null;\n",
+ " }\n",
+ " console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n",
+ " root._bokeh_is_loading = css_urls.length + js_urls.length;\n",
+ "\n",
+ " function on_load() {\n",
+ " root._bokeh_is_loading--;\n",
+ " if (root._bokeh_is_loading === 0) {\n",
+ " console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n",
+ " run_callbacks()\n",
+ " }\n",
+ " }\n",
+ "\n",
+ " function on_error() {\n",
+ " console.error(\"failed to load \" + url);\n",
+ " }\n",
+ "\n",
+ " for (var i = 0; i < css_urls.length; i++) {\n",
+ " var url = css_urls[i];\n",
+ " const element = document.createElement(\"link\");\n",
+ " element.onload = on_load;\n",
+ " element.onerror = on_error;\n",
+ " element.rel = \"stylesheet\";\n",
+ " element.type = \"text/css\";\n",
+ " element.href = url;\n",
+ " console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n",
+ " document.body.appendChild(element);\n",
+ " }\n",
+ "\n",
+ " for (var i = 0; i < js_urls.length; i++) {\n",
+ " var url = js_urls[i];\n",
+ " var element = document.createElement('script');\n",
+ " element.onload = on_load;\n",
+ " element.onerror = on_error;\n",
+ " element.async = false;\n",
+ " element.src = url;\n",
+ " console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n",
+ " document.head.appendChild(element);\n",
+ " }\n",
+ " };\n",
+ "\n",
+ " function inject_raw_css(css) {\n",
+ " const element = document.createElement(\"style\");\n",
+ " element.appendChild(document.createTextNode(css));\n",
+ " document.body.appendChild(element);\n",
+ " }\n",
+ "\n",
+ " \n",
+ " var js_urls = [\"https://cdn.pydata.org/bokeh/release/bokeh-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-widgets-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-tables-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-gl-1.4.0.min.js\"];\n",
+ " var css_urls = [];\n",
+ " \n",
+ "\n",
+ " var inline_js = [\n",
+ " function(Bokeh) {\n",
+ " Bokeh.set_log_level(\"info\");\n",
+ " },\n",
+ " function(Bokeh) {\n",
+ " \n",
+ " \n",
+ " }\n",
+ " ];\n",
+ "\n",
+ " function run_inline_js() {\n",
+ " \n",
+ " if (root.Bokeh !== undefined || force === true) {\n",
+ " \n",
+ " for (var i = 0; i < inline_js.length; i++) {\n",
+ " inline_js[i].call(root, root.Bokeh);\n",
+ " }\n",
+ " } else if (Date.now() < root._bokeh_timeout) {\n",
+ " setTimeout(run_inline_js, 100);\n",
+ " } else if (!root._bokeh_failed_load) {\n",
+ " console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n",
+ " root._bokeh_failed_load = true;\n",
+ " } else if (force !== true) {\n",
+ " var cell = $(document.getElementById(null)).parents('.cell').data().cell;\n",
+ " cell.output_area.append_execute_result(NB_LOAD_WARNING)\n",
+ " }\n",
+ "\n",
+ " }\n",
+ "\n",
+ " if (root._bokeh_is_loading === 0) {\n",
+ " console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n",
+ " run_inline_js();\n",
+ " } else {\n",
+ " load_libs(css_urls, js_urls, function() {\n",
+ " console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n",
+ " run_inline_js();\n",
+ " });\n",
+ " }\n",
+ "}(window));"
+ ],
+ "application/vnd.bokehjs_load.v0+json": "\n(function(root) {\n function now() {\n return new Date();\n }\n\n var force = true;\n\n if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n root._bokeh_onload_callbacks = [];\n root._bokeh_is_loading = undefined;\n }\n\n \n\n \n if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n root._bokeh_timeout = Date.now() + 5000;\n root._bokeh_failed_load = false;\n }\n\n var NB_LOAD_WARNING = {'data': {'text/html':\n \"
\\n\"+\n \"
\\n\"+\n \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n \"
\\n\"+\n \"
\\n\"+\n \"
re-rerun `output_notebook()` to attempt to load from CDN again, or
\"}};\n\n function display_loaded() {\n var el = document.getElementById(null);\n if (el != null) {\n el.textContent = \"BokehJS is loading...\";\n }\n if (root.Bokeh !== undefined) {\n if (el != null) {\n el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n }\n } else if (Date.now() < root._bokeh_timeout) {\n setTimeout(display_loaded, 100)\n }\n }\n\n\n function run_callbacks() {\n try {\n root._bokeh_onload_callbacks.forEach(function(callback) {\n if (callback != null)\n callback();\n });\n } finally {\n delete root._bokeh_onload_callbacks\n }\n console.debug(\"Bokeh: all callbacks have finished\");\n }\n\n function load_libs(css_urls, js_urls, callback) {\n if (css_urls == null) css_urls = [];\n if (js_urls == null) js_urls = [];\n\n root._bokeh_onload_callbacks.push(callback);\n if (root._bokeh_is_loading > 0) {\n console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n return null;\n }\n if (js_urls == null || js_urls.length === 0) {\n run_callbacks();\n return null;\n }\n console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n root._bokeh_is_loading = css_urls.length + js_urls.length;\n\n function on_load() {\n root._bokeh_is_loading--;\n if (root._bokeh_is_loading === 0) {\n console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n run_callbacks()\n }\n }\n\n function on_error() {\n console.error(\"failed to load \" + url);\n }\n\n for (var i = 0; i < css_urls.length; i++) {\n var url = css_urls[i];\n const element = document.createElement(\"link\");\n element.onload = on_load;\n element.onerror = on_error;\n element.rel = \"stylesheet\";\n element.type = \"text/css\";\n element.href = url;\n console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n document.body.appendChild(element);\n }\n\n for (var i = 0; i < js_urls.length; i++) {\n var url = js_urls[i];\n var element = document.createElement('script');\n element.onload = on_load;\n element.onerror = on_error;\n element.async = false;\n element.src = url;\n console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n document.head.appendChild(element);\n }\n };\n\n function inject_raw_css(css) {\n const element = document.createElement(\"style\");\n element.appendChild(document.createTextNode(css));\n document.body.appendChild(element);\n }\n\n \n var js_urls = [\"https://cdn.pydata.org/bokeh/release/bokeh-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-widgets-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-tables-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-gl-1.4.0.min.js\"];\n var css_urls = [];\n \n\n var inline_js = [\n function(Bokeh) {\n Bokeh.set_log_level(\"info\");\n },\n function(Bokeh) {\n \n \n }\n ];\n\n function run_inline_js() {\n \n if (root.Bokeh !== undefined || force === true) {\n \n for (var i = 0; i < inline_js.length; i++) {\n inline_js[i].call(root, root.Bokeh);\n }\n } else if (Date.now() < root._bokeh_timeout) {\n setTimeout(run_inline_js, 100);\n } else if (!root._bokeh_failed_load) {\n console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n root._bokeh_failed_load = true;\n } else if (force !== true) {\n var cell = $(document.getElementById(null)).parents('.cell').data().cell;\n cell.output_area.append_execute_result(NB_LOAD_WARNING)\n }\n\n }\n\n if (root._bokeh_is_loading === 0) {\n console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n run_inline_js();\n } else {\n load_libs(css_urls, js_urls, function() {\n console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n run_inline_js();\n });\n }\n}(window));"
+ },
+ "metadata": {
+ "tags": []
+ },
+ "output_type": "display_data"
+ },
+ {
+ "data": {
+ "text/html": [
+ "\n",
+ "\n",
+ "\n",
+ "\n",
+ "\n",
+ "\n",
+ " \n"
+ ]
+ },
+ "metadata": {
+ "tags": []
+ },
+ "output_type": "display_data"
+ },
+ {
+ "data": {
+ "application/javascript": [
+ "(function(root) {\n",
+ " function embed_document(root) {\n",
+ " \n",
+ " var docs_json = {\"b33996e2-7bdd-4097-888b-8bf79a526bff\":{\"roots\":{\"references\":[{\"attributes\":{\"above\":[{\"id\":\"1013\",\"type\":\"CategoricalAxis\"}],\"center\":[{\"id\":\"1016\",\"type\":\"Grid\"},{\"id\":\"1020\",\"type\":\"Grid\"}],\"left\":[{\"id\":\"1017\",\"type\":\"CategoricalAxis\"}],\"min_border_right\":300,\"plot_width\":1200,\"renderers\":[{\"id\":\"1030\",\"type\":\"GlyphRenderer\"}],\"title\":{\"id\":\"1003\",\"type\":\"Title\"},\"toolbar\":{\"id\":\"1023\",\"type\":\"Toolbar\"},\"toolbar_location\":\"below\",\"x_range\":{\"id\":\"1005\",\"type\":\"FactorRange\"},\"x_scale\":{\"id\":\"1009\",\"type\":\"CategoricalScale\"},\"y_range\":{\"id\":\"1007\",\"type\":\"FactorRange\"},\"y_scale\":{\"id\":\"1011\",\"type\":\"CategoricalScale\"}},\"id\":\"1002\",\"subtype\":\"Figure\",\"type\":\"Plot\"},{\"attributes\":{\"callback\":null,\"data\":{\"embeddings_1\":[\"Welcome to simple yet\",\"Welcome to simple yet\",\"Welcome to simple yet\",\"Welcome to simple yet\",\"Welcome to simple yet\",\"Welcome to simple yet\",\"Welcome to simple yet\",\"Welcome to simple yet\",\"surprisingly powerful\",\"surprisingly powerful\",\"surprisingly powerful\",\"surprisingly powerful\",\"surprisingly powerful\",\"surprisingly powerful\",\"surprisingly powerful\",\"surprisingly powerful\",\"multilingual\",\"multilingual\",\"multilingual\",\"multilingual\",\"multilingual\",\"multilingual\",\"multilingual\",\"multilingual\",\"natural language understanding\",\"natural language understanding\",\"natural language understanding\",\"natural language understanding\",\"natural language understanding\",\"natural language understanding\",\"natural language understanding\",\"natural language understanding\",\"models.\",\"models.\",\"models.\",\"models.\",\"models.\",\"models.\",\"models.\",\"models.\",\"What people mean\",\"What people mean\",\"What people mean\",\"What people mean\",\"What people mean\",\"What people mean\",\"What people mean\",\"What people mean\",\"matters more than\",\"matters more than\",\"matters more than\",\"matters more than\",\"matters more than\",\"matters more than\",\"matters more than\",\"matters more than\",\"the language they speak.\",\"the language they speak.\",\"the language they speak.\",\"the language they speak.\",\"the language they speak.\",\"the language they speak.\",\"the language they speak.\",\"the language they speak.\"],\"embeddings_2\":[\"Willkommen zu einfachen, aber\",\"verrassend krachtige\",\"multiling\\u00fce\",\"compr\\u00e9hension du langage naturel\",\"\\u043c\\u043e\\u0434\\u0435\\u043b\\u0438.\",\"\\u5927\\u5bb6\\u662f\\u4ec0\\u4e48\\u610f\\u601d\",\"\\ubcf4\\ub2e4 \\uc911\\uc694\\ud55c\",\".\\u0627\\u0644\\u0644\\u063a\\u0629 \\u0627\\u0644\\u062a\\u064a \\u064a\\u062a\\u062d\\u062f\\u062b\\u0648\\u0646\\u0647\\u0627\",\"Willkommen zu einfachen, aber\",\"verrassend krachtige\",\"multiling\\u00fce\",\"compr\\u00e9hension du langage naturel\",\"\\u043c\\u043e\\u0434\\u0435\\u043b\\u0438.\",\"\\u5927\\u5bb6\\u662f\\u4ec0\\u4e48\\u610f\\u601d\",\"\\ubcf4\\ub2e4 \\uc911\\uc694\\ud55c\",\".\\u0627\\u0644\\u0644\\u063a\\u0629 \\u0627\\u0644\\u062a\\u064a \\u064a\\u062a\\u062d\\u062f\\u062b\\u0648\\u0646\\u0647\\u0627\",\"Willkommen zu einfachen, aber\",\"verrassend krachtige\",\"multiling\\u00fce\",\"compr\\u00e9hension du langage naturel\",\"\\u043c\\u043e\\u0434\\u0435\\u043b\\u0438.\",\"\\u5927\\u5bb6\\u662f\\u4ec0\\u4e48\\u610f\\u601d\",\"\\ubcf4\\ub2e4 \\uc911\\uc694\\ud55c\",\".\\u0627\\u0644\\u0644\\u063a\\u0629 \\u0627\\u0644\\u062a\\u064a \\u064a\\u062a\\u062d\\u062f\\u062b\\u0648\\u0646\\u0647\\u0627\",\"Willkommen zu einfachen, aber\",\"verrassend krachtige\",\"multiling\\u00fce\",\"compr\\u00e9hension du langage naturel\",\"\\u043c\\u043e\\u0434\\u0435\\u043b\\u0438.\",\"\\u5927\\u5bb6\\u662f\\u4ec0\\u4e48\\u610f\\u601d\",\"\\ubcf4\\ub2e4 \\uc911\\uc694\\ud55c\",\".\\u0627\\u0644\\u0644\\u063a\\u0629 \\u0627\\u0644\\u062a\\u064a \\u064a\\u062a\\u062d\\u062f\\u062b\\u0648\\u0646\\u0647\\u0627\",\"Willkommen zu einfachen, aber\",\"verrassend krachtige\",\"multiling\\u00fce\",\"compr\\u00e9hension du langage naturel\",\"\\u043c\\u043e\\u0434\\u0435\\u043b\\u0438.\",\"\\u5927\\u5bb6\\u662f\\u4ec0\\u4e48\\u610f\\u601d\",\"\\ubcf4\\ub2e4 \\uc911\\uc694\\ud55c\",\".\\u0627\\u0644\\u0644\\u063a\\u0629 \\u0627\\u0644\\u062a\\u064a \\u064a\\u062a\\u062d\\u062f\\u062b\\u0648\\u0646\\u0647\\u0627\",\"Willkommen zu einfachen, aber\",\"verrassend krachtige\",\"multiling\\u00fce\",\"compr\\u00e9hension du langage naturel\",\"\\u043c\\u043e\\u0434\\u0435\\u043b\\u0438.\",\"\\u5927\\u5bb6\\u662f\\u4ec0\\u4e48\\u610f\\u601d\",\"\\ubcf4\\ub2e4 \\uc911\\uc694\\ud55c\",\".\\u0627\\u0644\\u0644\\u063a\\u0629 \\u0627\\u0644\\u062a\\u064a \\u064a\\u062a\\u062d\\u062f\\u062b\\u0648\\u0646\\u0647\\u0627\",\"Willkommen zu einfachen, aber\",\"verrassend krachtige\",\"multiling\\u00fce\",\"compr\\u00e9hension du langage naturel\",\"\\u043c\\u043e\\u0434\\u0435\\u043b\\u0438.\",\"\\u5927\\u5bb6\\u662f\\u4ec0\\u4e48\\u610f\\u601d\",\"\\ubcf4\\ub2e4 \\uc911\\uc694\\ud55c\",\".\\u0627\\u0644\\u0644\\u063a\\u0629 \\u0627\\u0644\\u062a\\u064a \\u064a\\u062a\\u062d\\u062f\\u062b\\u0648\\u0646\\u0647\\u0627\",\"Willkommen zu einfachen, aber\",\"verrassend krachtige\",\"multiling\\u00fce\",\"compr\\u00e9hension du langage naturel\",\"\\u043c\\u043e\\u0434\\u0435\\u043b\\u0438.\",\"\\u5927\\u5bb6\\u662f\\u4ec0\\u4e48\\u610f\\u601d\",\"\\ubcf4\\ub2e4 \\uc911\\uc694\\ud55c\",\".\\u0627\\u0644\\u0644\\u063a\\u0629 \\u0627\\u0644\\u062a\\u064a \\u064a\\u062a\\u062d\\u062f\\u062b\\u0648\\u0646\\u0647\\u0627\"],\"index\":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63],\"sim\":{\"__ndarray__\":\"AAAAwK4B6j8AAACgVl7iPwAAAMD5l94/AAAAQOVr4T8AAABA8/TfPwAAACDzG+A/AAAAAIa+4T8AAABAMivgPwAAAKBdieE/AAAAgBBH6T8AAAAA5griPwAAAIAUkeE/AAAAAJhp4D8AAACAkSTgPwAAAICvreI/AAAAgIcw4j8AAAAAELXfPwAAAKD19uE/AAAAwH/P6T8AAADAZbjjPwAAAABaX+I/AAAAgG6I4T8AAACAx/viPwAAACBrc+Y/AAAAQApe4T8AAACA32XiPwAAAAD0j+M/AAAAADsK7D8AAADgBnHhPwAAAAAIZeE/AAAAAHZm4T8AAACgXprmPwAAAAB5Jt8/AAAAQMci4D8AAACgPI7hPwAAAKBmSOE/AAAAQArv7D8AAABA7O7fPwAAAID9d+E/AAAAgIRv4T8AAACAN4PgPwAAAMD/yOA/AAAAgEIH4j8AAADAAFbiPwAAAMDwOuE/AAAAQE1v5j8AAABg2bLiPwAAAMDSyOI/AAAAwLXb4D8AAACAztLhPwAAAOB1HeI/AAAAgCd/4T8AAAAg1xHhPwAAAAAPQeE/AAAAAELU6T8AAAAgZjLiPwAAAACyYd8/AAAA4FKw4D8AAACgbUHkPwAAAEDcVeU/AAAAIHV34j8AAAAARznhPwAAAOCXLOE/AAAAQP8H6T8=\",\"dtype\":\"float64\",\"shape\":[64]}},\"selected\":{\"id\":\"1037\",\"type\":\"Selection\"},\"selection_policy\":{\"id\":\"1038\",\"type\":\"UnionRenderers\"}},\"id\":\"1026\",\"type\":\"ColumnDataSource\"},{\"attributes\":{\"text\":\"Multilingual Universal Sentence Encoder for Semantic Retrieval (Yang et al., 2019)\",\"text_font_size\":{\"value\":\"12pt\"}},\"id\":\"1003\",\"type\":\"Title\"},{\"attributes\":{},\"id\":\"1033\",\"type\":\"CategoricalTickFormatter\"},{\"attributes\":{\"callback\":null,\"factors\":[\".\\u0627\\u0644\\u0644\\u063a\\u0629 \\u0627\\u0644\\u062a\\u064a \\u064a\\u062a\\u062d\\u062f\\u062b\\u0648\\u0646\\u0647\\u0627\",\"\\ubcf4\\ub2e4 \\uc911\\uc694\\ud55c\",\"\\u5927\\u5bb6\\u662f\\u4ec0\\u4e48\\u610f\\u601d\",\"\\u043c\\u043e\\u0434\\u0435\\u043b\\u0438.\",\"compr\\u00e9hension du langage naturel\",\"multiling\\u00fce\",\"verrassend krachtige\",\"Willkommen zu einfachen, aber\"]},\"id\":\"1007\",\"type\":\"FactorRange\"},{\"attributes\":{\"callback\":null,\"factors\":[\"Welcome to simple yet\",\"surprisingly powerful\",\"multilingual\",\"natural language understanding\",\"models.\",\"What people mean\",\"matters more than\",\"the language they speak.\"]},\"id\":\"1005\",\"type\":\"FactorRange\"},{\"attributes\":{\"active_drag\":\"auto\",\"active_inspect\":\"auto\",\"active_multi\":null,\"active_scroll\":\"auto\",\"active_tap\":\"auto\",\"tools\":[{\"id\":\"1021\",\"type\":\"SaveTool\"},{\"id\":\"1022\",\"type\":\"HoverTool\"}]},\"id\":\"1023\",\"type\":\"Toolbar\"},{\"attributes\":{\"data_source\":{\"id\":\"1026\",\"type\":\"ColumnDataSource\"},\"glyph\":{\"id\":\"1028\",\"type\":\"Rect\"},\"hover_glyph\":null,\"muted_glyph\":null,\"nonselection_glyph\":{\"id\":\"1029\",\"type\":\"Rect\"},\"selection_glyph\":null,\"view\":{\"id\":\"1031\",\"type\":\"CDSView\"}},\"id\":\"1030\",\"type\":\"GlyphRenderer\"},{\"attributes\":{},\"id\":\"1009\",\"type\":\"CategoricalScale\"},{\"attributes\":{\"source\":{\"id\":\"1026\",\"type\":\"ColumnDataSource\"}},\"id\":\"1031\",\"type\":\"CDSView\"},{\"attributes\":{},\"id\":\"1011\",\"type\":\"CategoricalScale\"},{\"attributes\":{\"axis_line_color\":{\"value\":null},\"formatter\":{\"id\":\"1035\",\"type\":\"CategoricalTickFormatter\"},\"major_label_orientation\":0.7853981633974483,\"major_label_standoff\":16,\"major_label_text_font_size\":{\"value\":\"12pt\"},\"major_tick_line_color\":{\"value\":null},\"ticker\":{\"id\":\"1014\",\"type\":\"CategoricalTicker\"}},\"id\":\"1013\",\"type\":\"CategoricalAxis\"},{\"attributes\":{\"ticker\":{\"id\":\"1014\",\"type\":\"CategoricalTicker\"}},\"id\":\"1016\",\"type\":\"Grid\"},{\"attributes\":{},\"id\":\"1014\",\"type\":\"CategoricalTicker\"},{\"attributes\":{},\"id\":\"1037\",\"type\":\"Selection\"},{\"attributes\":{\"axis_line_color\":{\"value\":null},\"formatter\":{\"id\":\"1033\",\"type\":\"CategoricalTickFormatter\"},\"major_label_standoff\":16,\"major_label_text_font_size\":{\"value\":\"12pt\"},\"major_tick_line_color\":{\"value\":null},\"ticker\":{\"id\":\"1018\",\"type\":\"CategoricalTicker\"}},\"id\":\"1017\",\"type\":\"CategoricalAxis\"},{\"attributes\":{},\"id\":\"1038\",\"type\":\"UnionRenderers\"},{\"attributes\":{},\"id\":\"1018\",\"type\":\"CategoricalTicker\"},{\"attributes\":{},\"id\":\"1035\",\"type\":\"CategoricalTickFormatter\"},{\"attributes\":{\"dimension\":1,\"ticker\":{\"id\":\"1018\",\"type\":\"CategoricalTicker\"}},\"id\":\"1020\",\"type\":\"Grid\"},{\"attributes\":{},\"id\":\"1021\",\"type\":\"SaveTool\"},{\"attributes\":{\"callback\":null,\"tooltips\":[[\"pair\",\"@embeddings_1 ||| @embeddings_2\"],[\"sim\",\"@sim\"]]},\"id\":\"1022\",\"type\":\"HoverTool\"},{\"attributes\":{\"fill_color\":{\"field\":\"sim\",\"transform\":{\"id\":\"1001\",\"type\":\"LinearColorMapper\"}},\"height\":{\"units\":\"data\",\"value\":1},\"line_color\":{\"value\":null},\"width\":{\"units\":\"data\",\"value\":1},\"x\":{\"field\":\"embeddings_1\"},\"y\":{\"field\":\"embeddings_2\"}},\"id\":\"1028\",\"type\":\"Rect\"},{\"attributes\":{\"fill_alpha\":{\"value\":0.1},\"fill_color\":{\"value\":\"#1f77b4\"},\"height\":{\"units\":\"data\",\"value\":1},\"line_alpha\":{\"value\":0.1},\"line_color\":{\"value\":\"#1f77b4\"},\"width\":{\"units\":\"data\",\"value\":1},\"x\":{\"field\":\"embeddings_1\"},\"y\":{\"field\":\"embeddings_2\"}},\"id\":\"1029\",\"type\":\"Rect\"},{\"attributes\":{\"high\":0.9041796922683716,\"low\":0.4780258536338806,\"palette\":[\"#ffffcc\",\"#ffeda0\",\"#fed976\",\"#feb24c\",\"#fd8d3c\",\"#fc4e2a\",\"#e31a1c\",\"#bd0026\",\"#800026\"]},\"id\":\"1001\",\"type\":\"LinearColorMapper\"}],\"root_ids\":[\"1002\"]},\"title\":\"Bokeh Application\",\"version\":\"1.4.0\"}};\n",
+ " var render_items = [{\"docid\":\"b33996e2-7bdd-4097-888b-8bf79a526bff\",\"roots\":{\"1002\":\"c23bc7cf-8f04-4fa0-a38c-20c29e5098b9\"}}];\n",
+ " root.Bokeh.embed.embed_items_notebook(docs_json, render_items);\n",
+ "\n",
+ " }\n",
+ " if (root.Bokeh !== undefined) {\n",
+ " embed_document(root);\n",
+ " } else {\n",
+ " var attempts = 0;\n",
+ " var timer = setInterval(function(root) {\n",
+ " if (root.Bokeh !== undefined) {\n",
+ " clearInterval(timer);\n",
+ " embed_document(root);\n",
+ " } else {\n",
+ " attempts++;\n",
+ " if (attempts > 100) {\n",
+ " clearInterval(timer);\n",
+ " console.log(\"Bokeh: ERROR: Unable to run BokehJS code because BokehJS library is missing\");\n",
+ " }\n",
+ " }\n",
+ " }, 10, root)\n",
+ " }\n",
+ "})(window);"
+ ],
+ "application/vnd.bokehjs_exec.v0+json": ""
+ },
+ "metadata": {
+ "application/vnd.bokehjs_exec.v0+json": {
+ "id": "1002"
+ },
+ "tags": []
+ },
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "visualize_similarity(multilingual_in_en_result, multilingual_result,\n",
+ " multilingual_example_in_en, multilingual_example, \"Multilingual Universal Sentence Encoder for Semantic Retrieval (Yang et al., 2019)\")\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "h3TEhllsq3ax"
+ },
+ "source": [
+ "### English-Arabic Similarity"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "Q9UDpStmq7Ii"
+ },
+ "outputs": [
+ {
+ "data": {
+ "application/javascript": [
+ "\n",
+ "(function(root) {\n",
+ " function now() {\n",
+ " return new Date();\n",
+ " }\n",
+ "\n",
+ " var force = true;\n",
+ "\n",
+ " if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n",
+ " root._bokeh_onload_callbacks = [];\n",
+ " root._bokeh_is_loading = undefined;\n",
+ " }\n",
+ "\n",
+ " var JS_MIME_TYPE = 'application/javascript';\n",
+ " var HTML_MIME_TYPE = 'text/html';\n",
+ " var EXEC_MIME_TYPE = 'application/vnd.bokehjs_exec.v0+json';\n",
+ " var CLASS_NAME = 'output_bokeh rendered_html';\n",
+ "\n",
+ " /**\n",
+ " * Render data to the DOM node\n",
+ " */\n",
+ " function render(props, node) {\n",
+ " var script = document.createElement(\"script\");\n",
+ " node.appendChild(script);\n",
+ " }\n",
+ "\n",
+ " /**\n",
+ " * Handle when an output is cleared or removed\n",
+ " */\n",
+ " function handleClearOutput(event, handle) {\n",
+ " var cell = handle.cell;\n",
+ "\n",
+ " var id = cell.output_area._bokeh_element_id;\n",
+ " var server_id = cell.output_area._bokeh_server_id;\n",
+ " // Clean up Bokeh references\n",
+ " if (id != null && id in Bokeh.index) {\n",
+ " Bokeh.index[id].model.document.clear();\n",
+ " delete Bokeh.index[id];\n",
+ " }\n",
+ "\n",
+ " if (server_id !== undefined) {\n",
+ " // Clean up Bokeh references\n",
+ " var cmd = \"from bokeh.io.state import curstate; print(curstate().uuid_to_server['\" + server_id + \"'].get_sessions()[0].document.roots[0]._id)\";\n",
+ " cell.notebook.kernel.execute(cmd, {\n",
+ " iopub: {\n",
+ " output: function(msg) {\n",
+ " var id = msg.content.text.trim();\n",
+ " if (id in Bokeh.index) {\n",
+ " Bokeh.index[id].model.document.clear();\n",
+ " delete Bokeh.index[id];\n",
+ " }\n",
+ " }\n",
+ " }\n",
+ " });\n",
+ " // Destroy server and session\n",
+ " var cmd = \"import bokeh.io.notebook as ion; ion.destroy_server('\" + server_id + \"')\";\n",
+ " cell.notebook.kernel.execute(cmd);\n",
+ " }\n",
+ " }\n",
+ "\n",
+ " /**\n",
+ " * Handle when a new output is added\n",
+ " */\n",
+ " function handleAddOutput(event, handle) {\n",
+ " var output_area = handle.output_area;\n",
+ " var output = handle.output;\n",
+ "\n",
+ " // limit handleAddOutput to display_data with EXEC_MIME_TYPE content only\n",
+ " if ((output.output_type != \"display_data\") || (!output.data.hasOwnProperty(EXEC_MIME_TYPE))) {\n",
+ " return\n",
+ " }\n",
+ "\n",
+ " var toinsert = output_area.element.find(\".\" + CLASS_NAME.split(' ')[0]);\n",
+ "\n",
+ " if (output.metadata[EXEC_MIME_TYPE][\"id\"] !== undefined) {\n",
+ " toinsert[toinsert.length - 1].firstChild.textContent = output.data[JS_MIME_TYPE];\n",
+ " // store reference to embed id on output_area\n",
+ " output_area._bokeh_element_id = output.metadata[EXEC_MIME_TYPE][\"id\"];\n",
+ " }\n",
+ " if (output.metadata[EXEC_MIME_TYPE][\"server_id\"] !== undefined) {\n",
+ " var bk_div = document.createElement(\"div\");\n",
+ " bk_div.innerHTML = output.data[HTML_MIME_TYPE];\n",
+ " var script_attrs = bk_div.children[0].attributes;\n",
+ " for (var i = 0; i < script_attrs.length; i++) {\n",
+ " toinsert[toinsert.length - 1].firstChild.setAttribute(script_attrs[i].name, script_attrs[i].value);\n",
+ " }\n",
+ " // store reference to server id on output_area\n",
+ " output_area._bokeh_server_id = output.metadata[EXEC_MIME_TYPE][\"server_id\"];\n",
+ " }\n",
+ " }\n",
+ "\n",
+ " function register_renderer(events, OutputArea) {\n",
+ "\n",
+ " function append_mime(data, metadata, element) {\n",
+ " // create a DOM node to render to\n",
+ " var toinsert = this.create_output_subarea(\n",
+ " metadata,\n",
+ " CLASS_NAME,\n",
+ " EXEC_MIME_TYPE\n",
+ " );\n",
+ " this.keyboard_manager.register_events(toinsert);\n",
+ " // Render to node\n",
+ " var props = {data: data, metadata: metadata[EXEC_MIME_TYPE]};\n",
+ " render(props, toinsert[toinsert.length - 1]);\n",
+ " element.append(toinsert);\n",
+ " return toinsert\n",
+ " }\n",
+ "\n",
+ " /* Handle when an output is cleared or removed */\n",
+ " events.on('clear_output.CodeCell', handleClearOutput);\n",
+ " events.on('delete.Cell', handleClearOutput);\n",
+ "\n",
+ " /* Handle when a new output is added */\n",
+ " events.on('output_added.OutputArea', handleAddOutput);\n",
+ "\n",
+ " /**\n",
+ " * Register the mime type and append_mime function with output_area\n",
+ " */\n",
+ " OutputArea.prototype.register_mime_type(EXEC_MIME_TYPE, append_mime, {\n",
+ " /* Is output safe? */\n",
+ " safe: true,\n",
+ " /* Index of renderer in `output_area.display_order` */\n",
+ " index: 0\n",
+ " });\n",
+ " }\n",
+ "\n",
+ " // register the mime type if in Jupyter Notebook environment and previously unregistered\n",
+ " if (root.Jupyter !== undefined) {\n",
+ " var events = require('base/js/events');\n",
+ " var OutputArea = require('notebook/js/outputarea').OutputArea;\n",
+ "\n",
+ " if (OutputArea.prototype.mime_types().indexOf(EXEC_MIME_TYPE) == -1) {\n",
+ " register_renderer(events, OutputArea);\n",
+ " }\n",
+ " }\n",
+ "\n",
+ " \n",
+ " if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n",
+ " root._bokeh_timeout = Date.now() + 5000;\n",
+ " root._bokeh_failed_load = false;\n",
+ " }\n",
+ "\n",
+ " var NB_LOAD_WARNING = {'data': {'text/html':\n",
+ " \"
\\n\"+\n",
+ " \"
\\n\"+\n",
+ " \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n",
+ " \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n",
+ " \"
\\n\"+\n",
+ " \"
\\n\"+\n",
+ " \"
re-rerun `output_notebook()` to attempt to load from CDN again, or
\"}};\n",
+ "\n",
+ " function display_loaded() {\n",
+ " var el = document.getElementById(null);\n",
+ " if (el != null) {\n",
+ " el.textContent = \"BokehJS is loading...\";\n",
+ " }\n",
+ " if (root.Bokeh !== undefined) {\n",
+ " if (el != null) {\n",
+ " el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n",
+ " }\n",
+ " } else if (Date.now() < root._bokeh_timeout) {\n",
+ " setTimeout(display_loaded, 100)\n",
+ " }\n",
+ " }\n",
+ "\n",
+ "\n",
+ " function run_callbacks() {\n",
+ " try {\n",
+ " root._bokeh_onload_callbacks.forEach(function(callback) {\n",
+ " if (callback != null)\n",
+ " callback();\n",
+ " });\n",
+ " } finally {\n",
+ " delete root._bokeh_onload_callbacks\n",
+ " }\n",
+ " console.debug(\"Bokeh: all callbacks have finished\");\n",
+ " }\n",
+ "\n",
+ " function load_libs(css_urls, js_urls, callback) {\n",
+ " if (css_urls == null) css_urls = [];\n",
+ " if (js_urls == null) js_urls = [];\n",
+ "\n",
+ " root._bokeh_onload_callbacks.push(callback);\n",
+ " if (root._bokeh_is_loading > 0) {\n",
+ " console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n",
+ " return null;\n",
+ " }\n",
+ " if (js_urls == null || js_urls.length === 0) {\n",
+ " run_callbacks();\n",
+ " return null;\n",
+ " }\n",
+ " console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n",
+ " root._bokeh_is_loading = css_urls.length + js_urls.length;\n",
+ "\n",
+ " function on_load() {\n",
+ " root._bokeh_is_loading--;\n",
+ " if (root._bokeh_is_loading === 0) {\n",
+ " console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n",
+ " run_callbacks()\n",
+ " }\n",
+ " }\n",
+ "\n",
+ " function on_error() {\n",
+ " console.error(\"failed to load \" + url);\n",
+ " }\n",
+ "\n",
+ " for (var i = 0; i < css_urls.length; i++) {\n",
+ " var url = css_urls[i];\n",
+ " const element = document.createElement(\"link\");\n",
+ " element.onload = on_load;\n",
+ " element.onerror = on_error;\n",
+ " element.rel = \"stylesheet\";\n",
+ " element.type = \"text/css\";\n",
+ " element.href = url;\n",
+ " console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n",
+ " document.body.appendChild(element);\n",
+ " }\n",
+ "\n",
+ " for (var i = 0; i < js_urls.length; i++) {\n",
+ " var url = js_urls[i];\n",
+ " var element = document.createElement('script');\n",
+ " element.onload = on_load;\n",
+ " element.onerror = on_error;\n",
+ " element.async = false;\n",
+ " element.src = url;\n",
+ " console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n",
+ " document.head.appendChild(element);\n",
+ " }\n",
+ " };\n",
+ "\n",
+ " function inject_raw_css(css) {\n",
+ " const element = document.createElement(\"style\");\n",
+ " element.appendChild(document.createTextNode(css));\n",
+ " document.body.appendChild(element);\n",
+ " }\n",
+ "\n",
+ " \n",
+ " var js_urls = [\"https://cdn.pydata.org/bokeh/release/bokeh-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-widgets-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-tables-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-gl-1.4.0.min.js\"];\n",
+ " var css_urls = [];\n",
+ " \n",
+ "\n",
+ " var inline_js = [\n",
+ " function(Bokeh) {\n",
+ " Bokeh.set_log_level(\"info\");\n",
+ " },\n",
+ " function(Bokeh) {\n",
+ " \n",
+ " \n",
+ " }\n",
+ " ];\n",
+ "\n",
+ " function run_inline_js() {\n",
+ " \n",
+ " if (root.Bokeh !== undefined || force === true) {\n",
+ " \n",
+ " for (var i = 0; i < inline_js.length; i++) {\n",
+ " inline_js[i].call(root, root.Bokeh);\n",
+ " }\n",
+ " } else if (Date.now() < root._bokeh_timeout) {\n",
+ " setTimeout(run_inline_js, 100);\n",
+ " } else if (!root._bokeh_failed_load) {\n",
+ " console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n",
+ " root._bokeh_failed_load = true;\n",
+ " } else if (force !== true) {\n",
+ " var cell = $(document.getElementById(null)).parents('.cell').data().cell;\n",
+ " cell.output_area.append_execute_result(NB_LOAD_WARNING)\n",
+ " }\n",
+ "\n",
+ " }\n",
+ "\n",
+ " if (root._bokeh_is_loading === 0) {\n",
+ " console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n",
+ " run_inline_js();\n",
+ " } else {\n",
+ " load_libs(css_urls, js_urls, function() {\n",
+ " console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n",
+ " run_inline_js();\n",
+ " });\n",
+ " }\n",
+ "}(window));"
+ ],
+ "application/vnd.bokehjs_load.v0+json": "\n(function(root) {\n function now() {\n return new Date();\n }\n\n var force = true;\n\n if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n root._bokeh_onload_callbacks = [];\n root._bokeh_is_loading = undefined;\n }\n\n \n\n \n if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n root._bokeh_timeout = Date.now() + 5000;\n root._bokeh_failed_load = false;\n }\n\n var NB_LOAD_WARNING = {'data': {'text/html':\n \"
\\n\"+\n \"
\\n\"+\n \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n \"
\\n\"+\n \"
\\n\"+\n \"
re-rerun `output_notebook()` to attempt to load from CDN again, or
\"}};\n\n function display_loaded() {\n var el = document.getElementById(null);\n if (el != null) {\n el.textContent = \"BokehJS is loading...\";\n }\n if (root.Bokeh !== undefined) {\n if (el != null) {\n el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n }\n } else if (Date.now() < root._bokeh_timeout) {\n setTimeout(display_loaded, 100)\n }\n }\n\n\n function run_callbacks() {\n try {\n root._bokeh_onload_callbacks.forEach(function(callback) {\n if (callback != null)\n callback();\n });\n } finally {\n delete root._bokeh_onload_callbacks\n }\n console.debug(\"Bokeh: all callbacks have finished\");\n }\n\n function load_libs(css_urls, js_urls, callback) {\n if (css_urls == null) css_urls = [];\n if (js_urls == null) js_urls = [];\n\n root._bokeh_onload_callbacks.push(callback);\n if (root._bokeh_is_loading > 0) {\n console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n return null;\n }\n if (js_urls == null || js_urls.length === 0) {\n run_callbacks();\n return null;\n }\n console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n root._bokeh_is_loading = css_urls.length + js_urls.length;\n\n function on_load() {\n root._bokeh_is_loading--;\n if (root._bokeh_is_loading === 0) {\n console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n run_callbacks()\n }\n }\n\n function on_error() {\n console.error(\"failed to load \" + url);\n }\n\n for (var i = 0; i < css_urls.length; i++) {\n var url = css_urls[i];\n const element = document.createElement(\"link\");\n element.onload = on_load;\n element.onerror = on_error;\n element.rel = \"stylesheet\";\n element.type = \"text/css\";\n element.href = url;\n console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n document.body.appendChild(element);\n }\n\n for (var i = 0; i < js_urls.length; i++) {\n var url = js_urls[i];\n var element = document.createElement('script');\n element.onload = on_load;\n element.onerror = on_error;\n element.async = false;\n element.src = url;\n console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n document.head.appendChild(element);\n }\n };\n\n function inject_raw_css(css) {\n const element = document.createElement(\"style\");\n element.appendChild(document.createTextNode(css));\n document.body.appendChild(element);\n }\n\n \n var js_urls = [\"https://cdn.pydata.org/bokeh/release/bokeh-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-widgets-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-tables-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-gl-1.4.0.min.js\"];\n var css_urls = [];\n \n\n var inline_js = [\n function(Bokeh) {\n Bokeh.set_log_level(\"info\");\n },\n function(Bokeh) {\n \n \n }\n ];\n\n function run_inline_js() {\n \n if (root.Bokeh !== undefined || force === true) {\n \n for (var i = 0; i < inline_js.length; i++) {\n inline_js[i].call(root, root.Bokeh);\n }\n } else if (Date.now() < root._bokeh_timeout) {\n setTimeout(run_inline_js, 100);\n } else if (!root._bokeh_failed_load) {\n console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n root._bokeh_failed_load = true;\n } else if (force !== true) {\n var cell = $(document.getElementById(null)).parents('.cell').data().cell;\n cell.output_area.append_execute_result(NB_LOAD_WARNING)\n }\n\n }\n\n if (root._bokeh_is_loading === 0) {\n console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n run_inline_js();\n } else {\n load_libs(css_urls, js_urls, function() {\n console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n run_inline_js();\n });\n }\n}(window));"
+ },
+ "metadata": {
+ "tags": []
+ },
+ "output_type": "display_data"
+ },
+ {
+ "data": {
+ "text/html": [
+ "\n",
+ "\n",
+ "\n",
+ "\n",
+ "\n",
+ "\n",
+ " \n"
+ ]
+ },
+ "metadata": {
+ "tags": []
+ },
+ "output_type": "display_data"
+ },
+ {
+ "data": {
+ "application/javascript": [
+ "(function(root) {\n",
+ " function embed_document(root) {\n",
+ " \n",
+ " var docs_json = {\"a8ebb685-fc25-4a41-b1f1-80df179ccab5\":{\"roots\":{\"references\":[{\"attributes\":{\"above\":[{\"id\":\"1093\",\"type\":\"CategoricalAxis\"}],\"center\":[{\"id\":\"1096\",\"type\":\"Grid\"},{\"id\":\"1100\",\"type\":\"Grid\"}],\"left\":[{\"id\":\"1097\",\"type\":\"CategoricalAxis\"}],\"min_border_right\":300,\"plot_width\":1200,\"renderers\":[{\"id\":\"1110\",\"type\":\"GlyphRenderer\"}],\"title\":{\"id\":\"1083\",\"type\":\"Title\"},\"toolbar\":{\"id\":\"1103\",\"type\":\"Toolbar\"},\"toolbar_location\":\"below\",\"x_range\":{\"id\":\"1085\",\"type\":\"FactorRange\"},\"x_scale\":{\"id\":\"1089\",\"type\":\"CategoricalScale\"},\"y_range\":{\"id\":\"1087\",\"type\":\"FactorRange\"},\"y_scale\":{\"id\":\"1091\",\"type\":\"CategoricalScale\"}},\"id\":\"1082\",\"subtype\":\"Figure\",\"type\":\"Plot\"},{\"attributes\":{\"data_source\":{\"id\":\"1106\",\"type\":\"ColumnDataSource\"},\"glyph\":{\"id\":\"1108\",\"type\":\"Rect\"},\"hover_glyph\":null,\"muted_glyph\":null,\"nonselection_glyph\":{\"id\":\"1109\",\"type\":\"Rect\"},\"selection_glyph\":null,\"view\":{\"id\":\"1111\",\"type\":\"CDSView\"}},\"id\":\"1110\",\"type\":\"GlyphRenderer\"},{\"attributes\":{\"source\":{\"id\":\"1106\",\"type\":\"ColumnDataSource\"}},\"id\":\"1111\",\"type\":\"CDSView\"},{\"attributes\":{},\"id\":\"1120\",\"type\":\"CategoricalTickFormatter\"},{\"attributes\":{\"fill_alpha\":{\"value\":0.1},\"fill_color\":{\"value\":\"#1f77b4\"},\"height\":{\"units\":\"data\",\"value\":1},\"line_alpha\":{\"value\":0.1},\"line_color\":{\"value\":\"#1f77b4\"},\"width\":{\"units\":\"data\",\"value\":1},\"x\":{\"field\":\"embeddings_1\"},\"y\":{\"field\":\"embeddings_2\"}},\"id\":\"1109\",\"type\":\"Rect\"},{\"attributes\":{\"callback\":null,\"tooltips\":[[\"pair\",\"@embeddings_1 ||| @embeddings_2\"],[\"sim\",\"@sim\"]]},\"id\":\"1102\",\"type\":\"HoverTool\"},{\"attributes\":{\"active_drag\":\"auto\",\"active_inspect\":\"auto\",\"active_multi\":null,\"active_scroll\":\"auto\",\"active_tap\":\"auto\",\"tools\":[{\"id\":\"1101\",\"type\":\"SaveTool\"},{\"id\":\"1102\",\"type\":\"HoverTool\"}]},\"id\":\"1103\",\"type\":\"Toolbar\"},{\"attributes\":{},\"id\":\"1094\",\"type\":\"CategoricalTicker\"},{\"attributes\":{},\"id\":\"1098\",\"type\":\"CategoricalTicker\"},{\"attributes\":{\"dimension\":1,\"ticker\":{\"id\":\"1098\",\"type\":\"CategoricalTicker\"}},\"id\":\"1100\",\"type\":\"Grid\"},{\"attributes\":{},\"id\":\"1122\",\"type\":\"CategoricalTickFormatter\"},{\"attributes\":{},\"id\":\"1124\",\"type\":\"Selection\"},{\"attributes\":{\"axis_line_color\":{\"value\":null},\"formatter\":{\"id\":\"1120\",\"type\":\"CategoricalTickFormatter\"},\"major_label_standoff\":16,\"major_label_text_font_size\":{\"value\":\"12pt\"},\"major_tick_line_color\":{\"value\":null},\"ticker\":{\"id\":\"1098\",\"type\":\"CategoricalTicker\"}},\"id\":\"1097\",\"type\":\"CategoricalAxis\"},{\"attributes\":{},\"id\":\"1125\",\"type\":\"UnionRenderers\"},{\"attributes\":{\"ticker\":{\"id\":\"1094\",\"type\":\"CategoricalTicker\"}},\"id\":\"1096\",\"type\":\"Grid\"},{\"attributes\":{},\"id\":\"1091\",\"type\":\"CategoricalScale\"},{\"attributes\":{\"axis_line_color\":{\"value\":null},\"formatter\":{\"id\":\"1122\",\"type\":\"CategoricalTickFormatter\"},\"major_label_orientation\":0.7853981633974483,\"major_label_standoff\":16,\"major_label_text_font_size\":{\"value\":\"12pt\"},\"major_tick_line_color\":{\"value\":null},\"ticker\":{\"id\":\"1094\",\"type\":\"CategoricalTicker\"}},\"id\":\"1093\",\"type\":\"CategoricalAxis\"},{\"attributes\":{},\"id\":\"1089\",\"type\":\"CategoricalScale\"},{\"attributes\":{\"callback\":null,\"factors\":[\"\\u0623\\u0633\\u062a\\u0645\\u062a\\u0639 \\u0628\\u0627\\u0644\\u0645\\u0634\\u064a \\u0644\\u0645\\u0633\\u0627\\u0641\\u0627\\u062a \\u0637\\u0648\\u064a\\u0644\\u0629 \\u0639\\u0644\\u0649 \\u0637\\u0648\\u0644 \\u0627\\u0644\\u0634\\u0627\\u0637\\u0626 \\u0645\\u0639 \\u0643\\u0644\\u0628\\u064a.\",\"\\u0627\\u0644\\u062c\\u0631\\u0627\\u0621 \\u0644\\u0637\\u064a\\u0641\\u0629.\",\"\\u0643\\u0644\\u0628\"]},\"id\":\"1087\",\"type\":\"FactorRange\"},{\"attributes\":{\"callback\":null,\"factors\":[\"dog\",\"Puppies are nice.\",\"I enjoy taking long walks along the beach with my dog.\"]},\"id\":\"1085\",\"type\":\"FactorRange\"},{\"attributes\":{\"fill_color\":{\"field\":\"sim\",\"transform\":{\"id\":\"1081\",\"type\":\"LinearColorMapper\"}},\"height\":{\"units\":\"data\",\"value\":1},\"line_color\":{\"value\":null},\"width\":{\"units\":\"data\",\"value\":1},\"x\":{\"field\":\"embeddings_1\"},\"y\":{\"field\":\"embeddings_2\"}},\"id\":\"1108\",\"type\":\"Rect\"},{\"attributes\":{\"callback\":null,\"data\":{\"embeddings_1\":[\"dog\",\"dog\",\"dog\",\"Puppies are nice.\",\"Puppies are nice.\",\"Puppies are nice.\",\"I enjoy taking long walks along the beach with my dog.\",\"I enjoy taking long walks along the beach with my dog.\",\"I enjoy taking long walks along the beach with my dog.\"],\"embeddings_2\":[\"\\u0643\\u0644\\u0628\",\"\\u0627\\u0644\\u062c\\u0631\\u0627\\u0621 \\u0644\\u0637\\u064a\\u0641\\u0629.\",\"\\u0623\\u0633\\u062a\\u0645\\u062a\\u0639 \\u0628\\u0627\\u0644\\u0645\\u0634\\u064a \\u0644\\u0645\\u0633\\u0627\\u0641\\u0627\\u062a \\u0637\\u0648\\u064a\\u0644\\u0629 \\u0639\\u0644\\u0649 \\u0637\\u0648\\u0644 \\u0627\\u0644\\u0634\\u0627\\u0637\\u0626 \\u0645\\u0639 \\u0643\\u0644\\u0628\\u064a.\",\"\\u0643\\u0644\\u0628\",\"\\u0627\\u0644\\u062c\\u0631\\u0627\\u0621 \\u0644\\u0637\\u064a\\u0641\\u0629.\",\"\\u0623\\u0633\\u062a\\u0645\\u062a\\u0639 \\u0628\\u0627\\u0644\\u0645\\u0634\\u064a \\u0644\\u0645\\u0633\\u0627\\u0641\\u0627\\u062a \\u0637\\u0648\\u064a\\u0644\\u0629 \\u0639\\u0644\\u0649 \\u0637\\u0648\\u0644 \\u0627\\u0644\\u0634\\u0627\\u0637\\u0626 \\u0645\\u0639 \\u0643\\u0644\\u0628\\u064a.\",\"\\u0643\\u0644\\u0628\",\"\\u0627\\u0644\\u062c\\u0631\\u0627\\u0621 \\u0644\\u0637\\u064a\\u0641\\u0629.\",\"\\u0623\\u0633\\u062a\\u0645\\u062a\\u0639 \\u0628\\u0627\\u0644\\u0645\\u0634\\u064a \\u0644\\u0645\\u0633\\u0627\\u0641\\u0627\\u062a \\u0637\\u0648\\u064a\\u0644\\u0629 \\u0639\\u0644\\u0649 \\u0637\\u0648\\u0644 \\u0627\\u0644\\u0634\\u0627\\u0637\\u0626 \\u0645\\u0639 \\u0643\\u0644\\u0628\\u064a.\"],\"index\":[0,1,2,3,4,5,6,7,8],\"sim\":{\"__ndarray__\":\"AAAAQLvE6j8AAABAlJHkPwAAAAAOj+A/AAAAwOqU5D8AAACA9aTnPwAAAMACcuA/AAAAAFtl4j8AAACAFvjiPwAAAIB6QOg/\",\"dtype\":\"float64\",\"shape\":[9]}},\"selected\":{\"id\":\"1124\",\"type\":\"Selection\"},\"selection_policy\":{\"id\":\"1125\",\"type\":\"UnionRenderers\"}},\"id\":\"1106\",\"type\":\"ColumnDataSource\"},{\"attributes\":{},\"id\":\"1101\",\"type\":\"SaveTool\"},{\"attributes\":{\"text\":\"English-Arabic Similarity\",\"text_font_size\":{\"value\":\"12pt\"}},\"id\":\"1083\",\"type\":\"Title\"},{\"attributes\":{\"high\":0.8365150690078735,\"low\":0.5139173269271851,\"palette\":[\"#ffffcc\",\"#ffeda0\",\"#fed976\",\"#feb24c\",\"#fd8d3c\",\"#fc4e2a\",\"#e31a1c\",\"#bd0026\",\"#800026\"]},\"id\":\"1081\",\"type\":\"LinearColorMapper\"}],\"root_ids\":[\"1082\"]},\"title\":\"Bokeh Application\",\"version\":\"1.4.0\"}};\n",
+ " var render_items = [{\"docid\":\"a8ebb685-fc25-4a41-b1f1-80df179ccab5\",\"roots\":{\"1082\":\"d20b027b-6bf0-444d-8763-df8d299e1642\"}}];\n",
+ " root.Bokeh.embed.embed_items_notebook(docs_json, render_items);\n",
+ "\n",
+ " }\n",
+ " if (root.Bokeh !== undefined) {\n",
+ " embed_document(root);\n",
+ " } else {\n",
+ " var attempts = 0;\n",
+ " var timer = setInterval(function(root) {\n",
+ " if (root.Bokeh !== undefined) {\n",
+ " clearInterval(timer);\n",
+ " embed_document(root);\n",
+ " } else {\n",
+ " attempts++;\n",
+ " if (attempts > 100) {\n",
+ " clearInterval(timer);\n",
+ " console.log(\"Bokeh: ERROR: Unable to run BokehJS code because BokehJS library is missing\");\n",
+ " }\n",
+ " }\n",
+ " }, 10, root)\n",
+ " }\n",
+ "})(window);"
+ ],
+ "application/vnd.bokehjs_exec.v0+json": ""
+ },
+ "metadata": {
+ "application/vnd.bokehjs_exec.v0+json": {
+ "id": "1082"
+ },
+ "tags": []
+ },
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "visualize_similarity(en_result, ar_result, english_sentences, arabic_sentences, 'English-Arabic Similarity')"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "QF9z48HMp4WL"
+ },
+ "source": [
+ "### Engish-Russian Similarity"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "QE68UejYp86z"
+ },
+ "outputs": [
+ {
+ "data": {
+ "application/javascript": [
+ "\n",
+ "(function(root) {\n",
+ " function now() {\n",
+ " return new Date();\n",
+ " }\n",
+ "\n",
+ " var force = true;\n",
+ "\n",
+ " if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n",
+ " root._bokeh_onload_callbacks = [];\n",
+ " root._bokeh_is_loading = undefined;\n",
+ " }\n",
+ "\n",
+ " var JS_MIME_TYPE = 'application/javascript';\n",
+ " var HTML_MIME_TYPE = 'text/html';\n",
+ " var EXEC_MIME_TYPE = 'application/vnd.bokehjs_exec.v0+json';\n",
+ " var CLASS_NAME = 'output_bokeh rendered_html';\n",
+ "\n",
+ " /**\n",
+ " * Render data to the DOM node\n",
+ " */\n",
+ " function render(props, node) {\n",
+ " var script = document.createElement(\"script\");\n",
+ " node.appendChild(script);\n",
+ " }\n",
+ "\n",
+ " /**\n",
+ " * Handle when an output is cleared or removed\n",
+ " */\n",
+ " function handleClearOutput(event, handle) {\n",
+ " var cell = handle.cell;\n",
+ "\n",
+ " var id = cell.output_area._bokeh_element_id;\n",
+ " var server_id = cell.output_area._bokeh_server_id;\n",
+ " // Clean up Bokeh references\n",
+ " if (id != null && id in Bokeh.index) {\n",
+ " Bokeh.index[id].model.document.clear();\n",
+ " delete Bokeh.index[id];\n",
+ " }\n",
+ "\n",
+ " if (server_id !== undefined) {\n",
+ " // Clean up Bokeh references\n",
+ " var cmd = \"from bokeh.io.state import curstate; print(curstate().uuid_to_server['\" + server_id + \"'].get_sessions()[0].document.roots[0]._id)\";\n",
+ " cell.notebook.kernel.execute(cmd, {\n",
+ " iopub: {\n",
+ " output: function(msg) {\n",
+ " var id = msg.content.text.trim();\n",
+ " if (id in Bokeh.index) {\n",
+ " Bokeh.index[id].model.document.clear();\n",
+ " delete Bokeh.index[id];\n",
+ " }\n",
+ " }\n",
+ " }\n",
+ " });\n",
+ " // Destroy server and session\n",
+ " var cmd = \"import bokeh.io.notebook as ion; ion.destroy_server('\" + server_id + \"')\";\n",
+ " cell.notebook.kernel.execute(cmd);\n",
+ " }\n",
+ " }\n",
+ "\n",
+ " /**\n",
+ " * Handle when a new output is added\n",
+ " */\n",
+ " function handleAddOutput(event, handle) {\n",
+ " var output_area = handle.output_area;\n",
+ " var output = handle.output;\n",
+ "\n",
+ " // limit handleAddOutput to display_data with EXEC_MIME_TYPE content only\n",
+ " if ((output.output_type != \"display_data\") || (!output.data.hasOwnProperty(EXEC_MIME_TYPE))) {\n",
+ " return\n",
+ " }\n",
+ "\n",
+ " var toinsert = output_area.element.find(\".\" + CLASS_NAME.split(' ')[0]);\n",
+ "\n",
+ " if (output.metadata[EXEC_MIME_TYPE][\"id\"] !== undefined) {\n",
+ " toinsert[toinsert.length - 1].firstChild.textContent = output.data[JS_MIME_TYPE];\n",
+ " // store reference to embed id on output_area\n",
+ " output_area._bokeh_element_id = output.metadata[EXEC_MIME_TYPE][\"id\"];\n",
+ " }\n",
+ " if (output.metadata[EXEC_MIME_TYPE][\"server_id\"] !== undefined) {\n",
+ " var bk_div = document.createElement(\"div\");\n",
+ " bk_div.innerHTML = output.data[HTML_MIME_TYPE];\n",
+ " var script_attrs = bk_div.children[0].attributes;\n",
+ " for (var i = 0; i < script_attrs.length; i++) {\n",
+ " toinsert[toinsert.length - 1].firstChild.setAttribute(script_attrs[i].name, script_attrs[i].value);\n",
+ " }\n",
+ " // store reference to server id on output_area\n",
+ " output_area._bokeh_server_id = output.metadata[EXEC_MIME_TYPE][\"server_id\"];\n",
+ " }\n",
+ " }\n",
+ "\n",
+ " function register_renderer(events, OutputArea) {\n",
+ "\n",
+ " function append_mime(data, metadata, element) {\n",
+ " // create a DOM node to render to\n",
+ " var toinsert = this.create_output_subarea(\n",
+ " metadata,\n",
+ " CLASS_NAME,\n",
+ " EXEC_MIME_TYPE\n",
+ " );\n",
+ " this.keyboard_manager.register_events(toinsert);\n",
+ " // Render to node\n",
+ " var props = {data: data, metadata: metadata[EXEC_MIME_TYPE]};\n",
+ " render(props, toinsert[toinsert.length - 1]);\n",
+ " element.append(toinsert);\n",
+ " return toinsert\n",
+ " }\n",
+ "\n",
+ " /* Handle when an output is cleared or removed */\n",
+ " events.on('clear_output.CodeCell', handleClearOutput);\n",
+ " events.on('delete.Cell', handleClearOutput);\n",
+ "\n",
+ " /* Handle when a new output is added */\n",
+ " events.on('output_added.OutputArea', handleAddOutput);\n",
+ "\n",
+ " /**\n",
+ " * Register the mime type and append_mime function with output_area\n",
+ " */\n",
+ " OutputArea.prototype.register_mime_type(EXEC_MIME_TYPE, append_mime, {\n",
+ " /* Is output safe? */\n",
+ " safe: true,\n",
+ " /* Index of renderer in `output_area.display_order` */\n",
+ " index: 0\n",
+ " });\n",
+ " }\n",
+ "\n",
+ " // register the mime type if in Jupyter Notebook environment and previously unregistered\n",
+ " if (root.Jupyter !== undefined) {\n",
+ " var events = require('base/js/events');\n",
+ " var OutputArea = require('notebook/js/outputarea').OutputArea;\n",
+ "\n",
+ " if (OutputArea.prototype.mime_types().indexOf(EXEC_MIME_TYPE) == -1) {\n",
+ " register_renderer(events, OutputArea);\n",
+ " }\n",
+ " }\n",
+ "\n",
+ " \n",
+ " if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n",
+ " root._bokeh_timeout = Date.now() + 5000;\n",
+ " root._bokeh_failed_load = false;\n",
+ " }\n",
+ "\n",
+ " var NB_LOAD_WARNING = {'data': {'text/html':\n",
+ " \"
\\n\"+\n",
+ " \"
\\n\"+\n",
+ " \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n",
+ " \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n",
+ " \"
\\n\"+\n",
+ " \"
\\n\"+\n",
+ " \"
re-rerun `output_notebook()` to attempt to load from CDN again, or
\"}};\n",
+ "\n",
+ " function display_loaded() {\n",
+ " var el = document.getElementById(null);\n",
+ " if (el != null) {\n",
+ " el.textContent = \"BokehJS is loading...\";\n",
+ " }\n",
+ " if (root.Bokeh !== undefined) {\n",
+ " if (el != null) {\n",
+ " el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n",
+ " }\n",
+ " } else if (Date.now() < root._bokeh_timeout) {\n",
+ " setTimeout(display_loaded, 100)\n",
+ " }\n",
+ " }\n",
+ "\n",
+ "\n",
+ " function run_callbacks() {\n",
+ " try {\n",
+ " root._bokeh_onload_callbacks.forEach(function(callback) {\n",
+ " if (callback != null)\n",
+ " callback();\n",
+ " });\n",
+ " } finally {\n",
+ " delete root._bokeh_onload_callbacks\n",
+ " }\n",
+ " console.debug(\"Bokeh: all callbacks have finished\");\n",
+ " }\n",
+ "\n",
+ " function load_libs(css_urls, js_urls, callback) {\n",
+ " if (css_urls == null) css_urls = [];\n",
+ " if (js_urls == null) js_urls = [];\n",
+ "\n",
+ " root._bokeh_onload_callbacks.push(callback);\n",
+ " if (root._bokeh_is_loading > 0) {\n",
+ " console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n",
+ " return null;\n",
+ " }\n",
+ " if (js_urls == null || js_urls.length === 0) {\n",
+ " run_callbacks();\n",
+ " return null;\n",
+ " }\n",
+ " console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n",
+ " root._bokeh_is_loading = css_urls.length + js_urls.length;\n",
+ "\n",
+ " function on_load() {\n",
+ " root._bokeh_is_loading--;\n",
+ " if (root._bokeh_is_loading === 0) {\n",
+ " console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n",
+ " run_callbacks()\n",
+ " }\n",
+ " }\n",
+ "\n",
+ " function on_error() {\n",
+ " console.error(\"failed to load \" + url);\n",
+ " }\n",
+ "\n",
+ " for (var i = 0; i < css_urls.length; i++) {\n",
+ " var url = css_urls[i];\n",
+ " const element = document.createElement(\"link\");\n",
+ " element.onload = on_load;\n",
+ " element.onerror = on_error;\n",
+ " element.rel = \"stylesheet\";\n",
+ " element.type = \"text/css\";\n",
+ " element.href = url;\n",
+ " console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n",
+ " document.body.appendChild(element);\n",
+ " }\n",
+ "\n",
+ " for (var i = 0; i < js_urls.length; i++) {\n",
+ " var url = js_urls[i];\n",
+ " var element = document.createElement('script');\n",
+ " element.onload = on_load;\n",
+ " element.onerror = on_error;\n",
+ " element.async = false;\n",
+ " element.src = url;\n",
+ " console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n",
+ " document.head.appendChild(element);\n",
+ " }\n",
+ " };\n",
+ "\n",
+ " function inject_raw_css(css) {\n",
+ " const element = document.createElement(\"style\");\n",
+ " element.appendChild(document.createTextNode(css));\n",
+ " document.body.appendChild(element);\n",
+ " }\n",
+ "\n",
+ " \n",
+ " var js_urls = [\"https://cdn.pydata.org/bokeh/release/bokeh-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-widgets-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-tables-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-gl-1.4.0.min.js\"];\n",
+ " var css_urls = [];\n",
+ " \n",
+ "\n",
+ " var inline_js = [\n",
+ " function(Bokeh) {\n",
+ " Bokeh.set_log_level(\"info\");\n",
+ " },\n",
+ " function(Bokeh) {\n",
+ " \n",
+ " \n",
+ " }\n",
+ " ];\n",
+ "\n",
+ " function run_inline_js() {\n",
+ " \n",
+ " if (root.Bokeh !== undefined || force === true) {\n",
+ " \n",
+ " for (var i = 0; i < inline_js.length; i++) {\n",
+ " inline_js[i].call(root, root.Bokeh);\n",
+ " }\n",
+ " } else if (Date.now() < root._bokeh_timeout) {\n",
+ " setTimeout(run_inline_js, 100);\n",
+ " } else if (!root._bokeh_failed_load) {\n",
+ " console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n",
+ " root._bokeh_failed_load = true;\n",
+ " } else if (force !== true) {\n",
+ " var cell = $(document.getElementById(null)).parents('.cell').data().cell;\n",
+ " cell.output_area.append_execute_result(NB_LOAD_WARNING)\n",
+ " }\n",
+ "\n",
+ " }\n",
+ "\n",
+ " if (root._bokeh_is_loading === 0) {\n",
+ " console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n",
+ " run_inline_js();\n",
+ " } else {\n",
+ " load_libs(css_urls, js_urls, function() {\n",
+ " console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n",
+ " run_inline_js();\n",
+ " });\n",
+ " }\n",
+ "}(window));"
+ ],
+ "application/vnd.bokehjs_load.v0+json": "\n(function(root) {\n function now() {\n return new Date();\n }\n\n var force = true;\n\n if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n root._bokeh_onload_callbacks = [];\n root._bokeh_is_loading = undefined;\n }\n\n \n\n \n if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n root._bokeh_timeout = Date.now() + 5000;\n root._bokeh_failed_load = false;\n }\n\n var NB_LOAD_WARNING = {'data': {'text/html':\n \"
\\n\"+\n \"
\\n\"+\n \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n \"
\\n\"+\n \"
\\n\"+\n \"
re-rerun `output_notebook()` to attempt to load from CDN again, or
\"}};\n\n function display_loaded() {\n var el = document.getElementById(null);\n if (el != null) {\n el.textContent = \"BokehJS is loading...\";\n }\n if (root.Bokeh !== undefined) {\n if (el != null) {\n el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n }\n } else if (Date.now() < root._bokeh_timeout) {\n setTimeout(display_loaded, 100)\n }\n }\n\n\n function run_callbacks() {\n try {\n root._bokeh_onload_callbacks.forEach(function(callback) {\n if (callback != null)\n callback();\n });\n } finally {\n delete root._bokeh_onload_callbacks\n }\n console.debug(\"Bokeh: all callbacks have finished\");\n }\n\n function load_libs(css_urls, js_urls, callback) {\n if (css_urls == null) css_urls = [];\n if (js_urls == null) js_urls = [];\n\n root._bokeh_onload_callbacks.push(callback);\n if (root._bokeh_is_loading > 0) {\n console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n return null;\n }\n if (js_urls == null || js_urls.length === 0) {\n run_callbacks();\n return null;\n }\n console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n root._bokeh_is_loading = css_urls.length + js_urls.length;\n\n function on_load() {\n root._bokeh_is_loading--;\n if (root._bokeh_is_loading === 0) {\n console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n run_callbacks()\n }\n }\n\n function on_error() {\n console.error(\"failed to load \" + url);\n }\n\n for (var i = 0; i < css_urls.length; i++) {\n var url = css_urls[i];\n const element = document.createElement(\"link\");\n element.onload = on_load;\n element.onerror = on_error;\n element.rel = \"stylesheet\";\n element.type = \"text/css\";\n element.href = url;\n console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n document.body.appendChild(element);\n }\n\n for (var i = 0; i < js_urls.length; i++) {\n var url = js_urls[i];\n var element = document.createElement('script');\n element.onload = on_load;\n element.onerror = on_error;\n element.async = false;\n element.src = url;\n console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n document.head.appendChild(element);\n }\n };\n\n function inject_raw_css(css) {\n const element = document.createElement(\"style\");\n element.appendChild(document.createTextNode(css));\n document.body.appendChild(element);\n }\n\n \n var js_urls = [\"https://cdn.pydata.org/bokeh/release/bokeh-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-widgets-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-tables-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-gl-1.4.0.min.js\"];\n var css_urls = [];\n \n\n var inline_js = [\n function(Bokeh) {\n Bokeh.set_log_level(\"info\");\n },\n function(Bokeh) {\n \n \n }\n ];\n\n function run_inline_js() {\n \n if (root.Bokeh !== undefined || force === true) {\n \n for (var i = 0; i < inline_js.length; i++) {\n inline_js[i].call(root, root.Bokeh);\n }\n } else if (Date.now() < root._bokeh_timeout) {\n setTimeout(run_inline_js, 100);\n } else if (!root._bokeh_failed_load) {\n console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n root._bokeh_failed_load = true;\n } else if (force !== true) {\n var cell = $(document.getElementById(null)).parents('.cell').data().cell;\n cell.output_area.append_execute_result(NB_LOAD_WARNING)\n }\n\n }\n\n if (root._bokeh_is_loading === 0) {\n console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n run_inline_js();\n } else {\n load_libs(css_urls, js_urls, function() {\n console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n run_inline_js();\n });\n }\n}(window));"
+ },
+ "metadata": {
+ "tags": []
+ },
+ "output_type": "display_data"
+ },
+ {
+ "data": {
+ "text/html": [
+ "\n",
+ "\n",
+ "\n",
+ "\n",
+ "\n",
+ "\n",
+ " \n"
+ ]
+ },
+ "metadata": {
+ "tags": []
+ },
+ "output_type": "display_data"
+ },
+ {
+ "data": {
+ "application/javascript": [
+ "(function(root) {\n",
+ " function embed_document(root) {\n",
+ " \n",
+ " var docs_json = {\"c09fc8f6-d5fa-4ba4-ae4a-0ce276d613ba\":{\"roots\":{\"references\":[{\"attributes\":{\"above\":[{\"id\":\"1180\",\"type\":\"CategoricalAxis\"}],\"center\":[{\"id\":\"1183\",\"type\":\"Grid\"},{\"id\":\"1187\",\"type\":\"Grid\"}],\"left\":[{\"id\":\"1184\",\"type\":\"CategoricalAxis\"}],\"min_border_right\":300,\"plot_width\":1200,\"renderers\":[{\"id\":\"1197\",\"type\":\"GlyphRenderer\"}],\"title\":{\"id\":\"1170\",\"type\":\"Title\"},\"toolbar\":{\"id\":\"1190\",\"type\":\"Toolbar\"},\"toolbar_location\":\"below\",\"x_range\":{\"id\":\"1172\",\"type\":\"FactorRange\"},\"x_scale\":{\"id\":\"1176\",\"type\":\"CategoricalScale\"},\"y_range\":{\"id\":\"1174\",\"type\":\"FactorRange\"},\"y_scale\":{\"id\":\"1178\",\"type\":\"CategoricalScale\"}},\"id\":\"1169\",\"subtype\":\"Figure\",\"type\":\"Plot\"},{\"attributes\":{\"callback\":null,\"tooltips\":[[\"pair\",\"@embeddings_1 ||| @embeddings_2\"],[\"sim\",\"@sim\"]]},\"id\":\"1189\",\"type\":\"HoverTool\"},{\"attributes\":{},\"id\":\"1218\",\"type\":\"Selection\"},{\"attributes\":{},\"id\":\"1216\",\"type\":\"CategoricalTickFormatter\"},{\"attributes\":{},\"id\":\"1219\",\"type\":\"UnionRenderers\"},{\"attributes\":{\"ticker\":{\"id\":\"1181\",\"type\":\"CategoricalTicker\"}},\"id\":\"1183\",\"type\":\"Grid\"},{\"attributes\":{\"axis_line_color\":{\"value\":null},\"formatter\":{\"id\":\"1216\",\"type\":\"CategoricalTickFormatter\"},\"major_label_orientation\":0.7853981633974483,\"major_label_standoff\":16,\"major_label_text_font_size\":{\"value\":\"12pt\"},\"major_tick_line_color\":{\"value\":null},\"ticker\":{\"id\":\"1181\",\"type\":\"CategoricalTicker\"}},\"id\":\"1180\",\"type\":\"CategoricalAxis\"},{\"attributes\":{},\"id\":\"1178\",\"type\":\"CategoricalScale\"},{\"attributes\":{},\"id\":\"1176\",\"type\":\"CategoricalScale\"},{\"attributes\":{\"callback\":null,\"factors\":[\"\\u041c\\u043d\\u0435 \\u043d\\u0440\\u0430\\u0432\\u0438\\u0442\\u0441\\u044f \\u043f\\u043e\\u0434\\u043e\\u043b\\u0433\\u0443 \\u0433\\u0443\\u043b\\u044f\\u0442\\u044c \\u043f\\u043e \\u043f\\u043b\\u044f\\u0436\\u0443 \\u0441\\u043e \\u0441\\u0432\\u043e\\u0435\\u0439 \\u0441\\u043e\\u0431\\u0430\\u043a\\u043e\\u0439.\",\"\\u041c\\u0438\\u043b\\u044b\\u0435 \\u0449\\u0435\\u043d\\u043a\\u0438.\",\"\\u0441\\u043e\\u0431\\u0430\\u043a\\u0430\"]},\"id\":\"1174\",\"type\":\"FactorRange\"},{\"attributes\":{},\"id\":\"1185\",\"type\":\"CategoricalTicker\"},{\"attributes\":{\"axis_line_color\":{\"value\":null},\"formatter\":{\"id\":\"1214\",\"type\":\"CategoricalTickFormatter\"},\"major_label_standoff\":16,\"major_label_text_font_size\":{\"value\":\"12pt\"},\"major_tick_line_color\":{\"value\":null},\"ticker\":{\"id\":\"1185\",\"type\":\"CategoricalTicker\"}},\"id\":\"1184\",\"type\":\"CategoricalAxis\"},{\"attributes\":{\"fill_color\":{\"field\":\"sim\",\"transform\":{\"id\":\"1168\",\"type\":\"LinearColorMapper\"}},\"height\":{\"units\":\"data\",\"value\":1},\"line_color\":{\"value\":null},\"width\":{\"units\":\"data\",\"value\":1},\"x\":{\"field\":\"embeddings_1\"},\"y\":{\"field\":\"embeddings_2\"}},\"id\":\"1195\",\"type\":\"Rect\"},{\"attributes\":{\"high\":0.8845847249031067,\"low\":0.5800867676734924,\"palette\":[\"#ffffcc\",\"#ffeda0\",\"#fed976\",\"#feb24c\",\"#fd8d3c\",\"#fc4e2a\",\"#e31a1c\",\"#bd0026\",\"#800026\"]},\"id\":\"1168\",\"type\":\"LinearColorMapper\"},{\"attributes\":{},\"id\":\"1181\",\"type\":\"CategoricalTicker\"},{\"attributes\":{\"data_source\":{\"id\":\"1193\",\"type\":\"ColumnDataSource\"},\"glyph\":{\"id\":\"1195\",\"type\":\"Rect\"},\"hover_glyph\":null,\"muted_glyph\":null,\"nonselection_glyph\":{\"id\":\"1196\",\"type\":\"Rect\"},\"selection_glyph\":null,\"view\":{\"id\":\"1198\",\"type\":\"CDSView\"}},\"id\":\"1197\",\"type\":\"GlyphRenderer\"},{\"attributes\":{\"active_drag\":\"auto\",\"active_inspect\":\"auto\",\"active_multi\":null,\"active_scroll\":\"auto\",\"active_tap\":\"auto\",\"tools\":[{\"id\":\"1188\",\"type\":\"SaveTool\"},{\"id\":\"1189\",\"type\":\"HoverTool\"}]},\"id\":\"1190\",\"type\":\"Toolbar\"},{\"attributes\":{\"callback\":null,\"factors\":[\"dog\",\"Puppies are nice.\",\"I enjoy taking long walks along the beach with my dog.\"]},\"id\":\"1172\",\"type\":\"FactorRange\"},{\"attributes\":{\"text\":\"English-Russian Similarity\",\"text_font_size\":{\"value\":\"12pt\"}},\"id\":\"1170\",\"type\":\"Title\"},{\"attributes\":{\"dimension\":1,\"ticker\":{\"id\":\"1185\",\"type\":\"CategoricalTicker\"}},\"id\":\"1187\",\"type\":\"Grid\"},{\"attributes\":{\"callback\":null,\"data\":{\"embeddings_1\":[\"dog\",\"dog\",\"dog\",\"Puppies are nice.\",\"Puppies are nice.\",\"Puppies are nice.\",\"I enjoy taking long walks along the beach with my dog.\",\"I enjoy taking long walks along the beach with my dog.\",\"I enjoy taking long walks along the beach with my dog.\"],\"embeddings_2\":[\"\\u0441\\u043e\\u0431\\u0430\\u043a\\u0430\",\"\\u041c\\u0438\\u043b\\u044b\\u0435 \\u0449\\u0435\\u043d\\u043a\\u0438.\",\"\\u041c\\u043d\\u0435 \\u043d\\u0440\\u0430\\u0432\\u0438\\u0442\\u0441\\u044f \\u043f\\u043e\\u0434\\u043e\\u043b\\u0433\\u0443 \\u0433\\u0443\\u043b\\u044f\\u0442\\u044c \\u043f\\u043e \\u043f\\u043b\\u044f\\u0436\\u0443 \\u0441\\u043e \\u0441\\u0432\\u043e\\u0435\\u0439 \\u0441\\u043e\\u0431\\u0430\\u043a\\u043e\\u0439.\",\"\\u0441\\u043e\\u0431\\u0430\\u043a\\u0430\",\"\\u041c\\u0438\\u043b\\u044b\\u0435 \\u0449\\u0435\\u043d\\u043a\\u0438.\",\"\\u041c\\u043d\\u0435 \\u043d\\u0440\\u0430\\u0432\\u0438\\u0442\\u0441\\u044f \\u043f\\u043e\\u0434\\u043e\\u043b\\u0433\\u0443 \\u0433\\u0443\\u043b\\u044f\\u0442\\u044c \\u043f\\u043e \\u043f\\u043b\\u044f\\u0436\\u0443 \\u0441\\u043e \\u0441\\u0432\\u043e\\u0435\\u0439 \\u0441\\u043e\\u0431\\u0430\\u043a\\u043e\\u0439.\",\"\\u0441\\u043e\\u0431\\u0430\\u043a\\u0430\",\"\\u041c\\u0438\\u043b\\u044b\\u0435 \\u0449\\u0435\\u043d\\u043a\\u0438.\",\"\\u041c\\u043d\\u0435 \\u043d\\u0440\\u0430\\u0432\\u0438\\u0442\\u0441\\u044f \\u043f\\u043e\\u0434\\u043e\\u043b\\u0433\\u0443 \\u0433\\u0443\\u043b\\u044f\\u0442\\u044c \\u043f\\u043e \\u043f\\u043b\\u044f\\u0436\\u0443 \\u0441\\u043e \\u0441\\u0432\\u043e\\u0435\\u0439 \\u0441\\u043e\\u0431\\u0430\\u043a\\u043e\\u0439.\"],\"index\":[0,1,2,3,4,5,6,7,8],\"sim\":{\"__ndarray__\":\"AAAAoIRO7D8AAABgiPvkPwAAAMCko+M/AAAAgDBm5D8AAADAi1DoPwAAAKCr8eI/AAAAIBKQ4j8AAABARq3iPwAAAMAlV+g/\",\"dtype\":\"float64\",\"shape\":[9]}},\"selected\":{\"id\":\"1218\",\"type\":\"Selection\"},\"selection_policy\":{\"id\":\"1219\",\"type\":\"UnionRenderers\"}},\"id\":\"1193\",\"type\":\"ColumnDataSource\"},{\"attributes\":{},\"id\":\"1214\",\"type\":\"CategoricalTickFormatter\"},{\"attributes\":{\"fill_alpha\":{\"value\":0.1},\"fill_color\":{\"value\":\"#1f77b4\"},\"height\":{\"units\":\"data\",\"value\":1},\"line_alpha\":{\"value\":0.1},\"line_color\":{\"value\":\"#1f77b4\"},\"width\":{\"units\":\"data\",\"value\":1},\"x\":{\"field\":\"embeddings_1\"},\"y\":{\"field\":\"embeddings_2\"}},\"id\":\"1196\",\"type\":\"Rect\"},{\"attributes\":{\"source\":{\"id\":\"1193\",\"type\":\"ColumnDataSource\"}},\"id\":\"1198\",\"type\":\"CDSView\"},{\"attributes\":{},\"id\":\"1188\",\"type\":\"SaveTool\"}],\"root_ids\":[\"1169\"]},\"title\":\"Bokeh Application\",\"version\":\"1.4.0\"}};\n",
+ " var render_items = [{\"docid\":\"c09fc8f6-d5fa-4ba4-ae4a-0ce276d613ba\",\"roots\":{\"1169\":\"02b6f17f-c980-4d2c-b9d7-58e2d001b1bf\"}}];\n",
+ " root.Bokeh.embed.embed_items_notebook(docs_json, render_items);\n",
+ "\n",
+ " }\n",
+ " if (root.Bokeh !== undefined) {\n",
+ " embed_document(root);\n",
+ " } else {\n",
+ " var attempts = 0;\n",
+ " var timer = setInterval(function(root) {\n",
+ " if (root.Bokeh !== undefined) {\n",
+ " clearInterval(timer);\n",
+ " embed_document(root);\n",
+ " } else {\n",
+ " attempts++;\n",
+ " if (attempts > 100) {\n",
+ " clearInterval(timer);\n",
+ " console.log(\"Bokeh: ERROR: Unable to run BokehJS code because BokehJS library is missing\");\n",
+ " }\n",
+ " }\n",
+ " }, 10, root)\n",
+ " }\n",
+ "})(window);"
+ ],
+ "application/vnd.bokehjs_exec.v0+json": ""
+ },
+ "metadata": {
+ "application/vnd.bokehjs_exec.v0+json": {
+ "id": "1169"
+ },
+ "tags": []
+ },
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "visualize_similarity(en_result, ru_result, english_sentences, russian_sentences, 'English-Russian Similarity')"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "BJkL6Az0QXNN"
+ },
+ "source": [
+ "### English-Spanish Similarity"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "CH_BXVGhQ0GL"
+ },
+ "outputs": [
+ {
+ "data": {
+ "application/javascript": [
+ "\n",
+ "(function(root) {\n",
+ " function now() {\n",
+ " return new Date();\n",
+ " }\n",
+ "\n",
+ " var force = true;\n",
+ "\n",
+ " if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n",
+ " root._bokeh_onload_callbacks = [];\n",
+ " root._bokeh_is_loading = undefined;\n",
+ " }\n",
+ "\n",
+ " var JS_MIME_TYPE = 'application/javascript';\n",
+ " var HTML_MIME_TYPE = 'text/html';\n",
+ " var EXEC_MIME_TYPE = 'application/vnd.bokehjs_exec.v0+json';\n",
+ " var CLASS_NAME = 'output_bokeh rendered_html';\n",
+ "\n",
+ " /**\n",
+ " * Render data to the DOM node\n",
+ " */\n",
+ " function render(props, node) {\n",
+ " var script = document.createElement(\"script\");\n",
+ " node.appendChild(script);\n",
+ " }\n",
+ "\n",
+ " /**\n",
+ " * Handle when an output is cleared or removed\n",
+ " */\n",
+ " function handleClearOutput(event, handle) {\n",
+ " var cell = handle.cell;\n",
+ "\n",
+ " var id = cell.output_area._bokeh_element_id;\n",
+ " var server_id = cell.output_area._bokeh_server_id;\n",
+ " // Clean up Bokeh references\n",
+ " if (id != null && id in Bokeh.index) {\n",
+ " Bokeh.index[id].model.document.clear();\n",
+ " delete Bokeh.index[id];\n",
+ " }\n",
+ "\n",
+ " if (server_id !== undefined) {\n",
+ " // Clean up Bokeh references\n",
+ " var cmd = \"from bokeh.io.state import curstate; print(curstate().uuid_to_server['\" + server_id + \"'].get_sessions()[0].document.roots[0]._id)\";\n",
+ " cell.notebook.kernel.execute(cmd, {\n",
+ " iopub: {\n",
+ " output: function(msg) {\n",
+ " var id = msg.content.text.trim();\n",
+ " if (id in Bokeh.index) {\n",
+ " Bokeh.index[id].model.document.clear();\n",
+ " delete Bokeh.index[id];\n",
+ " }\n",
+ " }\n",
+ " }\n",
+ " });\n",
+ " // Destroy server and session\n",
+ " var cmd = \"import bokeh.io.notebook as ion; ion.destroy_server('\" + server_id + \"')\";\n",
+ " cell.notebook.kernel.execute(cmd);\n",
+ " }\n",
+ " }\n",
+ "\n",
+ " /**\n",
+ " * Handle when a new output is added\n",
+ " */\n",
+ " function handleAddOutput(event, handle) {\n",
+ " var output_area = handle.output_area;\n",
+ " var output = handle.output;\n",
+ "\n",
+ " // limit handleAddOutput to display_data with EXEC_MIME_TYPE content only\n",
+ " if ((output.output_type != \"display_data\") || (!output.data.hasOwnProperty(EXEC_MIME_TYPE))) {\n",
+ " return\n",
+ " }\n",
+ "\n",
+ " var toinsert = output_area.element.find(\".\" + CLASS_NAME.split(' ')[0]);\n",
+ "\n",
+ " if (output.metadata[EXEC_MIME_TYPE][\"id\"] !== undefined) {\n",
+ " toinsert[toinsert.length - 1].firstChild.textContent = output.data[JS_MIME_TYPE];\n",
+ " // store reference to embed id on output_area\n",
+ " output_area._bokeh_element_id = output.metadata[EXEC_MIME_TYPE][\"id\"];\n",
+ " }\n",
+ " if (output.metadata[EXEC_MIME_TYPE][\"server_id\"] !== undefined) {\n",
+ " var bk_div = document.createElement(\"div\");\n",
+ " bk_div.innerHTML = output.data[HTML_MIME_TYPE];\n",
+ " var script_attrs = bk_div.children[0].attributes;\n",
+ " for (var i = 0; i < script_attrs.length; i++) {\n",
+ " toinsert[toinsert.length - 1].firstChild.setAttribute(script_attrs[i].name, script_attrs[i].value);\n",
+ " }\n",
+ " // store reference to server id on output_area\n",
+ " output_area._bokeh_server_id = output.metadata[EXEC_MIME_TYPE][\"server_id\"];\n",
+ " }\n",
+ " }\n",
+ "\n",
+ " function register_renderer(events, OutputArea) {\n",
+ "\n",
+ " function append_mime(data, metadata, element) {\n",
+ " // create a DOM node to render to\n",
+ " var toinsert = this.create_output_subarea(\n",
+ " metadata,\n",
+ " CLASS_NAME,\n",
+ " EXEC_MIME_TYPE\n",
+ " );\n",
+ " this.keyboard_manager.register_events(toinsert);\n",
+ " // Render to node\n",
+ " var props = {data: data, metadata: metadata[EXEC_MIME_TYPE]};\n",
+ " render(props, toinsert[toinsert.length - 1]);\n",
+ " element.append(toinsert);\n",
+ " return toinsert\n",
+ " }\n",
+ "\n",
+ " /* Handle when an output is cleared or removed */\n",
+ " events.on('clear_output.CodeCell', handleClearOutput);\n",
+ " events.on('delete.Cell', handleClearOutput);\n",
+ "\n",
+ " /* Handle when a new output is added */\n",
+ " events.on('output_added.OutputArea', handleAddOutput);\n",
+ "\n",
+ " /**\n",
+ " * Register the mime type and append_mime function with output_area\n",
+ " */\n",
+ " OutputArea.prototype.register_mime_type(EXEC_MIME_TYPE, append_mime, {\n",
+ " /* Is output safe? */\n",
+ " safe: true,\n",
+ " /* Index of renderer in `output_area.display_order` */\n",
+ " index: 0\n",
+ " });\n",
+ " }\n",
+ "\n",
+ " // register the mime type if in Jupyter Notebook environment and previously unregistered\n",
+ " if (root.Jupyter !== undefined) {\n",
+ " var events = require('base/js/events');\n",
+ " var OutputArea = require('notebook/js/outputarea').OutputArea;\n",
+ "\n",
+ " if (OutputArea.prototype.mime_types().indexOf(EXEC_MIME_TYPE) == -1) {\n",
+ " register_renderer(events, OutputArea);\n",
+ " }\n",
+ " }\n",
+ "\n",
+ " \n",
+ " if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n",
+ " root._bokeh_timeout = Date.now() + 5000;\n",
+ " root._bokeh_failed_load = false;\n",
+ " }\n",
+ "\n",
+ " var NB_LOAD_WARNING = {'data': {'text/html':\n",
+ " \"
\\n\"+\n",
+ " \"
\\n\"+\n",
+ " \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n",
+ " \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n",
+ " \"
\\n\"+\n",
+ " \"
\\n\"+\n",
+ " \"
re-rerun `output_notebook()` to attempt to load from CDN again, or
\"}};\n",
+ "\n",
+ " function display_loaded() {\n",
+ " var el = document.getElementById(null);\n",
+ " if (el != null) {\n",
+ " el.textContent = \"BokehJS is loading...\";\n",
+ " }\n",
+ " if (root.Bokeh !== undefined) {\n",
+ " if (el != null) {\n",
+ " el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n",
+ " }\n",
+ " } else if (Date.now() < root._bokeh_timeout) {\n",
+ " setTimeout(display_loaded, 100)\n",
+ " }\n",
+ " }\n",
+ "\n",
+ "\n",
+ " function run_callbacks() {\n",
+ " try {\n",
+ " root._bokeh_onload_callbacks.forEach(function(callback) {\n",
+ " if (callback != null)\n",
+ " callback();\n",
+ " });\n",
+ " } finally {\n",
+ " delete root._bokeh_onload_callbacks\n",
+ " }\n",
+ " console.debug(\"Bokeh: all callbacks have finished\");\n",
+ " }\n",
+ "\n",
+ " function load_libs(css_urls, js_urls, callback) {\n",
+ " if (css_urls == null) css_urls = [];\n",
+ " if (js_urls == null) js_urls = [];\n",
+ "\n",
+ " root._bokeh_onload_callbacks.push(callback);\n",
+ " if (root._bokeh_is_loading > 0) {\n",
+ " console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n",
+ " return null;\n",
+ " }\n",
+ " if (js_urls == null || js_urls.length === 0) {\n",
+ " run_callbacks();\n",
+ " return null;\n",
+ " }\n",
+ " console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n",
+ " root._bokeh_is_loading = css_urls.length + js_urls.length;\n",
+ "\n",
+ " function on_load() {\n",
+ " root._bokeh_is_loading--;\n",
+ " if (root._bokeh_is_loading === 0) {\n",
+ " console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n",
+ " run_callbacks()\n",
+ " }\n",
+ " }\n",
+ "\n",
+ " function on_error() {\n",
+ " console.error(\"failed to load \" + url);\n",
+ " }\n",
+ "\n",
+ " for (var i = 0; i < css_urls.length; i++) {\n",
+ " var url = css_urls[i];\n",
+ " const element = document.createElement(\"link\");\n",
+ " element.onload = on_load;\n",
+ " element.onerror = on_error;\n",
+ " element.rel = \"stylesheet\";\n",
+ " element.type = \"text/css\";\n",
+ " element.href = url;\n",
+ " console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n",
+ " document.body.appendChild(element);\n",
+ " }\n",
+ "\n",
+ " for (var i = 0; i < js_urls.length; i++) {\n",
+ " var url = js_urls[i];\n",
+ " var element = document.createElement('script');\n",
+ " element.onload = on_load;\n",
+ " element.onerror = on_error;\n",
+ " element.async = false;\n",
+ " element.src = url;\n",
+ " console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n",
+ " document.head.appendChild(element);\n",
+ " }\n",
+ " };\n",
+ "\n",
+ " function inject_raw_css(css) {\n",
+ " const element = document.createElement(\"style\");\n",
+ " element.appendChild(document.createTextNode(css));\n",
+ " document.body.appendChild(element);\n",
+ " }\n",
+ "\n",
+ " \n",
+ " var js_urls = [\"https://cdn.pydata.org/bokeh/release/bokeh-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-widgets-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-tables-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-gl-1.4.0.min.js\"];\n",
+ " var css_urls = [];\n",
+ " \n",
+ "\n",
+ " var inline_js = [\n",
+ " function(Bokeh) {\n",
+ " Bokeh.set_log_level(\"info\");\n",
+ " },\n",
+ " function(Bokeh) {\n",
+ " \n",
+ " \n",
+ " }\n",
+ " ];\n",
+ "\n",
+ " function run_inline_js() {\n",
+ " \n",
+ " if (root.Bokeh !== undefined || force === true) {\n",
+ " \n",
+ " for (var i = 0; i < inline_js.length; i++) {\n",
+ " inline_js[i].call(root, root.Bokeh);\n",
+ " }\n",
+ " } else if (Date.now() < root._bokeh_timeout) {\n",
+ " setTimeout(run_inline_js, 100);\n",
+ " } else if (!root._bokeh_failed_load) {\n",
+ " console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n",
+ " root._bokeh_failed_load = true;\n",
+ " } else if (force !== true) {\n",
+ " var cell = $(document.getElementById(null)).parents('.cell').data().cell;\n",
+ " cell.output_area.append_execute_result(NB_LOAD_WARNING)\n",
+ " }\n",
+ "\n",
+ " }\n",
+ "\n",
+ " if (root._bokeh_is_loading === 0) {\n",
+ " console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n",
+ " run_inline_js();\n",
+ " } else {\n",
+ " load_libs(css_urls, js_urls, function() {\n",
+ " console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n",
+ " run_inline_js();\n",
+ " });\n",
+ " }\n",
+ "}(window));"
+ ],
+ "application/vnd.bokehjs_load.v0+json": "\n(function(root) {\n function now() {\n return new Date();\n }\n\n var force = true;\n\n if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n root._bokeh_onload_callbacks = [];\n root._bokeh_is_loading = undefined;\n }\n\n \n\n \n if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n root._bokeh_timeout = Date.now() + 5000;\n root._bokeh_failed_load = false;\n }\n\n var NB_LOAD_WARNING = {'data': {'text/html':\n \"
\\n\"+\n \"
\\n\"+\n \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n \"
\\n\"+\n \"
\\n\"+\n \"
re-rerun `output_notebook()` to attempt to load from CDN again, or
\"}};\n\n function display_loaded() {\n var el = document.getElementById(null);\n if (el != null) {\n el.textContent = \"BokehJS is loading...\";\n }\n if (root.Bokeh !== undefined) {\n if (el != null) {\n el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n }\n } else if (Date.now() < root._bokeh_timeout) {\n setTimeout(display_loaded, 100)\n }\n }\n\n\n function run_callbacks() {\n try {\n root._bokeh_onload_callbacks.forEach(function(callback) {\n if (callback != null)\n callback();\n });\n } finally {\n delete root._bokeh_onload_callbacks\n }\n console.debug(\"Bokeh: all callbacks have finished\");\n }\n\n function load_libs(css_urls, js_urls, callback) {\n if (css_urls == null) css_urls = [];\n if (js_urls == null) js_urls = [];\n\n root._bokeh_onload_callbacks.push(callback);\n if (root._bokeh_is_loading > 0) {\n console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n return null;\n }\n if (js_urls == null || js_urls.length === 0) {\n run_callbacks();\n return null;\n }\n console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n root._bokeh_is_loading = css_urls.length + js_urls.length;\n\n function on_load() {\n root._bokeh_is_loading--;\n if (root._bokeh_is_loading === 0) {\n console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n run_callbacks()\n }\n }\n\n function on_error() {\n console.error(\"failed to load \" + url);\n }\n\n for (var i = 0; i < css_urls.length; i++) {\n var url = css_urls[i];\n const element = document.createElement(\"link\");\n element.onload = on_load;\n element.onerror = on_error;\n element.rel = \"stylesheet\";\n element.type = \"text/css\";\n element.href = url;\n console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n document.body.appendChild(element);\n }\n\n for (var i = 0; i < js_urls.length; i++) {\n var url = js_urls[i];\n var element = document.createElement('script');\n element.onload = on_load;\n element.onerror = on_error;\n element.async = false;\n element.src = url;\n console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n document.head.appendChild(element);\n }\n };\n\n function inject_raw_css(css) {\n const element = document.createElement(\"style\");\n element.appendChild(document.createTextNode(css));\n document.body.appendChild(element);\n }\n\n \n var js_urls = [\"https://cdn.pydata.org/bokeh/release/bokeh-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-widgets-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-tables-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-gl-1.4.0.min.js\"];\n var css_urls = [];\n \n\n var inline_js = [\n function(Bokeh) {\n Bokeh.set_log_level(\"info\");\n },\n function(Bokeh) {\n \n \n }\n ];\n\n function run_inline_js() {\n \n if (root.Bokeh !== undefined || force === true) {\n \n for (var i = 0; i < inline_js.length; i++) {\n inline_js[i].call(root, root.Bokeh);\n }\n } else if (Date.now() < root._bokeh_timeout) {\n setTimeout(run_inline_js, 100);\n } else if (!root._bokeh_failed_load) {\n console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n root._bokeh_failed_load = true;\n } else if (force !== true) {\n var cell = $(document.getElementById(null)).parents('.cell').data().cell;\n cell.output_area.append_execute_result(NB_LOAD_WARNING)\n }\n\n }\n\n if (root._bokeh_is_loading === 0) {\n console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n run_inline_js();\n } else {\n load_libs(css_urls, js_urls, function() {\n console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n run_inline_js();\n });\n }\n}(window));"
+ },
+ "metadata": {
+ "tags": []
+ },
+ "output_type": "display_data"
+ },
+ {
+ "data": {
+ "text/html": [
+ "\n",
+ "\n",
+ "\n",
+ "\n",
+ "\n",
+ "\n",
+ " \n"
+ ]
+ },
+ "metadata": {
+ "tags": []
+ },
+ "output_type": "display_data"
+ },
+ {
+ "data": {
+ "application/javascript": [
+ "(function(root) {\n",
+ " function embed_document(root) {\n",
+ " \n",
+ " var docs_json = {\"bb848e12-e360-4876-aa19-21896caab34d\":{\"roots\":{\"references\":[{\"attributes\":{\"above\":[{\"id\":\"1274\",\"type\":\"CategoricalAxis\"}],\"center\":[{\"id\":\"1277\",\"type\":\"Grid\"},{\"id\":\"1281\",\"type\":\"Grid\"}],\"left\":[{\"id\":\"1278\",\"type\":\"CategoricalAxis\"}],\"min_border_right\":300,\"plot_width\":1200,\"renderers\":[{\"id\":\"1291\",\"type\":\"GlyphRenderer\"}],\"title\":{\"id\":\"1264\",\"type\":\"Title\"},\"toolbar\":{\"id\":\"1284\",\"type\":\"Toolbar\"},\"toolbar_location\":\"below\",\"x_range\":{\"id\":\"1266\",\"type\":\"FactorRange\"},\"x_scale\":{\"id\":\"1270\",\"type\":\"CategoricalScale\"},\"y_range\":{\"id\":\"1268\",\"type\":\"FactorRange\"},\"y_scale\":{\"id\":\"1272\",\"type\":\"CategoricalScale\"}},\"id\":\"1263\",\"subtype\":\"Figure\",\"type\":\"Plot\"},{\"attributes\":{},\"id\":\"1272\",\"type\":\"CategoricalScale\"},{\"attributes\":{},\"id\":\"1270\",\"type\":\"CategoricalScale\"},{\"attributes\":{\"callback\":null,\"factors\":[\"Disfruto de dar largos paseos por la playa con mi perro.\",\"Los cachorros son agradables.\",\"perro\"]},\"id\":\"1268\",\"type\":\"FactorRange\"},{\"attributes\":{\"callback\":null,\"factors\":[\"dog\",\"Puppies are nice.\",\"I enjoy taking long walks along the beach with my dog.\"]},\"id\":\"1266\",\"type\":\"FactorRange\"},{\"attributes\":{\"fill_color\":{\"field\":\"sim\",\"transform\":{\"id\":\"1262\",\"type\":\"LinearColorMapper\"}},\"height\":{\"units\":\"data\",\"value\":1},\"line_color\":{\"value\":null},\"width\":{\"units\":\"data\",\"value\":1},\"x\":{\"field\":\"embeddings_1\"},\"y\":{\"field\":\"embeddings_2\"}},\"id\":\"1289\",\"type\":\"Rect\"},{\"attributes\":{\"callback\":null,\"data\":{\"embeddings_1\":[\"dog\",\"dog\",\"dog\",\"Puppies are nice.\",\"Puppies are nice.\",\"Puppies are nice.\",\"I enjoy taking long walks along the beach with my dog.\",\"I enjoy taking long walks along the beach with my dog.\",\"I enjoy taking long walks along the beach with my dog.\"],\"embeddings_2\":[\"perro\",\"Los cachorros son agradables.\",\"Disfruto de dar largos paseos por la playa con mi perro.\",\"perro\",\"Los cachorros son agradables.\",\"Disfruto de dar largos paseos por la playa con mi perro.\",\"perro\",\"Los cachorros son agradables.\",\"Disfruto de dar largos paseos por la playa con mi perro.\"],\"index\":[0,1,2,3,4,5,6,7,8],\"sim\":{\"__ndarray__\":\"AAAA4AIT7T8AAAAAcHfkPwAAAKAngeI/AAAAQNoA5D8AAADAvfvoPwAAAGCGFeI/AAAAgMNr4j8AAAAArbPjPwAAAGCJ1eo/\",\"dtype\":\"float64\",\"shape\":[9]}},\"selected\":{\"id\":\"1319\",\"type\":\"Selection\"},\"selection_policy\":{\"id\":\"1320\",\"type\":\"UnionRenderers\"}},\"id\":\"1287\",\"type\":\"ColumnDataSource\"},{\"attributes\":{\"high\":0.9085707068443298,\"low\":0.5651275515556335,\"palette\":[\"#ffffcc\",\"#ffeda0\",\"#fed976\",\"#feb24c\",\"#fd8d3c\",\"#fc4e2a\",\"#e31a1c\",\"#bd0026\",\"#800026\"]},\"id\":\"1262\",\"type\":\"LinearColorMapper\"},{\"attributes\":{\"source\":{\"id\":\"1287\",\"type\":\"ColumnDataSource\"}},\"id\":\"1292\",\"type\":\"CDSView\"},{\"attributes\":{\"text\":\"English-Spanish Similarity\",\"text_font_size\":{\"value\":\"12pt\"}},\"id\":\"1264\",\"type\":\"Title\"},{\"attributes\":{\"callback\":null,\"tooltips\":[[\"pair\",\"@embeddings_1 ||| @embeddings_2\"],[\"sim\",\"@sim\"]]},\"id\":\"1283\",\"type\":\"HoverTool\"},{\"attributes\":{\"fill_alpha\":{\"value\":0.1},\"fill_color\":{\"value\":\"#1f77b4\"},\"height\":{\"units\":\"data\",\"value\":1},\"line_alpha\":{\"value\":0.1},\"line_color\":{\"value\":\"#1f77b4\"},\"width\":{\"units\":\"data\",\"value\":1},\"x\":{\"field\":\"embeddings_1\"},\"y\":{\"field\":\"embeddings_2\"}},\"id\":\"1290\",\"type\":\"Rect\"},{\"attributes\":{\"ticker\":{\"id\":\"1275\",\"type\":\"CategoricalTicker\"}},\"id\":\"1277\",\"type\":\"Grid\"},{\"attributes\":{},\"id\":\"1282\",\"type\":\"SaveTool\"},{\"attributes\":{\"data_source\":{\"id\":\"1287\",\"type\":\"ColumnDataSource\"},\"glyph\":{\"id\":\"1289\",\"type\":\"Rect\"},\"hover_glyph\":null,\"muted_glyph\":null,\"nonselection_glyph\":{\"id\":\"1290\",\"type\":\"Rect\"},\"selection_glyph\":null,\"view\":{\"id\":\"1292\",\"type\":\"CDSView\"}},\"id\":\"1291\",\"type\":\"GlyphRenderer\"},{\"attributes\":{\"dimension\":1,\"ticker\":{\"id\":\"1279\",\"type\":\"CategoricalTicker\"}},\"id\":\"1281\",\"type\":\"Grid\"},{\"attributes\":{},\"id\":\"1315\",\"type\":\"CategoricalTickFormatter\"},{\"attributes\":{\"active_drag\":\"auto\",\"active_inspect\":\"auto\",\"active_multi\":null,\"active_scroll\":\"auto\",\"active_tap\":\"auto\",\"tools\":[{\"id\":\"1282\",\"type\":\"SaveTool\"},{\"id\":\"1283\",\"type\":\"HoverTool\"}]},\"id\":\"1284\",\"type\":\"Toolbar\"},{\"attributes\":{},\"id\":\"1320\",\"type\":\"UnionRenderers\"},{\"attributes\":{},\"id\":\"1279\",\"type\":\"CategoricalTicker\"},{\"attributes\":{\"axis_line_color\":{\"value\":null},\"formatter\":{\"id\":\"1315\",\"type\":\"CategoricalTickFormatter\"},\"major_label_standoff\":16,\"major_label_text_font_size\":{\"value\":\"12pt\"},\"major_tick_line_color\":{\"value\":null},\"ticker\":{\"id\":\"1279\",\"type\":\"CategoricalTicker\"}},\"id\":\"1278\",\"type\":\"CategoricalAxis\"},{\"attributes\":{},\"id\":\"1319\",\"type\":\"Selection\"},{\"attributes\":{},\"id\":\"1317\",\"type\":\"CategoricalTickFormatter\"},{\"attributes\":{\"axis_line_color\":{\"value\":null},\"formatter\":{\"id\":\"1317\",\"type\":\"CategoricalTickFormatter\"},\"major_label_orientation\":0.7853981633974483,\"major_label_standoff\":16,\"major_label_text_font_size\":{\"value\":\"12pt\"},\"major_tick_line_color\":{\"value\":null},\"ticker\":{\"id\":\"1275\",\"type\":\"CategoricalTicker\"}},\"id\":\"1274\",\"type\":\"CategoricalAxis\"},{\"attributes\":{},\"id\":\"1275\",\"type\":\"CategoricalTicker\"}],\"root_ids\":[\"1263\"]},\"title\":\"Bokeh Application\",\"version\":\"1.4.0\"}};\n",
+ " var render_items = [{\"docid\":\"bb848e12-e360-4876-aa19-21896caab34d\",\"roots\":{\"1263\":\"81e993c9-fc6b-4169-8c6b-a0101097b959\"}}];\n",
+ " root.Bokeh.embed.embed_items_notebook(docs_json, render_items);\n",
+ "\n",
+ " }\n",
+ " if (root.Bokeh !== undefined) {\n",
+ " embed_document(root);\n",
+ " } else {\n",
+ " var attempts = 0;\n",
+ " var timer = setInterval(function(root) {\n",
+ " if (root.Bokeh !== undefined) {\n",
+ " clearInterval(timer);\n",
+ " embed_document(root);\n",
+ " } else {\n",
+ " attempts++;\n",
+ " if (attempts > 100) {\n",
+ " clearInterval(timer);\n",
+ " console.log(\"Bokeh: ERROR: Unable to run BokehJS code because BokehJS library is missing\");\n",
+ " }\n",
+ " }\n",
+ " }, 10, root)\n",
+ " }\n",
+ "})(window);"
+ ],
+ "application/vnd.bokehjs_exec.v0+json": ""
+ },
+ "metadata": {
+ "application/vnd.bokehjs_exec.v0+json": {
+ "id": "1263"
+ },
+ "tags": []
+ },
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "visualize_similarity(en_result, es_result, english_sentences, spanish_sentences, 'English-Spanish Similarity')"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "imn28LCiQO7d"
+ },
+ "source": [
+ "### English-Italian Similarity"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "X9uD3DirPIGd"
+ },
+ "outputs": [
+ {
+ "data": {
+ "application/javascript": [
+ "\n",
+ "(function(root) {\n",
+ " function now() {\n",
+ " return new Date();\n",
+ " }\n",
+ "\n",
+ " var force = true;\n",
+ "\n",
+ " if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n",
+ " root._bokeh_onload_callbacks = [];\n",
+ " root._bokeh_is_loading = undefined;\n",
+ " }\n",
+ "\n",
+ " var JS_MIME_TYPE = 'application/javascript';\n",
+ " var HTML_MIME_TYPE = 'text/html';\n",
+ " var EXEC_MIME_TYPE = 'application/vnd.bokehjs_exec.v0+json';\n",
+ " var CLASS_NAME = 'output_bokeh rendered_html';\n",
+ "\n",
+ " /**\n",
+ " * Render data to the DOM node\n",
+ " */\n",
+ " function render(props, node) {\n",
+ " var script = document.createElement(\"script\");\n",
+ " node.appendChild(script);\n",
+ " }\n",
+ "\n",
+ " /**\n",
+ " * Handle when an output is cleared or removed\n",
+ " */\n",
+ " function handleClearOutput(event, handle) {\n",
+ " var cell = handle.cell;\n",
+ "\n",
+ " var id = cell.output_area._bokeh_element_id;\n",
+ " var server_id = cell.output_area._bokeh_server_id;\n",
+ " // Clean up Bokeh references\n",
+ " if (id != null && id in Bokeh.index) {\n",
+ " Bokeh.index[id].model.document.clear();\n",
+ " delete Bokeh.index[id];\n",
+ " }\n",
+ "\n",
+ " if (server_id !== undefined) {\n",
+ " // Clean up Bokeh references\n",
+ " var cmd = \"from bokeh.io.state import curstate; print(curstate().uuid_to_server['\" + server_id + \"'].get_sessions()[0].document.roots[0]._id)\";\n",
+ " cell.notebook.kernel.execute(cmd, {\n",
+ " iopub: {\n",
+ " output: function(msg) {\n",
+ " var id = msg.content.text.trim();\n",
+ " if (id in Bokeh.index) {\n",
+ " Bokeh.index[id].model.document.clear();\n",
+ " delete Bokeh.index[id];\n",
+ " }\n",
+ " }\n",
+ " }\n",
+ " });\n",
+ " // Destroy server and session\n",
+ " var cmd = \"import bokeh.io.notebook as ion; ion.destroy_server('\" + server_id + \"')\";\n",
+ " cell.notebook.kernel.execute(cmd);\n",
+ " }\n",
+ " }\n",
+ "\n",
+ " /**\n",
+ " * Handle when a new output is added\n",
+ " */\n",
+ " function handleAddOutput(event, handle) {\n",
+ " var output_area = handle.output_area;\n",
+ " var output = handle.output;\n",
+ "\n",
+ " // limit handleAddOutput to display_data with EXEC_MIME_TYPE content only\n",
+ " if ((output.output_type != \"display_data\") || (!output.data.hasOwnProperty(EXEC_MIME_TYPE))) {\n",
+ " return\n",
+ " }\n",
+ "\n",
+ " var toinsert = output_area.element.find(\".\" + CLASS_NAME.split(' ')[0]);\n",
+ "\n",
+ " if (output.metadata[EXEC_MIME_TYPE][\"id\"] !== undefined) {\n",
+ " toinsert[toinsert.length - 1].firstChild.textContent = output.data[JS_MIME_TYPE];\n",
+ " // store reference to embed id on output_area\n",
+ " output_area._bokeh_element_id = output.metadata[EXEC_MIME_TYPE][\"id\"];\n",
+ " }\n",
+ " if (output.metadata[EXEC_MIME_TYPE][\"server_id\"] !== undefined) {\n",
+ " var bk_div = document.createElement(\"div\");\n",
+ " bk_div.innerHTML = output.data[HTML_MIME_TYPE];\n",
+ " var script_attrs = bk_div.children[0].attributes;\n",
+ " for (var i = 0; i < script_attrs.length; i++) {\n",
+ " toinsert[toinsert.length - 1].firstChild.setAttribute(script_attrs[i].name, script_attrs[i].value);\n",
+ " }\n",
+ " // store reference to server id on output_area\n",
+ " output_area._bokeh_server_id = output.metadata[EXEC_MIME_TYPE][\"server_id\"];\n",
+ " }\n",
+ " }\n",
+ "\n",
+ " function register_renderer(events, OutputArea) {\n",
+ "\n",
+ " function append_mime(data, metadata, element) {\n",
+ " // create a DOM node to render to\n",
+ " var toinsert = this.create_output_subarea(\n",
+ " metadata,\n",
+ " CLASS_NAME,\n",
+ " EXEC_MIME_TYPE\n",
+ " );\n",
+ " this.keyboard_manager.register_events(toinsert);\n",
+ " // Render to node\n",
+ " var props = {data: data, metadata: metadata[EXEC_MIME_TYPE]};\n",
+ " render(props, toinsert[toinsert.length - 1]);\n",
+ " element.append(toinsert);\n",
+ " return toinsert\n",
+ " }\n",
+ "\n",
+ " /* Handle when an output is cleared or removed */\n",
+ " events.on('clear_output.CodeCell', handleClearOutput);\n",
+ " events.on('delete.Cell', handleClearOutput);\n",
+ "\n",
+ " /* Handle when a new output is added */\n",
+ " events.on('output_added.OutputArea', handleAddOutput);\n",
+ "\n",
+ " /**\n",
+ " * Register the mime type and append_mime function with output_area\n",
+ " */\n",
+ " OutputArea.prototype.register_mime_type(EXEC_MIME_TYPE, append_mime, {\n",
+ " /* Is output safe? */\n",
+ " safe: true,\n",
+ " /* Index of renderer in `output_area.display_order` */\n",
+ " index: 0\n",
+ " });\n",
+ " }\n",
+ "\n",
+ " // register the mime type if in Jupyter Notebook environment and previously unregistered\n",
+ " if (root.Jupyter !== undefined) {\n",
+ " var events = require('base/js/events');\n",
+ " var OutputArea = require('notebook/js/outputarea').OutputArea;\n",
+ "\n",
+ " if (OutputArea.prototype.mime_types().indexOf(EXEC_MIME_TYPE) == -1) {\n",
+ " register_renderer(events, OutputArea);\n",
+ " }\n",
+ " }\n",
+ "\n",
+ " \n",
+ " if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n",
+ " root._bokeh_timeout = Date.now() + 5000;\n",
+ " root._bokeh_failed_load = false;\n",
+ " }\n",
+ "\n",
+ " var NB_LOAD_WARNING = {'data': {'text/html':\n",
+ " \"
\\n\"+\n",
+ " \"
\\n\"+\n",
+ " \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n",
+ " \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n",
+ " \"
\\n\"+\n",
+ " \"
\\n\"+\n",
+ " \"
re-rerun `output_notebook()` to attempt to load from CDN again, or
\"}};\n",
+ "\n",
+ " function display_loaded() {\n",
+ " var el = document.getElementById(null);\n",
+ " if (el != null) {\n",
+ " el.textContent = \"BokehJS is loading...\";\n",
+ " }\n",
+ " if (root.Bokeh !== undefined) {\n",
+ " if (el != null) {\n",
+ " el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n",
+ " }\n",
+ " } else if (Date.now() < root._bokeh_timeout) {\n",
+ " setTimeout(display_loaded, 100)\n",
+ " }\n",
+ " }\n",
+ "\n",
+ "\n",
+ " function run_callbacks() {\n",
+ " try {\n",
+ " root._bokeh_onload_callbacks.forEach(function(callback) {\n",
+ " if (callback != null)\n",
+ " callback();\n",
+ " });\n",
+ " } finally {\n",
+ " delete root._bokeh_onload_callbacks\n",
+ " }\n",
+ " console.debug(\"Bokeh: all callbacks have finished\");\n",
+ " }\n",
+ "\n",
+ " function load_libs(css_urls, js_urls, callback) {\n",
+ " if (css_urls == null) css_urls = [];\n",
+ " if (js_urls == null) js_urls = [];\n",
+ "\n",
+ " root._bokeh_onload_callbacks.push(callback);\n",
+ " if (root._bokeh_is_loading > 0) {\n",
+ " console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n",
+ " return null;\n",
+ " }\n",
+ " if (js_urls == null || js_urls.length === 0) {\n",
+ " run_callbacks();\n",
+ " return null;\n",
+ " }\n",
+ " console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n",
+ " root._bokeh_is_loading = css_urls.length + js_urls.length;\n",
+ "\n",
+ " function on_load() {\n",
+ " root._bokeh_is_loading--;\n",
+ " if (root._bokeh_is_loading === 0) {\n",
+ " console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n",
+ " run_callbacks()\n",
+ " }\n",
+ " }\n",
+ "\n",
+ " function on_error() {\n",
+ " console.error(\"failed to load \" + url);\n",
+ " }\n",
+ "\n",
+ " for (var i = 0; i < css_urls.length; i++) {\n",
+ " var url = css_urls[i];\n",
+ " const element = document.createElement(\"link\");\n",
+ " element.onload = on_load;\n",
+ " element.onerror = on_error;\n",
+ " element.rel = \"stylesheet\";\n",
+ " element.type = \"text/css\";\n",
+ " element.href = url;\n",
+ " console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n",
+ " document.body.appendChild(element);\n",
+ " }\n",
+ "\n",
+ " for (var i = 0; i < js_urls.length; i++) {\n",
+ " var url = js_urls[i];\n",
+ " var element = document.createElement('script');\n",
+ " element.onload = on_load;\n",
+ " element.onerror = on_error;\n",
+ " element.async = false;\n",
+ " element.src = url;\n",
+ " console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n",
+ " document.head.appendChild(element);\n",
+ " }\n",
+ " };\n",
+ "\n",
+ " function inject_raw_css(css) {\n",
+ " const element = document.createElement(\"style\");\n",
+ " element.appendChild(document.createTextNode(css));\n",
+ " document.body.appendChild(element);\n",
+ " }\n",
+ "\n",
+ " \n",
+ " var js_urls = [\"https://cdn.pydata.org/bokeh/release/bokeh-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-widgets-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-tables-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-gl-1.4.0.min.js\"];\n",
+ " var css_urls = [];\n",
+ " \n",
+ "\n",
+ " var inline_js = [\n",
+ " function(Bokeh) {\n",
+ " Bokeh.set_log_level(\"info\");\n",
+ " },\n",
+ " function(Bokeh) {\n",
+ " \n",
+ " \n",
+ " }\n",
+ " ];\n",
+ "\n",
+ " function run_inline_js() {\n",
+ " \n",
+ " if (root.Bokeh !== undefined || force === true) {\n",
+ " \n",
+ " for (var i = 0; i < inline_js.length; i++) {\n",
+ " inline_js[i].call(root, root.Bokeh);\n",
+ " }\n",
+ " } else if (Date.now() < root._bokeh_timeout) {\n",
+ " setTimeout(run_inline_js, 100);\n",
+ " } else if (!root._bokeh_failed_load) {\n",
+ " console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n",
+ " root._bokeh_failed_load = true;\n",
+ " } else if (force !== true) {\n",
+ " var cell = $(document.getElementById(null)).parents('.cell').data().cell;\n",
+ " cell.output_area.append_execute_result(NB_LOAD_WARNING)\n",
+ " }\n",
+ "\n",
+ " }\n",
+ "\n",
+ " if (root._bokeh_is_loading === 0) {\n",
+ " console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n",
+ " run_inline_js();\n",
+ " } else {\n",
+ " load_libs(css_urls, js_urls, function() {\n",
+ " console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n",
+ " run_inline_js();\n",
+ " });\n",
+ " }\n",
+ "}(window));"
+ ],
+ "application/vnd.bokehjs_load.v0+json": "\n(function(root) {\n function now() {\n return new Date();\n }\n\n var force = true;\n\n if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n root._bokeh_onload_callbacks = [];\n root._bokeh_is_loading = undefined;\n }\n\n \n\n \n if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n root._bokeh_timeout = Date.now() + 5000;\n root._bokeh_failed_load = false;\n }\n\n var NB_LOAD_WARNING = {'data': {'text/html':\n \"
\\n\"+\n \"
\\n\"+\n \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n \"
\\n\"+\n \"
\\n\"+\n \"
re-rerun `output_notebook()` to attempt to load from CDN again, or
\"}};\n\n function display_loaded() {\n var el = document.getElementById(null);\n if (el != null) {\n el.textContent = \"BokehJS is loading...\";\n }\n if (root.Bokeh !== undefined) {\n if (el != null) {\n el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n }\n } else if (Date.now() < root._bokeh_timeout) {\n setTimeout(display_loaded, 100)\n }\n }\n\n\n function run_callbacks() {\n try {\n root._bokeh_onload_callbacks.forEach(function(callback) {\n if (callback != null)\n callback();\n });\n } finally {\n delete root._bokeh_onload_callbacks\n }\n console.debug(\"Bokeh: all callbacks have finished\");\n }\n\n function load_libs(css_urls, js_urls, callback) {\n if (css_urls == null) css_urls = [];\n if (js_urls == null) js_urls = [];\n\n root._bokeh_onload_callbacks.push(callback);\n if (root._bokeh_is_loading > 0) {\n console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n return null;\n }\n if (js_urls == null || js_urls.length === 0) {\n run_callbacks();\n return null;\n }\n console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n root._bokeh_is_loading = css_urls.length + js_urls.length;\n\n function on_load() {\n root._bokeh_is_loading--;\n if (root._bokeh_is_loading === 0) {\n console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n run_callbacks()\n }\n }\n\n function on_error() {\n console.error(\"failed to load \" + url);\n }\n\n for (var i = 0; i < css_urls.length; i++) {\n var url = css_urls[i];\n const element = document.createElement(\"link\");\n element.onload = on_load;\n element.onerror = on_error;\n element.rel = \"stylesheet\";\n element.type = \"text/css\";\n element.href = url;\n console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n document.body.appendChild(element);\n }\n\n for (var i = 0; i < js_urls.length; i++) {\n var url = js_urls[i];\n var element = document.createElement('script');\n element.onload = on_load;\n element.onerror = on_error;\n element.async = false;\n element.src = url;\n console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n document.head.appendChild(element);\n }\n };\n\n function inject_raw_css(css) {\n const element = document.createElement(\"style\");\n element.appendChild(document.createTextNode(css));\n document.body.appendChild(element);\n }\n\n \n var js_urls = [\"https://cdn.pydata.org/bokeh/release/bokeh-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-widgets-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-tables-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-gl-1.4.0.min.js\"];\n var css_urls = [];\n \n\n var inline_js = [\n function(Bokeh) {\n Bokeh.set_log_level(\"info\");\n },\n function(Bokeh) {\n \n \n }\n ];\n\n function run_inline_js() {\n \n if (root.Bokeh !== undefined || force === true) {\n \n for (var i = 0; i < inline_js.length; i++) {\n inline_js[i].call(root, root.Bokeh);\n }\n } else if (Date.now() < root._bokeh_timeout) {\n setTimeout(run_inline_js, 100);\n } else if (!root._bokeh_failed_load) {\n console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n root._bokeh_failed_load = true;\n } else if (force !== true) {\n var cell = $(document.getElementById(null)).parents('.cell').data().cell;\n cell.output_area.append_execute_result(NB_LOAD_WARNING)\n }\n\n }\n\n if (root._bokeh_is_loading === 0) {\n console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n run_inline_js();\n } else {\n load_libs(css_urls, js_urls, function() {\n console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n run_inline_js();\n });\n }\n}(window));"
+ },
+ "metadata": {
+ "tags": []
+ },
+ "output_type": "display_data"
+ },
+ {
+ "data": {
+ "text/html": [
+ "\n",
+ "\n",
+ "\n",
+ "\n",
+ "\n",
+ "\n",
+ " \n"
+ ]
+ },
+ "metadata": {
+ "tags": []
+ },
+ "output_type": "display_data"
+ },
+ {
+ "data": {
+ "application/javascript": [
+ "(function(root) {\n",
+ " function embed_document(root) {\n",
+ " \n",
+ " var docs_json = {\"85d1d6b6-c6cd-4e71-b29b-2cc49ada74c5\":{\"roots\":{\"references\":[{\"attributes\":{\"above\":[{\"id\":\"1375\",\"type\":\"CategoricalAxis\"}],\"center\":[{\"id\":\"1378\",\"type\":\"Grid\"},{\"id\":\"1382\",\"type\":\"Grid\"}],\"left\":[{\"id\":\"1379\",\"type\":\"CategoricalAxis\"}],\"min_border_right\":300,\"plot_width\":1200,\"renderers\":[{\"id\":\"1392\",\"type\":\"GlyphRenderer\"}],\"title\":{\"id\":\"1365\",\"type\":\"Title\"},\"toolbar\":{\"id\":\"1385\",\"type\":\"Toolbar\"},\"toolbar_location\":\"below\",\"x_range\":{\"id\":\"1367\",\"type\":\"FactorRange\"},\"x_scale\":{\"id\":\"1371\",\"type\":\"CategoricalScale\"},\"y_range\":{\"id\":\"1369\",\"type\":\"FactorRange\"},\"y_scale\":{\"id\":\"1373\",\"type\":\"CategoricalScale\"}},\"id\":\"1364\",\"subtype\":\"Figure\",\"type\":\"Plot\"},{\"attributes\":{},\"id\":\"1371\",\"type\":\"CategoricalScale\"},{\"attributes\":{},\"id\":\"1427\",\"type\":\"Selection\"},{\"attributes\":{},\"id\":\"1373\",\"type\":\"CategoricalScale\"},{\"attributes\":{},\"id\":\"1423\",\"type\":\"CategoricalTickFormatter\"},{\"attributes\":{\"axis_line_color\":{\"value\":null},\"formatter\":{\"id\":\"1425\",\"type\":\"CategoricalTickFormatter\"},\"major_label_orientation\":0.7853981633974483,\"major_label_standoff\":16,\"major_label_text_font_size\":{\"value\":\"12pt\"},\"major_tick_line_color\":{\"value\":null},\"ticker\":{\"id\":\"1376\",\"type\":\"CategoricalTicker\"}},\"id\":\"1375\",\"type\":\"CategoricalAxis\"},{\"attributes\":{\"callback\":null,\"factors\":[\"Mi piace fare lunghe passeggiate lungo la spiaggia con il mio cane.\",\"I cuccioli sono carini.\",\"cane\"]},\"id\":\"1369\",\"type\":\"FactorRange\"},{\"attributes\":{},\"id\":\"1376\",\"type\":\"CategoricalTicker\"},{\"attributes\":{},\"id\":\"1425\",\"type\":\"CategoricalTickFormatter\"},{\"attributes\":{\"ticker\":{\"id\":\"1376\",\"type\":\"CategoricalTicker\"}},\"id\":\"1378\",\"type\":\"Grid\"},{\"attributes\":{\"axis_line_color\":{\"value\":null},\"formatter\":{\"id\":\"1423\",\"type\":\"CategoricalTickFormatter\"},\"major_label_standoff\":16,\"major_label_text_font_size\":{\"value\":\"12pt\"},\"major_tick_line_color\":{\"value\":null},\"ticker\":{\"id\":\"1380\",\"type\":\"CategoricalTicker\"}},\"id\":\"1379\",\"type\":\"CategoricalAxis\"},{\"attributes\":{},\"id\":\"1380\",\"type\":\"CategoricalTicker\"},{\"attributes\":{\"callback\":null,\"data\":{\"embeddings_1\":[\"dog\",\"dog\",\"dog\",\"Puppies are nice.\",\"Puppies are nice.\",\"Puppies are nice.\",\"I enjoy taking long walks along the beach with my dog.\",\"I enjoy taking long walks along the beach with my dog.\",\"I enjoy taking long walks along the beach with my dog.\"],\"embeddings_2\":[\"cane\",\"I cuccioli sono carini.\",\"Mi piace fare lunghe passeggiate lungo la spiaggia con il mio cane.\",\"cane\",\"I cuccioli sono carini.\",\"Mi piace fare lunghe passeggiate lungo la spiaggia con il mio cane.\",\"cane\",\"I cuccioli sono carini.\",\"Mi piace fare lunghe passeggiate lungo la spiaggia con il mio cane.\"],\"index\":[0,1,2,3,4,5,6,7,8],\"sim\":{\"__ndarray__\":\"AAAAoHcI7T8AAADAU2/jPwAAAIBLIeM/AAAAAO8N5D8AAAAA5GToPwAAAIDhjeI/AAAAQLlt4j8AAAAAEj3iPwAAAGCPHuw/\",\"dtype\":\"float64\",\"shape\":[9]}},\"selected\":{\"id\":\"1427\",\"type\":\"Selection\"},\"selection_policy\":{\"id\":\"1428\",\"type\":\"UnionRenderers\"}},\"id\":\"1388\",\"type\":\"ColumnDataSource\"},{\"attributes\":{\"dimension\":1,\"ticker\":{\"id\":\"1380\",\"type\":\"CategoricalTicker\"}},\"id\":\"1382\",\"type\":\"Grid\"},{\"attributes\":{\"active_drag\":\"auto\",\"active_inspect\":\"auto\",\"active_multi\":null,\"active_scroll\":\"auto\",\"active_tap\":\"auto\",\"tools\":[{\"id\":\"1383\",\"type\":\"SaveTool\"},{\"id\":\"1384\",\"type\":\"HoverTool\"}]},\"id\":\"1385\",\"type\":\"Toolbar\"},{\"attributes\":{},\"id\":\"1383\",\"type\":\"SaveTool\"},{\"attributes\":{\"callback\":null,\"tooltips\":[[\"pair\",\"@embeddings_1 ||| @embeddings_2\"],[\"sim\",\"@sim\"]]},\"id\":\"1384\",\"type\":\"HoverTool\"},{\"attributes\":{\"fill_alpha\":{\"value\":0.1},\"fill_color\":{\"value\":\"#1f77b4\"},\"height\":{\"units\":\"data\",\"value\":1},\"line_alpha\":{\"value\":0.1},\"line_color\":{\"value\":\"#1f77b4\"},\"width\":{\"units\":\"data\",\"value\":1},\"x\":{\"field\":\"embeddings_1\"},\"y\":{\"field\":\"embeddings_2\"}},\"id\":\"1391\",\"type\":\"Rect\"},{\"attributes\":{\"text\":\"English-Italian Similarity\",\"text_font_size\":{\"value\":\"12pt\"}},\"id\":\"1365\",\"type\":\"Title\"},{\"attributes\":{\"fill_color\":{\"field\":\"sim\",\"transform\":{\"id\":\"1363\",\"type\":\"LinearColorMapper\"}},\"height\":{\"units\":\"data\",\"value\":1},\"line_color\":{\"value\":null},\"width\":{\"units\":\"data\",\"value\":1},\"x\":{\"field\":\"embeddings_1\"},\"y\":{\"field\":\"embeddings_2\"}},\"id\":\"1390\",\"type\":\"Rect\"},{\"attributes\":{\"callback\":null,\"factors\":[\"dog\",\"Puppies are nice.\",\"I enjoy taking long walks along the beach with my dog.\"]},\"id\":\"1367\",\"type\":\"FactorRange\"},{\"attributes\":{\"data_source\":{\"id\":\"1388\",\"type\":\"ColumnDataSource\"},\"glyph\":{\"id\":\"1390\",\"type\":\"Rect\"},\"hover_glyph\":null,\"muted_glyph\":null,\"nonselection_glyph\":{\"id\":\"1391\",\"type\":\"Rect\"},\"selection_glyph\":null,\"view\":{\"id\":\"1393\",\"type\":\"CDSView\"}},\"id\":\"1392\",\"type\":\"GlyphRenderer\"},{\"attributes\":{\"source\":{\"id\":\"1388\",\"type\":\"ColumnDataSource\"}},\"id\":\"1393\",\"type\":\"CDSView\"},{\"attributes\":{\"high\":0.90728360414505,\"low\":0.5699548721313477,\"palette\":[\"#ffffcc\",\"#ffeda0\",\"#fed976\",\"#feb24c\",\"#fd8d3c\",\"#fc4e2a\",\"#e31a1c\",\"#bd0026\",\"#800026\"]},\"id\":\"1363\",\"type\":\"LinearColorMapper\"},{\"attributes\":{},\"id\":\"1428\",\"type\":\"UnionRenderers\"}],\"root_ids\":[\"1364\"]},\"title\":\"Bokeh Application\",\"version\":\"1.4.0\"}};\n",
+ " var render_items = [{\"docid\":\"85d1d6b6-c6cd-4e71-b29b-2cc49ada74c5\",\"roots\":{\"1364\":\"5e20475c-62a7-4a19-87ed-a605dc444c96\"}}];\n",
+ " root.Bokeh.embed.embed_items_notebook(docs_json, render_items);\n",
+ "\n",
+ " }\n",
+ " if (root.Bokeh !== undefined) {\n",
+ " embed_document(root);\n",
+ " } else {\n",
+ " var attempts = 0;\n",
+ " var timer = setInterval(function(root) {\n",
+ " if (root.Bokeh !== undefined) {\n",
+ " clearInterval(timer);\n",
+ " embed_document(root);\n",
+ " } else {\n",
+ " attempts++;\n",
+ " if (attempts > 100) {\n",
+ " clearInterval(timer);\n",
+ " console.log(\"Bokeh: ERROR: Unable to run BokehJS code because BokehJS library is missing\");\n",
+ " }\n",
+ " }\n",
+ " }, 10, root)\n",
+ " }\n",
+ "})(window);"
+ ],
+ "application/vnd.bokehjs_exec.v0+json": ""
+ },
+ "metadata": {
+ "application/vnd.bokehjs_exec.v0+json": {
+ "id": "1364"
+ },
+ "tags": []
+ },
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "visualize_similarity(en_result, it_result, english_sentences, italian_sentences, 'English-Italian Similarity')"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "m6ySvEGbQaTM"
+ },
+ "source": [
+ "### Italian-Spanish Similarity"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "irfwIeitQ7V6"
+ },
+ "outputs": [
+ {
+ "data": {
+ "application/javascript": [
+ "\n",
+ "(function(root) {\n",
+ " function now() {\n",
+ " return new Date();\n",
+ " }\n",
+ "\n",
+ " var force = true;\n",
+ "\n",
+ " if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n",
+ " root._bokeh_onload_callbacks = [];\n",
+ " root._bokeh_is_loading = undefined;\n",
+ " }\n",
+ "\n",
+ " var JS_MIME_TYPE = 'application/javascript';\n",
+ " var HTML_MIME_TYPE = 'text/html';\n",
+ " var EXEC_MIME_TYPE = 'application/vnd.bokehjs_exec.v0+json';\n",
+ " var CLASS_NAME = 'output_bokeh rendered_html';\n",
+ "\n",
+ " /**\n",
+ " * Render data to the DOM node\n",
+ " */\n",
+ " function render(props, node) {\n",
+ " var script = document.createElement(\"script\");\n",
+ " node.appendChild(script);\n",
+ " }\n",
+ "\n",
+ " /**\n",
+ " * Handle when an output is cleared or removed\n",
+ " */\n",
+ " function handleClearOutput(event, handle) {\n",
+ " var cell = handle.cell;\n",
+ "\n",
+ " var id = cell.output_area._bokeh_element_id;\n",
+ " var server_id = cell.output_area._bokeh_server_id;\n",
+ " // Clean up Bokeh references\n",
+ " if (id != null && id in Bokeh.index) {\n",
+ " Bokeh.index[id].model.document.clear();\n",
+ " delete Bokeh.index[id];\n",
+ " }\n",
+ "\n",
+ " if (server_id !== undefined) {\n",
+ " // Clean up Bokeh references\n",
+ " var cmd = \"from bokeh.io.state import curstate; print(curstate().uuid_to_server['\" + server_id + \"'].get_sessions()[0].document.roots[0]._id)\";\n",
+ " cell.notebook.kernel.execute(cmd, {\n",
+ " iopub: {\n",
+ " output: function(msg) {\n",
+ " var id = msg.content.text.trim();\n",
+ " if (id in Bokeh.index) {\n",
+ " Bokeh.index[id].model.document.clear();\n",
+ " delete Bokeh.index[id];\n",
+ " }\n",
+ " }\n",
+ " }\n",
+ " });\n",
+ " // Destroy server and session\n",
+ " var cmd = \"import bokeh.io.notebook as ion; ion.destroy_server('\" + server_id + \"')\";\n",
+ " cell.notebook.kernel.execute(cmd);\n",
+ " }\n",
+ " }\n",
+ "\n",
+ " /**\n",
+ " * Handle when a new output is added\n",
+ " */\n",
+ " function handleAddOutput(event, handle) {\n",
+ " var output_area = handle.output_area;\n",
+ " var output = handle.output;\n",
+ "\n",
+ " // limit handleAddOutput to display_data with EXEC_MIME_TYPE content only\n",
+ " if ((output.output_type != \"display_data\") || (!output.data.hasOwnProperty(EXEC_MIME_TYPE))) {\n",
+ " return\n",
+ " }\n",
+ "\n",
+ " var toinsert = output_area.element.find(\".\" + CLASS_NAME.split(' ')[0]);\n",
+ "\n",
+ " if (output.metadata[EXEC_MIME_TYPE][\"id\"] !== undefined) {\n",
+ " toinsert[toinsert.length - 1].firstChild.textContent = output.data[JS_MIME_TYPE];\n",
+ " // store reference to embed id on output_area\n",
+ " output_area._bokeh_element_id = output.metadata[EXEC_MIME_TYPE][\"id\"];\n",
+ " }\n",
+ " if (output.metadata[EXEC_MIME_TYPE][\"server_id\"] !== undefined) {\n",
+ " var bk_div = document.createElement(\"div\");\n",
+ " bk_div.innerHTML = output.data[HTML_MIME_TYPE];\n",
+ " var script_attrs = bk_div.children[0].attributes;\n",
+ " for (var i = 0; i < script_attrs.length; i++) {\n",
+ " toinsert[toinsert.length - 1].firstChild.setAttribute(script_attrs[i].name, script_attrs[i].value);\n",
+ " }\n",
+ " // store reference to server id on output_area\n",
+ " output_area._bokeh_server_id = output.metadata[EXEC_MIME_TYPE][\"server_id\"];\n",
+ " }\n",
+ " }\n",
+ "\n",
+ " function register_renderer(events, OutputArea) {\n",
+ "\n",
+ " function append_mime(data, metadata, element) {\n",
+ " // create a DOM node to render to\n",
+ " var toinsert = this.create_output_subarea(\n",
+ " metadata,\n",
+ " CLASS_NAME,\n",
+ " EXEC_MIME_TYPE\n",
+ " );\n",
+ " this.keyboard_manager.register_events(toinsert);\n",
+ " // Render to node\n",
+ " var props = {data: data, metadata: metadata[EXEC_MIME_TYPE]};\n",
+ " render(props, toinsert[toinsert.length - 1]);\n",
+ " element.append(toinsert);\n",
+ " return toinsert\n",
+ " }\n",
+ "\n",
+ " /* Handle when an output is cleared or removed */\n",
+ " events.on('clear_output.CodeCell', handleClearOutput);\n",
+ " events.on('delete.Cell', handleClearOutput);\n",
+ "\n",
+ " /* Handle when a new output is added */\n",
+ " events.on('output_added.OutputArea', handleAddOutput);\n",
+ "\n",
+ " /**\n",
+ " * Register the mime type and append_mime function with output_area\n",
+ " */\n",
+ " OutputArea.prototype.register_mime_type(EXEC_MIME_TYPE, append_mime, {\n",
+ " /* Is output safe? */\n",
+ " safe: true,\n",
+ " /* Index of renderer in `output_area.display_order` */\n",
+ " index: 0\n",
+ " });\n",
+ " }\n",
+ "\n",
+ " // register the mime type if in Jupyter Notebook environment and previously unregistered\n",
+ " if (root.Jupyter !== undefined) {\n",
+ " var events = require('base/js/events');\n",
+ " var OutputArea = require('notebook/js/outputarea').OutputArea;\n",
+ "\n",
+ " if (OutputArea.prototype.mime_types().indexOf(EXEC_MIME_TYPE) == -1) {\n",
+ " register_renderer(events, OutputArea);\n",
+ " }\n",
+ " }\n",
+ "\n",
+ " \n",
+ " if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n",
+ " root._bokeh_timeout = Date.now() + 5000;\n",
+ " root._bokeh_failed_load = false;\n",
+ " }\n",
+ "\n",
+ " var NB_LOAD_WARNING = {'data': {'text/html':\n",
+ " \"
\\n\"+\n",
+ " \"
\\n\"+\n",
+ " \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n",
+ " \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n",
+ " \"
\\n\"+\n",
+ " \"
\\n\"+\n",
+ " \"
re-rerun `output_notebook()` to attempt to load from CDN again, or
\"}};\n",
+ "\n",
+ " function display_loaded() {\n",
+ " var el = document.getElementById(null);\n",
+ " if (el != null) {\n",
+ " el.textContent = \"BokehJS is loading...\";\n",
+ " }\n",
+ " if (root.Bokeh !== undefined) {\n",
+ " if (el != null) {\n",
+ " el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n",
+ " }\n",
+ " } else if (Date.now() < root._bokeh_timeout) {\n",
+ " setTimeout(display_loaded, 100)\n",
+ " }\n",
+ " }\n",
+ "\n",
+ "\n",
+ " function run_callbacks() {\n",
+ " try {\n",
+ " root._bokeh_onload_callbacks.forEach(function(callback) {\n",
+ " if (callback != null)\n",
+ " callback();\n",
+ " });\n",
+ " } finally {\n",
+ " delete root._bokeh_onload_callbacks\n",
+ " }\n",
+ " console.debug(\"Bokeh: all callbacks have finished\");\n",
+ " }\n",
+ "\n",
+ " function load_libs(css_urls, js_urls, callback) {\n",
+ " if (css_urls == null) css_urls = [];\n",
+ " if (js_urls == null) js_urls = [];\n",
+ "\n",
+ " root._bokeh_onload_callbacks.push(callback);\n",
+ " if (root._bokeh_is_loading > 0) {\n",
+ " console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n",
+ " return null;\n",
+ " }\n",
+ " if (js_urls == null || js_urls.length === 0) {\n",
+ " run_callbacks();\n",
+ " return null;\n",
+ " }\n",
+ " console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n",
+ " root._bokeh_is_loading = css_urls.length + js_urls.length;\n",
+ "\n",
+ " function on_load() {\n",
+ " root._bokeh_is_loading--;\n",
+ " if (root._bokeh_is_loading === 0) {\n",
+ " console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n",
+ " run_callbacks()\n",
+ " }\n",
+ " }\n",
+ "\n",
+ " function on_error() {\n",
+ " console.error(\"failed to load \" + url);\n",
+ " }\n",
+ "\n",
+ " for (var i = 0; i < css_urls.length; i++) {\n",
+ " var url = css_urls[i];\n",
+ " const element = document.createElement(\"link\");\n",
+ " element.onload = on_load;\n",
+ " element.onerror = on_error;\n",
+ " element.rel = \"stylesheet\";\n",
+ " element.type = \"text/css\";\n",
+ " element.href = url;\n",
+ " console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n",
+ " document.body.appendChild(element);\n",
+ " }\n",
+ "\n",
+ " for (var i = 0; i < js_urls.length; i++) {\n",
+ " var url = js_urls[i];\n",
+ " var element = document.createElement('script');\n",
+ " element.onload = on_load;\n",
+ " element.onerror = on_error;\n",
+ " element.async = false;\n",
+ " element.src = url;\n",
+ " console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n",
+ " document.head.appendChild(element);\n",
+ " }\n",
+ " };\n",
+ "\n",
+ " function inject_raw_css(css) {\n",
+ " const element = document.createElement(\"style\");\n",
+ " element.appendChild(document.createTextNode(css));\n",
+ " document.body.appendChild(element);\n",
+ " }\n",
+ "\n",
+ " \n",
+ " var js_urls = [\"https://cdn.pydata.org/bokeh/release/bokeh-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-widgets-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-tables-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-gl-1.4.0.min.js\"];\n",
+ " var css_urls = [];\n",
+ " \n",
+ "\n",
+ " var inline_js = [\n",
+ " function(Bokeh) {\n",
+ " Bokeh.set_log_level(\"info\");\n",
+ " },\n",
+ " function(Bokeh) {\n",
+ " \n",
+ " \n",
+ " }\n",
+ " ];\n",
+ "\n",
+ " function run_inline_js() {\n",
+ " \n",
+ " if (root.Bokeh !== undefined || force === true) {\n",
+ " \n",
+ " for (var i = 0; i < inline_js.length; i++) {\n",
+ " inline_js[i].call(root, root.Bokeh);\n",
+ " }\n",
+ " } else if (Date.now() < root._bokeh_timeout) {\n",
+ " setTimeout(run_inline_js, 100);\n",
+ " } else if (!root._bokeh_failed_load) {\n",
+ " console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n",
+ " root._bokeh_failed_load = true;\n",
+ " } else if (force !== true) {\n",
+ " var cell = $(document.getElementById(null)).parents('.cell').data().cell;\n",
+ " cell.output_area.append_execute_result(NB_LOAD_WARNING)\n",
+ " }\n",
+ "\n",
+ " }\n",
+ "\n",
+ " if (root._bokeh_is_loading === 0) {\n",
+ " console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n",
+ " run_inline_js();\n",
+ " } else {\n",
+ " load_libs(css_urls, js_urls, function() {\n",
+ " console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n",
+ " run_inline_js();\n",
+ " });\n",
+ " }\n",
+ "}(window));"
+ ],
+ "application/vnd.bokehjs_load.v0+json": "\n(function(root) {\n function now() {\n return new Date();\n }\n\n var force = true;\n\n if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n root._bokeh_onload_callbacks = [];\n root._bokeh_is_loading = undefined;\n }\n\n \n\n \n if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n root._bokeh_timeout = Date.now() + 5000;\n root._bokeh_failed_load = false;\n }\n\n var NB_LOAD_WARNING = {'data': {'text/html':\n \"
\\n\"+\n \"
\\n\"+\n \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n \"
\\n\"+\n \"
\\n\"+\n \"
re-rerun `output_notebook()` to attempt to load from CDN again, or
\"}};\n\n function display_loaded() {\n var el = document.getElementById(null);\n if (el != null) {\n el.textContent = \"BokehJS is loading...\";\n }\n if (root.Bokeh !== undefined) {\n if (el != null) {\n el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n }\n } else if (Date.now() < root._bokeh_timeout) {\n setTimeout(display_loaded, 100)\n }\n }\n\n\n function run_callbacks() {\n try {\n root._bokeh_onload_callbacks.forEach(function(callback) {\n if (callback != null)\n callback();\n });\n } finally {\n delete root._bokeh_onload_callbacks\n }\n console.debug(\"Bokeh: all callbacks have finished\");\n }\n\n function load_libs(css_urls, js_urls, callback) {\n if (css_urls == null) css_urls = [];\n if (js_urls == null) js_urls = [];\n\n root._bokeh_onload_callbacks.push(callback);\n if (root._bokeh_is_loading > 0) {\n console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n return null;\n }\n if (js_urls == null || js_urls.length === 0) {\n run_callbacks();\n return null;\n }\n console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n root._bokeh_is_loading = css_urls.length + js_urls.length;\n\n function on_load() {\n root._bokeh_is_loading--;\n if (root._bokeh_is_loading === 0) {\n console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n run_callbacks()\n }\n }\n\n function on_error() {\n console.error(\"failed to load \" + url);\n }\n\n for (var i = 0; i < css_urls.length; i++) {\n var url = css_urls[i];\n const element = document.createElement(\"link\");\n element.onload = on_load;\n element.onerror = on_error;\n element.rel = \"stylesheet\";\n element.type = \"text/css\";\n element.href = url;\n console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n document.body.appendChild(element);\n }\n\n for (var i = 0; i < js_urls.length; i++) {\n var url = js_urls[i];\n var element = document.createElement('script');\n element.onload = on_load;\n element.onerror = on_error;\n element.async = false;\n element.src = url;\n console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n document.head.appendChild(element);\n }\n };\n\n function inject_raw_css(css) {\n const element = document.createElement(\"style\");\n element.appendChild(document.createTextNode(css));\n document.body.appendChild(element);\n }\n\n \n var js_urls = [\"https://cdn.pydata.org/bokeh/release/bokeh-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-widgets-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-tables-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-gl-1.4.0.min.js\"];\n var css_urls = [];\n \n\n var inline_js = [\n function(Bokeh) {\n Bokeh.set_log_level(\"info\");\n },\n function(Bokeh) {\n \n \n }\n ];\n\n function run_inline_js() {\n \n if (root.Bokeh !== undefined || force === true) {\n \n for (var i = 0; i < inline_js.length; i++) {\n inline_js[i].call(root, root.Bokeh);\n }\n } else if (Date.now() < root._bokeh_timeout) {\n setTimeout(run_inline_js, 100);\n } else if (!root._bokeh_failed_load) {\n console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n root._bokeh_failed_load = true;\n } else if (force !== true) {\n var cell = $(document.getElementById(null)).parents('.cell').data().cell;\n cell.output_area.append_execute_result(NB_LOAD_WARNING)\n }\n\n }\n\n if (root._bokeh_is_loading === 0) {\n console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n run_inline_js();\n } else {\n load_libs(css_urls, js_urls, function() {\n console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n run_inline_js();\n });\n }\n}(window));"
+ },
+ "metadata": {
+ "tags": []
+ },
+ "output_type": "display_data"
+ },
+ {
+ "data": {
+ "text/html": [
+ "\n",
+ "\n",
+ "\n",
+ "\n",
+ "\n",
+ "\n",
+ " \n"
+ ]
+ },
+ "metadata": {
+ "tags": []
+ },
+ "output_type": "display_data"
+ },
+ {
+ "data": {
+ "application/javascript": [
+ "(function(root) {\n",
+ " function embed_document(root) {\n",
+ " \n",
+ " var docs_json = {\"9eb33289-2019-49f5-b3a2-c34c93bf2800\":{\"roots\":{\"references\":[{\"attributes\":{\"above\":[{\"id\":\"1483\",\"type\":\"CategoricalAxis\"}],\"center\":[{\"id\":\"1486\",\"type\":\"Grid\"},{\"id\":\"1490\",\"type\":\"Grid\"}],\"left\":[{\"id\":\"1487\",\"type\":\"CategoricalAxis\"}],\"min_border_right\":300,\"plot_width\":1200,\"renderers\":[{\"id\":\"1500\",\"type\":\"GlyphRenderer\"}],\"title\":{\"id\":\"1473\",\"type\":\"Title\"},\"toolbar\":{\"id\":\"1493\",\"type\":\"Toolbar\"},\"toolbar_location\":\"below\",\"x_range\":{\"id\":\"1475\",\"type\":\"FactorRange\"},\"x_scale\":{\"id\":\"1479\",\"type\":\"CategoricalScale\"},\"y_range\":{\"id\":\"1477\",\"type\":\"FactorRange\"},\"y_scale\":{\"id\":\"1481\",\"type\":\"CategoricalScale\"}},\"id\":\"1472\",\"subtype\":\"Figure\",\"type\":\"Plot\"},{\"attributes\":{\"active_drag\":\"auto\",\"active_inspect\":\"auto\",\"active_multi\":null,\"active_scroll\":\"auto\",\"active_tap\":\"auto\",\"tools\":[{\"id\":\"1491\",\"type\":\"SaveTool\"},{\"id\":\"1492\",\"type\":\"HoverTool\"}]},\"id\":\"1493\",\"type\":\"Toolbar\"},{\"attributes\":{\"callback\":null,\"tooltips\":[[\"pair\",\"@embeddings_1 ||| @embeddings_2\"],[\"sim\",\"@sim\"]]},\"id\":\"1492\",\"type\":\"HoverTool\"},{\"attributes\":{\"fill_alpha\":{\"value\":0.1},\"fill_color\":{\"value\":\"#1f77b4\"},\"height\":{\"units\":\"data\",\"value\":1},\"line_alpha\":{\"value\":0.1},\"line_color\":{\"value\":\"#1f77b4\"},\"width\":{\"units\":\"data\",\"value\":1},\"x\":{\"field\":\"embeddings_1\"},\"y\":{\"field\":\"embeddings_2\"}},\"id\":\"1499\",\"type\":\"Rect\"},{\"attributes\":{\"fill_color\":{\"field\":\"sim\",\"transform\":{\"id\":\"1471\",\"type\":\"LinearColorMapper\"}},\"height\":{\"units\":\"data\",\"value\":1},\"line_color\":{\"value\":null},\"width\":{\"units\":\"data\",\"value\":1},\"x\":{\"field\":\"embeddings_1\"},\"y\":{\"field\":\"embeddings_2\"}},\"id\":\"1498\",\"type\":\"Rect\"},{\"attributes\":{\"data_source\":{\"id\":\"1496\",\"type\":\"ColumnDataSource\"},\"glyph\":{\"id\":\"1498\",\"type\":\"Rect\"},\"hover_glyph\":null,\"muted_glyph\":null,\"nonselection_glyph\":{\"id\":\"1499\",\"type\":\"Rect\"},\"selection_glyph\":null,\"view\":{\"id\":\"1501\",\"type\":\"CDSView\"}},\"id\":\"1500\",\"type\":\"GlyphRenderer\"},{\"attributes\":{\"ticker\":{\"id\":\"1484\",\"type\":\"CategoricalTicker\"}},\"id\":\"1486\",\"type\":\"Grid\"},{\"attributes\":{\"source\":{\"id\":\"1496\",\"type\":\"ColumnDataSource\"}},\"id\":\"1501\",\"type\":\"CDSView\"},{\"attributes\":{\"callback\":null,\"factors\":[\"Disfruto de dar largos paseos por la playa con mi perro.\",\"Los cachorros son agradables.\",\"perro\"]},\"id\":\"1477\",\"type\":\"FactorRange\"},{\"attributes\":{},\"id\":\"1491\",\"type\":\"SaveTool\"},{\"attributes\":{\"high\":0.9059451818466187,\"low\":0.564821720123291,\"palette\":[\"#ffffcc\",\"#ffeda0\",\"#fed976\",\"#feb24c\",\"#fd8d3c\",\"#fc4e2a\",\"#e31a1c\",\"#bd0026\",\"#800026\"]},\"id\":\"1471\",\"type\":\"LinearColorMapper\"},{\"attributes\":{},\"id\":\"1488\",\"type\":\"CategoricalTicker\"},{\"attributes\":{\"axis_line_color\":{\"value\":null},\"formatter\":{\"id\":\"1540\",\"type\":\"CategoricalTickFormatter\"},\"major_label_orientation\":0.7853981633974483,\"major_label_standoff\":16,\"major_label_text_font_size\":{\"value\":\"12pt\"},\"major_tick_line_color\":{\"value\":null},\"ticker\":{\"id\":\"1484\",\"type\":\"CategoricalTicker\"}},\"id\":\"1483\",\"type\":\"CategoricalAxis\"},{\"attributes\":{},\"id\":\"1484\",\"type\":\"CategoricalTicker\"},{\"attributes\":{\"axis_line_color\":{\"value\":null},\"formatter\":{\"id\":\"1538\",\"type\":\"CategoricalTickFormatter\"},\"major_label_standoff\":16,\"major_label_text_font_size\":{\"value\":\"12pt\"},\"major_tick_line_color\":{\"value\":null},\"ticker\":{\"id\":\"1488\",\"type\":\"CategoricalTicker\"}},\"id\":\"1487\",\"type\":\"CategoricalAxis\"},{\"attributes\":{},\"id\":\"1481\",\"type\":\"CategoricalScale\"},{\"attributes\":{},\"id\":\"1543\",\"type\":\"UnionRenderers\"},{\"attributes\":{},\"id\":\"1479\",\"type\":\"CategoricalScale\"},{\"attributes\":{},\"id\":\"1542\",\"type\":\"Selection\"},{\"attributes\":{},\"id\":\"1540\",\"type\":\"CategoricalTickFormatter\"},{\"attributes\":{\"text\":\"Italian-Spanish Similarity\",\"text_font_size\":{\"value\":\"12pt\"}},\"id\":\"1473\",\"type\":\"Title\"},{\"attributes\":{\"callback\":null,\"factors\":[\"cane\",\"I cuccioli sono carini.\",\"Mi piace fare lunghe passeggiate lungo la spiaggia con il mio cane.\"]},\"id\":\"1475\",\"type\":\"FactorRange\"},{\"attributes\":{},\"id\":\"1538\",\"type\":\"CategoricalTickFormatter\"},{\"attributes\":{\"callback\":null,\"data\":{\"embeddings_1\":[\"cane\",\"cane\",\"cane\",\"I cuccioli sono carini.\",\"I cuccioli sono carini.\",\"I cuccioli sono carini.\",\"Mi piace fare lunghe passeggiate lungo la spiaggia con il mio cane.\",\"Mi piace fare lunghe passeggiate lungo la spiaggia con il mio cane.\",\"Mi piace fare lunghe passeggiate lungo la spiaggia con il mio cane.\"],\"embeddings_2\":[\"perro\",\"Los cachorros son agradables.\",\"Disfruto de dar largos paseos por la playa con mi perro.\",\"perro\",\"Los cachorros son agradables.\",\"Disfruto de dar largos paseos por la playa con mi perro.\",\"perro\",\"Los cachorros son agradables.\",\"Disfruto de dar largos paseos por la playa con mi perro.\"],\"index\":[0,1,2,3,4,5,6,7,8],\"sim\":{\"__ndarray__\":\"AAAAwID97D8AAACAiUjkPwAAAICyQuI/AAAAQJl+4z8AAABAqebnPwAAAAAFE+I/AAAAgDrw4j8AAAAg++LjPwAAAEDo1eo/\",\"dtype\":\"float64\",\"shape\":[9]}},\"selected\":{\"id\":\"1542\",\"type\":\"Selection\"},\"selection_policy\":{\"id\":\"1543\",\"type\":\"UnionRenderers\"}},\"id\":\"1496\",\"type\":\"ColumnDataSource\"},{\"attributes\":{\"dimension\":1,\"ticker\":{\"id\":\"1488\",\"type\":\"CategoricalTicker\"}},\"id\":\"1490\",\"type\":\"Grid\"}],\"root_ids\":[\"1472\"]},\"title\":\"Bokeh Application\",\"version\":\"1.4.0\"}};\n",
+ " var render_items = [{\"docid\":\"9eb33289-2019-49f5-b3a2-c34c93bf2800\",\"roots\":{\"1472\":\"6f559e42-2dec-4a29-a3d9-62c969a8c08a\"}}];\n",
+ " root.Bokeh.embed.embed_items_notebook(docs_json, render_items);\n",
+ "\n",
+ " }\n",
+ " if (root.Bokeh !== undefined) {\n",
+ " embed_document(root);\n",
+ " } else {\n",
+ " var attempts = 0;\n",
+ " var timer = setInterval(function(root) {\n",
+ " if (root.Bokeh !== undefined) {\n",
+ " clearInterval(timer);\n",
+ " embed_document(root);\n",
+ " } else {\n",
+ " attempts++;\n",
+ " if (attempts > 100) {\n",
+ " clearInterval(timer);\n",
+ " console.log(\"Bokeh: ERROR: Unable to run BokehJS code because BokehJS library is missing\");\n",
+ " }\n",
+ " }\n",
+ " }, 10, root)\n",
+ " }\n",
+ "})(window);"
+ ],
+ "application/vnd.bokehjs_exec.v0+json": ""
+ },
+ "metadata": {
+ "application/vnd.bokehjs_exec.v0+json": {
+ "id": "1472"
+ },
+ "tags": []
+ },
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "visualize_similarity(it_result, es_result, italian_sentences, spanish_sentences, 'Italian-Spanish Similarity')"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "ueoRO8balwwr"
+ },
+ "source": [
+ "### English-Chinese Similarity"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "xA7anofVlxL7"
+ },
+ "outputs": [
+ {
+ "data": {
+ "application/javascript": [
+ "\n",
+ "(function(root) {\n",
+ " function now() {\n",
+ " return new Date();\n",
+ " }\n",
+ "\n",
+ " var force = true;\n",
+ "\n",
+ " if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n",
+ " root._bokeh_onload_callbacks = [];\n",
+ " root._bokeh_is_loading = undefined;\n",
+ " }\n",
+ "\n",
+ " var JS_MIME_TYPE = 'application/javascript';\n",
+ " var HTML_MIME_TYPE = 'text/html';\n",
+ " var EXEC_MIME_TYPE = 'application/vnd.bokehjs_exec.v0+json';\n",
+ " var CLASS_NAME = 'output_bokeh rendered_html';\n",
+ "\n",
+ " /**\n",
+ " * Render data to the DOM node\n",
+ " */\n",
+ " function render(props, node) {\n",
+ " var script = document.createElement(\"script\");\n",
+ " node.appendChild(script);\n",
+ " }\n",
+ "\n",
+ " /**\n",
+ " * Handle when an output is cleared or removed\n",
+ " */\n",
+ " function handleClearOutput(event, handle) {\n",
+ " var cell = handle.cell;\n",
+ "\n",
+ " var id = cell.output_area._bokeh_element_id;\n",
+ " var server_id = cell.output_area._bokeh_server_id;\n",
+ " // Clean up Bokeh references\n",
+ " if (id != null && id in Bokeh.index) {\n",
+ " Bokeh.index[id].model.document.clear();\n",
+ " delete Bokeh.index[id];\n",
+ " }\n",
+ "\n",
+ " if (server_id !== undefined) {\n",
+ " // Clean up Bokeh references\n",
+ " var cmd = \"from bokeh.io.state import curstate; print(curstate().uuid_to_server['\" + server_id + \"'].get_sessions()[0].document.roots[0]._id)\";\n",
+ " cell.notebook.kernel.execute(cmd, {\n",
+ " iopub: {\n",
+ " output: function(msg) {\n",
+ " var id = msg.content.text.trim();\n",
+ " if (id in Bokeh.index) {\n",
+ " Bokeh.index[id].model.document.clear();\n",
+ " delete Bokeh.index[id];\n",
+ " }\n",
+ " }\n",
+ " }\n",
+ " });\n",
+ " // Destroy server and session\n",
+ " var cmd = \"import bokeh.io.notebook as ion; ion.destroy_server('\" + server_id + \"')\";\n",
+ " cell.notebook.kernel.execute(cmd);\n",
+ " }\n",
+ " }\n",
+ "\n",
+ " /**\n",
+ " * Handle when a new output is added\n",
+ " */\n",
+ " function handleAddOutput(event, handle) {\n",
+ " var output_area = handle.output_area;\n",
+ " var output = handle.output;\n",
+ "\n",
+ " // limit handleAddOutput to display_data with EXEC_MIME_TYPE content only\n",
+ " if ((output.output_type != \"display_data\") || (!output.data.hasOwnProperty(EXEC_MIME_TYPE))) {\n",
+ " return\n",
+ " }\n",
+ "\n",
+ " var toinsert = output_area.element.find(\".\" + CLASS_NAME.split(' ')[0]);\n",
+ "\n",
+ " if (output.metadata[EXEC_MIME_TYPE][\"id\"] !== undefined) {\n",
+ " toinsert[toinsert.length - 1].firstChild.textContent = output.data[JS_MIME_TYPE];\n",
+ " // store reference to embed id on output_area\n",
+ " output_area._bokeh_element_id = output.metadata[EXEC_MIME_TYPE][\"id\"];\n",
+ " }\n",
+ " if (output.metadata[EXEC_MIME_TYPE][\"server_id\"] !== undefined) {\n",
+ " var bk_div = document.createElement(\"div\");\n",
+ " bk_div.innerHTML = output.data[HTML_MIME_TYPE];\n",
+ " var script_attrs = bk_div.children[0].attributes;\n",
+ " for (var i = 0; i < script_attrs.length; i++) {\n",
+ " toinsert[toinsert.length - 1].firstChild.setAttribute(script_attrs[i].name, script_attrs[i].value);\n",
+ " }\n",
+ " // store reference to server id on output_area\n",
+ " output_area._bokeh_server_id = output.metadata[EXEC_MIME_TYPE][\"server_id\"];\n",
+ " }\n",
+ " }\n",
+ "\n",
+ " function register_renderer(events, OutputArea) {\n",
+ "\n",
+ " function append_mime(data, metadata, element) {\n",
+ " // create a DOM node to render to\n",
+ " var toinsert = this.create_output_subarea(\n",
+ " metadata,\n",
+ " CLASS_NAME,\n",
+ " EXEC_MIME_TYPE\n",
+ " );\n",
+ " this.keyboard_manager.register_events(toinsert);\n",
+ " // Render to node\n",
+ " var props = {data: data, metadata: metadata[EXEC_MIME_TYPE]};\n",
+ " render(props, toinsert[toinsert.length - 1]);\n",
+ " element.append(toinsert);\n",
+ " return toinsert\n",
+ " }\n",
+ "\n",
+ " /* Handle when an output is cleared or removed */\n",
+ " events.on('clear_output.CodeCell', handleClearOutput);\n",
+ " events.on('delete.Cell', handleClearOutput);\n",
+ "\n",
+ " /* Handle when a new output is added */\n",
+ " events.on('output_added.OutputArea', handleAddOutput);\n",
+ "\n",
+ " /**\n",
+ " * Register the mime type and append_mime function with output_area\n",
+ " */\n",
+ " OutputArea.prototype.register_mime_type(EXEC_MIME_TYPE, append_mime, {\n",
+ " /* Is output safe? */\n",
+ " safe: true,\n",
+ " /* Index of renderer in `output_area.display_order` */\n",
+ " index: 0\n",
+ " });\n",
+ " }\n",
+ "\n",
+ " // register the mime type if in Jupyter Notebook environment and previously unregistered\n",
+ " if (root.Jupyter !== undefined) {\n",
+ " var events = require('base/js/events');\n",
+ " var OutputArea = require('notebook/js/outputarea').OutputArea;\n",
+ "\n",
+ " if (OutputArea.prototype.mime_types().indexOf(EXEC_MIME_TYPE) == -1) {\n",
+ " register_renderer(events, OutputArea);\n",
+ " }\n",
+ " }\n",
+ "\n",
+ " \n",
+ " if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n",
+ " root._bokeh_timeout = Date.now() + 5000;\n",
+ " root._bokeh_failed_load = false;\n",
+ " }\n",
+ "\n",
+ " var NB_LOAD_WARNING = {'data': {'text/html':\n",
+ " \"
\\n\"+\n",
+ " \"
\\n\"+\n",
+ " \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n",
+ " \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n",
+ " \"
\\n\"+\n",
+ " \"
\\n\"+\n",
+ " \"
re-rerun `output_notebook()` to attempt to load from CDN again, or
\"}};\n",
+ "\n",
+ " function display_loaded() {\n",
+ " var el = document.getElementById(null);\n",
+ " if (el != null) {\n",
+ " el.textContent = \"BokehJS is loading...\";\n",
+ " }\n",
+ " if (root.Bokeh !== undefined) {\n",
+ " if (el != null) {\n",
+ " el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n",
+ " }\n",
+ " } else if (Date.now() < root._bokeh_timeout) {\n",
+ " setTimeout(display_loaded, 100)\n",
+ " }\n",
+ " }\n",
+ "\n",
+ "\n",
+ " function run_callbacks() {\n",
+ " try {\n",
+ " root._bokeh_onload_callbacks.forEach(function(callback) {\n",
+ " if (callback != null)\n",
+ " callback();\n",
+ " });\n",
+ " } finally {\n",
+ " delete root._bokeh_onload_callbacks\n",
+ " }\n",
+ " console.debug(\"Bokeh: all callbacks have finished\");\n",
+ " }\n",
+ "\n",
+ " function load_libs(css_urls, js_urls, callback) {\n",
+ " if (css_urls == null) css_urls = [];\n",
+ " if (js_urls == null) js_urls = [];\n",
+ "\n",
+ " root._bokeh_onload_callbacks.push(callback);\n",
+ " if (root._bokeh_is_loading > 0) {\n",
+ " console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n",
+ " return null;\n",
+ " }\n",
+ " if (js_urls == null || js_urls.length === 0) {\n",
+ " run_callbacks();\n",
+ " return null;\n",
+ " }\n",
+ " console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n",
+ " root._bokeh_is_loading = css_urls.length + js_urls.length;\n",
+ "\n",
+ " function on_load() {\n",
+ " root._bokeh_is_loading--;\n",
+ " if (root._bokeh_is_loading === 0) {\n",
+ " console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n",
+ " run_callbacks()\n",
+ " }\n",
+ " }\n",
+ "\n",
+ " function on_error() {\n",
+ " console.error(\"failed to load \" + url);\n",
+ " }\n",
+ "\n",
+ " for (var i = 0; i < css_urls.length; i++) {\n",
+ " var url = css_urls[i];\n",
+ " const element = document.createElement(\"link\");\n",
+ " element.onload = on_load;\n",
+ " element.onerror = on_error;\n",
+ " element.rel = \"stylesheet\";\n",
+ " element.type = \"text/css\";\n",
+ " element.href = url;\n",
+ " console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n",
+ " document.body.appendChild(element);\n",
+ " }\n",
+ "\n",
+ " for (var i = 0; i < js_urls.length; i++) {\n",
+ " var url = js_urls[i];\n",
+ " var element = document.createElement('script');\n",
+ " element.onload = on_load;\n",
+ " element.onerror = on_error;\n",
+ " element.async = false;\n",
+ " element.src = url;\n",
+ " console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n",
+ " document.head.appendChild(element);\n",
+ " }\n",
+ " };\n",
+ "\n",
+ " function inject_raw_css(css) {\n",
+ " const element = document.createElement(\"style\");\n",
+ " element.appendChild(document.createTextNode(css));\n",
+ " document.body.appendChild(element);\n",
+ " }\n",
+ "\n",
+ " \n",
+ " var js_urls = [\"https://cdn.pydata.org/bokeh/release/bokeh-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-widgets-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-tables-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-gl-1.4.0.min.js\"];\n",
+ " var css_urls = [];\n",
+ " \n",
+ "\n",
+ " var inline_js = [\n",
+ " function(Bokeh) {\n",
+ " Bokeh.set_log_level(\"info\");\n",
+ " },\n",
+ " function(Bokeh) {\n",
+ " \n",
+ " \n",
+ " }\n",
+ " ];\n",
+ "\n",
+ " function run_inline_js() {\n",
+ " \n",
+ " if (root.Bokeh !== undefined || force === true) {\n",
+ " \n",
+ " for (var i = 0; i < inline_js.length; i++) {\n",
+ " inline_js[i].call(root, root.Bokeh);\n",
+ " }\n",
+ " } else if (Date.now() < root._bokeh_timeout) {\n",
+ " setTimeout(run_inline_js, 100);\n",
+ " } else if (!root._bokeh_failed_load) {\n",
+ " console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n",
+ " root._bokeh_failed_load = true;\n",
+ " } else if (force !== true) {\n",
+ " var cell = $(document.getElementById(null)).parents('.cell').data().cell;\n",
+ " cell.output_area.append_execute_result(NB_LOAD_WARNING)\n",
+ " }\n",
+ "\n",
+ " }\n",
+ "\n",
+ " if (root._bokeh_is_loading === 0) {\n",
+ " console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n",
+ " run_inline_js();\n",
+ " } else {\n",
+ " load_libs(css_urls, js_urls, function() {\n",
+ " console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n",
+ " run_inline_js();\n",
+ " });\n",
+ " }\n",
+ "}(window));"
+ ],
+ "application/vnd.bokehjs_load.v0+json": "\n(function(root) {\n function now() {\n return new Date();\n }\n\n var force = true;\n\n if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n root._bokeh_onload_callbacks = [];\n root._bokeh_is_loading = undefined;\n }\n\n \n\n \n if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n root._bokeh_timeout = Date.now() + 5000;\n root._bokeh_failed_load = false;\n }\n\n var NB_LOAD_WARNING = {'data': {'text/html':\n \"
\\n\"+\n \"
\\n\"+\n \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n \"
\\n\"+\n \"
\\n\"+\n \"
re-rerun `output_notebook()` to attempt to load from CDN again, or
\"}};\n\n function display_loaded() {\n var el = document.getElementById(null);\n if (el != null) {\n el.textContent = \"BokehJS is loading...\";\n }\n if (root.Bokeh !== undefined) {\n if (el != null) {\n el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n }\n } else if (Date.now() < root._bokeh_timeout) {\n setTimeout(display_loaded, 100)\n }\n }\n\n\n function run_callbacks() {\n try {\n root._bokeh_onload_callbacks.forEach(function(callback) {\n if (callback != null)\n callback();\n });\n } finally {\n delete root._bokeh_onload_callbacks\n }\n console.debug(\"Bokeh: all callbacks have finished\");\n }\n\n function load_libs(css_urls, js_urls, callback) {\n if (css_urls == null) css_urls = [];\n if (js_urls == null) js_urls = [];\n\n root._bokeh_onload_callbacks.push(callback);\n if (root._bokeh_is_loading > 0) {\n console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n return null;\n }\n if (js_urls == null || js_urls.length === 0) {\n run_callbacks();\n return null;\n }\n console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n root._bokeh_is_loading = css_urls.length + js_urls.length;\n\n function on_load() {\n root._bokeh_is_loading--;\n if (root._bokeh_is_loading === 0) {\n console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n run_callbacks()\n }\n }\n\n function on_error() {\n console.error(\"failed to load \" + url);\n }\n\n for (var i = 0; i < css_urls.length; i++) {\n var url = css_urls[i];\n const element = document.createElement(\"link\");\n element.onload = on_load;\n element.onerror = on_error;\n element.rel = \"stylesheet\";\n element.type = \"text/css\";\n element.href = url;\n console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n document.body.appendChild(element);\n }\n\n for (var i = 0; i < js_urls.length; i++) {\n var url = js_urls[i];\n var element = document.createElement('script');\n element.onload = on_load;\n element.onerror = on_error;\n element.async = false;\n element.src = url;\n console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n document.head.appendChild(element);\n }\n };\n\n function inject_raw_css(css) {\n const element = document.createElement(\"style\");\n element.appendChild(document.createTextNode(css));\n document.body.appendChild(element);\n }\n\n \n var js_urls = [\"https://cdn.pydata.org/bokeh/release/bokeh-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-widgets-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-tables-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-gl-1.4.0.min.js\"];\n var css_urls = [];\n \n\n var inline_js = [\n function(Bokeh) {\n Bokeh.set_log_level(\"info\");\n },\n function(Bokeh) {\n \n \n }\n ];\n\n function run_inline_js() {\n \n if (root.Bokeh !== undefined || force === true) {\n \n for (var i = 0; i < inline_js.length; i++) {\n inline_js[i].call(root, root.Bokeh);\n }\n } else if (Date.now() < root._bokeh_timeout) {\n setTimeout(run_inline_js, 100);\n } else if (!root._bokeh_failed_load) {\n console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n root._bokeh_failed_load = true;\n } else if (force !== true) {\n var cell = $(document.getElementById(null)).parents('.cell').data().cell;\n cell.output_area.append_execute_result(NB_LOAD_WARNING)\n }\n\n }\n\n if (root._bokeh_is_loading === 0) {\n console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n run_inline_js();\n } else {\n load_libs(css_urls, js_urls, function() {\n console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n run_inline_js();\n });\n }\n}(window));"
+ },
+ "metadata": {
+ "tags": []
+ },
+ "output_type": "display_data"
+ },
+ {
+ "data": {
+ "text/html": [
+ "\n",
+ "\n",
+ "\n",
+ "\n",
+ "\n",
+ "\n",
+ " \n"
+ ]
+ },
+ "metadata": {
+ "tags": []
+ },
+ "output_type": "display_data"
+ },
+ {
+ "data": {
+ "application/javascript": [
+ "(function(root) {\n",
+ " function embed_document(root) {\n",
+ " \n",
+ " var docs_json = {\"e65d5833-780d-410f-a7be-915f30e03b59\":{\"roots\":{\"references\":[{\"attributes\":{\"above\":[{\"id\":\"1598\",\"type\":\"CategoricalAxis\"}],\"center\":[{\"id\":\"1601\",\"type\":\"Grid\"},{\"id\":\"1605\",\"type\":\"Grid\"}],\"left\":[{\"id\":\"1602\",\"type\":\"CategoricalAxis\"}],\"min_border_right\":300,\"plot_width\":1200,\"renderers\":[{\"id\":\"1615\",\"type\":\"GlyphRenderer\"}],\"title\":{\"id\":\"1588\",\"type\":\"Title\"},\"toolbar\":{\"id\":\"1608\",\"type\":\"Toolbar\"},\"toolbar_location\":\"below\",\"x_range\":{\"id\":\"1590\",\"type\":\"FactorRange\"},\"x_scale\":{\"id\":\"1594\",\"type\":\"CategoricalScale\"},\"y_range\":{\"id\":\"1592\",\"type\":\"FactorRange\"},\"y_scale\":{\"id\":\"1596\",\"type\":\"CategoricalScale\"}},\"id\":\"1587\",\"subtype\":\"Figure\",\"type\":\"Plot\"},{\"attributes\":{},\"id\":\"1664\",\"type\":\"Selection\"},{\"attributes\":{\"dimension\":1,\"ticker\":{\"id\":\"1603\",\"type\":\"CategoricalTicker\"}},\"id\":\"1605\",\"type\":\"Grid\"},{\"attributes\":{},\"id\":\"1665\",\"type\":\"UnionRenderers\"},{\"attributes\":{},\"id\":\"1603\",\"type\":\"CategoricalTicker\"},{\"attributes\":{\"axis_line_color\":{\"value\":null},\"formatter\":{\"id\":\"1660\",\"type\":\"CategoricalTickFormatter\"},\"major_label_standoff\":16,\"major_label_text_font_size\":{\"value\":\"12pt\"},\"major_tick_line_color\":{\"value\":null},\"ticker\":{\"id\":\"1603\",\"type\":\"CategoricalTicker\"}},\"id\":\"1602\",\"type\":\"CategoricalAxis\"},{\"attributes\":{\"axis_line_color\":{\"value\":null},\"formatter\":{\"id\":\"1662\",\"type\":\"CategoricalTickFormatter\"},\"major_label_orientation\":0.7853981633974483,\"major_label_standoff\":16,\"major_label_text_font_size\":{\"value\":\"12pt\"},\"major_tick_line_color\":{\"value\":null},\"ticker\":{\"id\":\"1599\",\"type\":\"CategoricalTicker\"}},\"id\":\"1598\",\"type\":\"CategoricalAxis\"},{\"attributes\":{\"callback\":null,\"tooltips\":[[\"pair\",\"@embeddings_1 ||| @embeddings_2\"],[\"sim\",\"@sim\"]]},\"id\":\"1607\",\"type\":\"HoverTool\"},{\"attributes\":{},\"id\":\"1599\",\"type\":\"CategoricalTicker\"},{\"attributes\":{},\"id\":\"1596\",\"type\":\"CategoricalScale\"},{\"attributes\":{},\"id\":\"1594\",\"type\":\"CategoricalScale\"},{\"attributes\":{\"callback\":null,\"factors\":[\"\\u6211\\u559c\\u6b22\\u548c\\u6211\\u7684\\u72d7\\u4e00\\u8d77\\u6cbf\\u7740\\u6d77\\u6ee9\\u6563\\u6b65\\u3002\",\"\\u5c0f\\u72d7\\u5f88\\u597d\\u3002\",\"\\u72d7\"]},\"id\":\"1592\",\"type\":\"FactorRange\"},{\"attributes\":{\"callback\":null,\"factors\":[\"dog\",\"Puppies are nice.\",\"I enjoy taking long walks along the beach with my dog.\"]},\"id\":\"1590\",\"type\":\"FactorRange\"},{\"attributes\":{\"callback\":null,\"data\":{\"embeddings_1\":[\"dog\",\"dog\",\"dog\",\"Puppies are nice.\",\"Puppies are nice.\",\"Puppies are nice.\",\"I enjoy taking long walks along the beach with my dog.\",\"I enjoy taking long walks along the beach with my dog.\",\"I enjoy taking long walks along the beach with my dog.\"],\"embeddings_2\":[\"\\u72d7\",\"\\u5c0f\\u72d7\\u5f88\\u597d\\u3002\",\"\\u6211\\u559c\\u6b22\\u548c\\u6211\\u7684\\u72d7\\u4e00\\u8d77\\u6cbf\\u7740\\u6d77\\u6ee9\\u6563\\u6b65\\u3002\",\"\\u72d7\",\"\\u5c0f\\u72d7\\u5f88\\u597d\\u3002\",\"\\u6211\\u559c\\u6b22\\u548c\\u6211\\u7684\\u72d7\\u4e00\\u8d77\\u6cbf\\u7740\\u6d77\\u6ee9\\u6563\\u6b65\\u3002\",\"\\u72d7\",\"\\u5c0f\\u72d7\\u5f88\\u597d\\u3002\",\"\\u6211\\u559c\\u6b22\\u548c\\u6211\\u7684\\u72d7\\u4e00\\u8d77\\u6cbf\\u7740\\u6d77\\u6ee9\\u6563\\u6b65\\u3002\"],\"index\":[0,1,2,3,4,5,6,7,8],\"sim\":{\"__ndarray__\":\"AAAAAF0H7D8AAAAgK6rlPwAAACCmLeM/AAAAgLC95D8AAABAJcfnPwAAAMDQZOM/AAAAAFSj4j8AAACg4TfjPwAAAIBszug/\",\"dtype\":\"float64\",\"shape\":[9]}},\"selected\":{\"id\":\"1664\",\"type\":\"Selection\"},\"selection_policy\":{\"id\":\"1665\",\"type\":\"UnionRenderers\"}},\"id\":\"1611\",\"type\":\"ColumnDataSource\"},{\"attributes\":{\"active_drag\":\"auto\",\"active_inspect\":\"auto\",\"active_multi\":null,\"active_scroll\":\"auto\",\"active_tap\":\"auto\",\"tools\":[{\"id\":\"1606\",\"type\":\"SaveTool\"},{\"id\":\"1607\",\"type\":\"HoverTool\"}]},\"id\":\"1608\",\"type\":\"Toolbar\"},{\"attributes\":{\"text\":\"English-Chinese Similarity\",\"text_font_size\":{\"value\":\"12pt\"}},\"id\":\"1588\",\"type\":\"Title\"},{\"attributes\":{},\"id\":\"1662\",\"type\":\"CategoricalTickFormatter\"},{\"attributes\":{\"high\":0.8758988380432129,\"low\":0.5824375152587891,\"palette\":[\"#ffffcc\",\"#ffeda0\",\"#fed976\",\"#feb24c\",\"#fd8d3c\",\"#fc4e2a\",\"#e31a1c\",\"#bd0026\",\"#800026\"]},\"id\":\"1586\",\"type\":\"LinearColorMapper\"},{\"attributes\":{\"source\":{\"id\":\"1611\",\"type\":\"ColumnDataSource\"}},\"id\":\"1616\",\"type\":\"CDSView\"},{\"attributes\":{},\"id\":\"1660\",\"type\":\"CategoricalTickFormatter\"},{\"attributes\":{\"data_source\":{\"id\":\"1611\",\"type\":\"ColumnDataSource\"},\"glyph\":{\"id\":\"1613\",\"type\":\"Rect\"},\"hover_glyph\":null,\"muted_glyph\":null,\"nonselection_glyph\":{\"id\":\"1614\",\"type\":\"Rect\"},\"selection_glyph\":null,\"view\":{\"id\":\"1616\",\"type\":\"CDSView\"}},\"id\":\"1615\",\"type\":\"GlyphRenderer\"},{\"attributes\":{\"fill_color\":{\"field\":\"sim\",\"transform\":{\"id\":\"1586\",\"type\":\"LinearColorMapper\"}},\"height\":{\"units\":\"data\",\"value\":1},\"line_color\":{\"value\":null},\"width\":{\"units\":\"data\",\"value\":1},\"x\":{\"field\":\"embeddings_1\"},\"y\":{\"field\":\"embeddings_2\"}},\"id\":\"1613\",\"type\":\"Rect\"},{\"attributes\":{\"fill_alpha\":{\"value\":0.1},\"fill_color\":{\"value\":\"#1f77b4\"},\"height\":{\"units\":\"data\",\"value\":1},\"line_alpha\":{\"value\":0.1},\"line_color\":{\"value\":\"#1f77b4\"},\"width\":{\"units\":\"data\",\"value\":1},\"x\":{\"field\":\"embeddings_1\"},\"y\":{\"field\":\"embeddings_2\"}},\"id\":\"1614\",\"type\":\"Rect\"},{\"attributes\":{},\"id\":\"1606\",\"type\":\"SaveTool\"},{\"attributes\":{\"ticker\":{\"id\":\"1599\",\"type\":\"CategoricalTicker\"}},\"id\":\"1601\",\"type\":\"Grid\"}],\"root_ids\":[\"1587\"]},\"title\":\"Bokeh Application\",\"version\":\"1.4.0\"}};\n",
+ " var render_items = [{\"docid\":\"e65d5833-780d-410f-a7be-915f30e03b59\",\"roots\":{\"1587\":\"5b8e7d08-b7e7-4a05-a22d-c27aa1873e6d\"}}];\n",
+ " root.Bokeh.embed.embed_items_notebook(docs_json, render_items);\n",
+ "\n",
+ " }\n",
+ " if (root.Bokeh !== undefined) {\n",
+ " embed_document(root);\n",
+ " } else {\n",
+ " var attempts = 0;\n",
+ " var timer = setInterval(function(root) {\n",
+ " if (root.Bokeh !== undefined) {\n",
+ " clearInterval(timer);\n",
+ " embed_document(root);\n",
+ " } else {\n",
+ " attempts++;\n",
+ " if (attempts > 100) {\n",
+ " clearInterval(timer);\n",
+ " console.log(\"Bokeh: ERROR: Unable to run BokehJS code because BokehJS library is missing\");\n",
+ " }\n",
+ " }\n",
+ " }, 10, root)\n",
+ " }\n",
+ "})(window);"
+ ],
+ "application/vnd.bokehjs_exec.v0+json": ""
+ },
+ "metadata": {
+ "application/vnd.bokehjs_exec.v0+json": {
+ "id": "1587"
+ },
+ "tags": []
+ },
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "visualize_similarity(en_result, zh_result, english_sentences, chinese_sentences, 'English-Chinese Similarity')"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "8zV1BJc3mL3W"
+ },
+ "source": [
+ "### English-Korean Similarity"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "iqWy1e1UmQeX"
+ },
+ "outputs": [
+ {
+ "data": {
+ "application/javascript": [
+ "\n",
+ "(function(root) {\n",
+ " function now() {\n",
+ " return new Date();\n",
+ " }\n",
+ "\n",
+ " var force = true;\n",
+ "\n",
+ " if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n",
+ " root._bokeh_onload_callbacks = [];\n",
+ " root._bokeh_is_loading = undefined;\n",
+ " }\n",
+ "\n",
+ " var JS_MIME_TYPE = 'application/javascript';\n",
+ " var HTML_MIME_TYPE = 'text/html';\n",
+ " var EXEC_MIME_TYPE = 'application/vnd.bokehjs_exec.v0+json';\n",
+ " var CLASS_NAME = 'output_bokeh rendered_html';\n",
+ "\n",
+ " /**\n",
+ " * Render data to the DOM node\n",
+ " */\n",
+ " function render(props, node) {\n",
+ " var script = document.createElement(\"script\");\n",
+ " node.appendChild(script);\n",
+ " }\n",
+ "\n",
+ " /**\n",
+ " * Handle when an output is cleared or removed\n",
+ " */\n",
+ " function handleClearOutput(event, handle) {\n",
+ " var cell = handle.cell;\n",
+ "\n",
+ " var id = cell.output_area._bokeh_element_id;\n",
+ " var server_id = cell.output_area._bokeh_server_id;\n",
+ " // Clean up Bokeh references\n",
+ " if (id != null && id in Bokeh.index) {\n",
+ " Bokeh.index[id].model.document.clear();\n",
+ " delete Bokeh.index[id];\n",
+ " }\n",
+ "\n",
+ " if (server_id !== undefined) {\n",
+ " // Clean up Bokeh references\n",
+ " var cmd = \"from bokeh.io.state import curstate; print(curstate().uuid_to_server['\" + server_id + \"'].get_sessions()[0].document.roots[0]._id)\";\n",
+ " cell.notebook.kernel.execute(cmd, {\n",
+ " iopub: {\n",
+ " output: function(msg) {\n",
+ " var id = msg.content.text.trim();\n",
+ " if (id in Bokeh.index) {\n",
+ " Bokeh.index[id].model.document.clear();\n",
+ " delete Bokeh.index[id];\n",
+ " }\n",
+ " }\n",
+ " }\n",
+ " });\n",
+ " // Destroy server and session\n",
+ " var cmd = \"import bokeh.io.notebook as ion; ion.destroy_server('\" + server_id + \"')\";\n",
+ " cell.notebook.kernel.execute(cmd);\n",
+ " }\n",
+ " }\n",
+ "\n",
+ " /**\n",
+ " * Handle when a new output is added\n",
+ " */\n",
+ " function handleAddOutput(event, handle) {\n",
+ " var output_area = handle.output_area;\n",
+ " var output = handle.output;\n",
+ "\n",
+ " // limit handleAddOutput to display_data with EXEC_MIME_TYPE content only\n",
+ " if ((output.output_type != \"display_data\") || (!output.data.hasOwnProperty(EXEC_MIME_TYPE))) {\n",
+ " return\n",
+ " }\n",
+ "\n",
+ " var toinsert = output_area.element.find(\".\" + CLASS_NAME.split(' ')[0]);\n",
+ "\n",
+ " if (output.metadata[EXEC_MIME_TYPE][\"id\"] !== undefined) {\n",
+ " toinsert[toinsert.length - 1].firstChild.textContent = output.data[JS_MIME_TYPE];\n",
+ " // store reference to embed id on output_area\n",
+ " output_area._bokeh_element_id = output.metadata[EXEC_MIME_TYPE][\"id\"];\n",
+ " }\n",
+ " if (output.metadata[EXEC_MIME_TYPE][\"server_id\"] !== undefined) {\n",
+ " var bk_div = document.createElement(\"div\");\n",
+ " bk_div.innerHTML = output.data[HTML_MIME_TYPE];\n",
+ " var script_attrs = bk_div.children[0].attributes;\n",
+ " for (var i = 0; i < script_attrs.length; i++) {\n",
+ " toinsert[toinsert.length - 1].firstChild.setAttribute(script_attrs[i].name, script_attrs[i].value);\n",
+ " }\n",
+ " // store reference to server id on output_area\n",
+ " output_area._bokeh_server_id = output.metadata[EXEC_MIME_TYPE][\"server_id\"];\n",
+ " }\n",
+ " }\n",
+ "\n",
+ " function register_renderer(events, OutputArea) {\n",
+ "\n",
+ " function append_mime(data, metadata, element) {\n",
+ " // create a DOM node to render to\n",
+ " var toinsert = this.create_output_subarea(\n",
+ " metadata,\n",
+ " CLASS_NAME,\n",
+ " EXEC_MIME_TYPE\n",
+ " );\n",
+ " this.keyboard_manager.register_events(toinsert);\n",
+ " // Render to node\n",
+ " var props = {data: data, metadata: metadata[EXEC_MIME_TYPE]};\n",
+ " render(props, toinsert[toinsert.length - 1]);\n",
+ " element.append(toinsert);\n",
+ " return toinsert\n",
+ " }\n",
+ "\n",
+ " /* Handle when an output is cleared or removed */\n",
+ " events.on('clear_output.CodeCell', handleClearOutput);\n",
+ " events.on('delete.Cell', handleClearOutput);\n",
+ "\n",
+ " /* Handle when a new output is added */\n",
+ " events.on('output_added.OutputArea', handleAddOutput);\n",
+ "\n",
+ " /**\n",
+ " * Register the mime type and append_mime function with output_area\n",
+ " */\n",
+ " OutputArea.prototype.register_mime_type(EXEC_MIME_TYPE, append_mime, {\n",
+ " /* Is output safe? */\n",
+ " safe: true,\n",
+ " /* Index of renderer in `output_area.display_order` */\n",
+ " index: 0\n",
+ " });\n",
+ " }\n",
+ "\n",
+ " // register the mime type if in Jupyter Notebook environment and previously unregistered\n",
+ " if (root.Jupyter !== undefined) {\n",
+ " var events = require('base/js/events');\n",
+ " var OutputArea = require('notebook/js/outputarea').OutputArea;\n",
+ "\n",
+ " if (OutputArea.prototype.mime_types().indexOf(EXEC_MIME_TYPE) == -1) {\n",
+ " register_renderer(events, OutputArea);\n",
+ " }\n",
+ " }\n",
+ "\n",
+ " \n",
+ " if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n",
+ " root._bokeh_timeout = Date.now() + 5000;\n",
+ " root._bokeh_failed_load = false;\n",
+ " }\n",
+ "\n",
+ " var NB_LOAD_WARNING = {'data': {'text/html':\n",
+ " \"
\\n\"+\n",
+ " \"
\\n\"+\n",
+ " \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n",
+ " \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n",
+ " \"
\\n\"+\n",
+ " \"
\\n\"+\n",
+ " \"
re-rerun `output_notebook()` to attempt to load from CDN again, or
\"}};\n",
+ "\n",
+ " function display_loaded() {\n",
+ " var el = document.getElementById(null);\n",
+ " if (el != null) {\n",
+ " el.textContent = \"BokehJS is loading...\";\n",
+ " }\n",
+ " if (root.Bokeh !== undefined) {\n",
+ " if (el != null) {\n",
+ " el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n",
+ " }\n",
+ " } else if (Date.now() < root._bokeh_timeout) {\n",
+ " setTimeout(display_loaded, 100)\n",
+ " }\n",
+ " }\n",
+ "\n",
+ "\n",
+ " function run_callbacks() {\n",
+ " try {\n",
+ " root._bokeh_onload_callbacks.forEach(function(callback) {\n",
+ " if (callback != null)\n",
+ " callback();\n",
+ " });\n",
+ " } finally {\n",
+ " delete root._bokeh_onload_callbacks\n",
+ " }\n",
+ " console.debug(\"Bokeh: all callbacks have finished\");\n",
+ " }\n",
+ "\n",
+ " function load_libs(css_urls, js_urls, callback) {\n",
+ " if (css_urls == null) css_urls = [];\n",
+ " if (js_urls == null) js_urls = [];\n",
+ "\n",
+ " root._bokeh_onload_callbacks.push(callback);\n",
+ " if (root._bokeh_is_loading > 0) {\n",
+ " console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n",
+ " return null;\n",
+ " }\n",
+ " if (js_urls == null || js_urls.length === 0) {\n",
+ " run_callbacks();\n",
+ " return null;\n",
+ " }\n",
+ " console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n",
+ " root._bokeh_is_loading = css_urls.length + js_urls.length;\n",
+ "\n",
+ " function on_load() {\n",
+ " root._bokeh_is_loading--;\n",
+ " if (root._bokeh_is_loading === 0) {\n",
+ " console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n",
+ " run_callbacks()\n",
+ " }\n",
+ " }\n",
+ "\n",
+ " function on_error() {\n",
+ " console.error(\"failed to load \" + url);\n",
+ " }\n",
+ "\n",
+ " for (var i = 0; i < css_urls.length; i++) {\n",
+ " var url = css_urls[i];\n",
+ " const element = document.createElement(\"link\");\n",
+ " element.onload = on_load;\n",
+ " element.onerror = on_error;\n",
+ " element.rel = \"stylesheet\";\n",
+ " element.type = \"text/css\";\n",
+ " element.href = url;\n",
+ " console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n",
+ " document.body.appendChild(element);\n",
+ " }\n",
+ "\n",
+ " for (var i = 0; i < js_urls.length; i++) {\n",
+ " var url = js_urls[i];\n",
+ " var element = document.createElement('script');\n",
+ " element.onload = on_load;\n",
+ " element.onerror = on_error;\n",
+ " element.async = false;\n",
+ " element.src = url;\n",
+ " console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n",
+ " document.head.appendChild(element);\n",
+ " }\n",
+ " };\n",
+ "\n",
+ " function inject_raw_css(css) {\n",
+ " const element = document.createElement(\"style\");\n",
+ " element.appendChild(document.createTextNode(css));\n",
+ " document.body.appendChild(element);\n",
+ " }\n",
+ "\n",
+ " \n",
+ " var js_urls = [\"https://cdn.pydata.org/bokeh/release/bokeh-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-widgets-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-tables-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-gl-1.4.0.min.js\"];\n",
+ " var css_urls = [];\n",
+ " \n",
+ "\n",
+ " var inline_js = [\n",
+ " function(Bokeh) {\n",
+ " Bokeh.set_log_level(\"info\");\n",
+ " },\n",
+ " function(Bokeh) {\n",
+ " \n",
+ " \n",
+ " }\n",
+ " ];\n",
+ "\n",
+ " function run_inline_js() {\n",
+ " \n",
+ " if (root.Bokeh !== undefined || force === true) {\n",
+ " \n",
+ " for (var i = 0; i < inline_js.length; i++) {\n",
+ " inline_js[i].call(root, root.Bokeh);\n",
+ " }\n",
+ " } else if (Date.now() < root._bokeh_timeout) {\n",
+ " setTimeout(run_inline_js, 100);\n",
+ " } else if (!root._bokeh_failed_load) {\n",
+ " console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n",
+ " root._bokeh_failed_load = true;\n",
+ " } else if (force !== true) {\n",
+ " var cell = $(document.getElementById(null)).parents('.cell').data().cell;\n",
+ " cell.output_area.append_execute_result(NB_LOAD_WARNING)\n",
+ " }\n",
+ "\n",
+ " }\n",
+ "\n",
+ " if (root._bokeh_is_loading === 0) {\n",
+ " console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n",
+ " run_inline_js();\n",
+ " } else {\n",
+ " load_libs(css_urls, js_urls, function() {\n",
+ " console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n",
+ " run_inline_js();\n",
+ " });\n",
+ " }\n",
+ "}(window));"
+ ],
+ "application/vnd.bokehjs_load.v0+json": "\n(function(root) {\n function now() {\n return new Date();\n }\n\n var force = true;\n\n if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n root._bokeh_onload_callbacks = [];\n root._bokeh_is_loading = undefined;\n }\n\n \n\n \n if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n root._bokeh_timeout = Date.now() + 5000;\n root._bokeh_failed_load = false;\n }\n\n var NB_LOAD_WARNING = {'data': {'text/html':\n \"
\\n\"+\n \"
\\n\"+\n \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n \"
\\n\"+\n \"
\\n\"+\n \"
re-rerun `output_notebook()` to attempt to load from CDN again, or
\"}};\n\n function display_loaded() {\n var el = document.getElementById(null);\n if (el != null) {\n el.textContent = \"BokehJS is loading...\";\n }\n if (root.Bokeh !== undefined) {\n if (el != null) {\n el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n }\n } else if (Date.now() < root._bokeh_timeout) {\n setTimeout(display_loaded, 100)\n }\n }\n\n\n function run_callbacks() {\n try {\n root._bokeh_onload_callbacks.forEach(function(callback) {\n if (callback != null)\n callback();\n });\n } finally {\n delete root._bokeh_onload_callbacks\n }\n console.debug(\"Bokeh: all callbacks have finished\");\n }\n\n function load_libs(css_urls, js_urls, callback) {\n if (css_urls == null) css_urls = [];\n if (js_urls == null) js_urls = [];\n\n root._bokeh_onload_callbacks.push(callback);\n if (root._bokeh_is_loading > 0) {\n console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n return null;\n }\n if (js_urls == null || js_urls.length === 0) {\n run_callbacks();\n return null;\n }\n console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n root._bokeh_is_loading = css_urls.length + js_urls.length;\n\n function on_load() {\n root._bokeh_is_loading--;\n if (root._bokeh_is_loading === 0) {\n console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n run_callbacks()\n }\n }\n\n function on_error() {\n console.error(\"failed to load \" + url);\n }\n\n for (var i = 0; i < css_urls.length; i++) {\n var url = css_urls[i];\n const element = document.createElement(\"link\");\n element.onload = on_load;\n element.onerror = on_error;\n element.rel = \"stylesheet\";\n element.type = \"text/css\";\n element.href = url;\n console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n document.body.appendChild(element);\n }\n\n for (var i = 0; i < js_urls.length; i++) {\n var url = js_urls[i];\n var element = document.createElement('script');\n element.onload = on_load;\n element.onerror = on_error;\n element.async = false;\n element.src = url;\n console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n document.head.appendChild(element);\n }\n };\n\n function inject_raw_css(css) {\n const element = document.createElement(\"style\");\n element.appendChild(document.createTextNode(css));\n document.body.appendChild(element);\n }\n\n \n var js_urls = [\"https://cdn.pydata.org/bokeh/release/bokeh-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-widgets-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-tables-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-gl-1.4.0.min.js\"];\n var css_urls = [];\n \n\n var inline_js = [\n function(Bokeh) {\n Bokeh.set_log_level(\"info\");\n },\n function(Bokeh) {\n \n \n }\n ];\n\n function run_inline_js() {\n \n if (root.Bokeh !== undefined || force === true) {\n \n for (var i = 0; i < inline_js.length; i++) {\n inline_js[i].call(root, root.Bokeh);\n }\n } else if (Date.now() < root._bokeh_timeout) {\n setTimeout(run_inline_js, 100);\n } else if (!root._bokeh_failed_load) {\n console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n root._bokeh_failed_load = true;\n } else if (force !== true) {\n var cell = $(document.getElementById(null)).parents('.cell').data().cell;\n cell.output_area.append_execute_result(NB_LOAD_WARNING)\n }\n\n }\n\n if (root._bokeh_is_loading === 0) {\n console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n run_inline_js();\n } else {\n load_libs(css_urls, js_urls, function() {\n console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n run_inline_js();\n });\n }\n}(window));"
+ },
+ "metadata": {
+ "tags": []
+ },
+ "output_type": "display_data"
+ },
+ {
+ "data": {
+ "text/html": [
+ "\n",
+ "\n",
+ "\n",
+ "\n",
+ "\n",
+ "\n",
+ " \n"
+ ]
+ },
+ "metadata": {
+ "tags": []
+ },
+ "output_type": "display_data"
+ },
+ {
+ "data": {
+ "application/javascript": [
+ "(function(root) {\n",
+ " function embed_document(root) {\n",
+ " \n",
+ " var docs_json = {\"1bbe715c-608d-49a6-8927-e818fa752480\":{\"roots\":{\"references\":[{\"attributes\":{\"above\":[{\"id\":\"1720\",\"type\":\"CategoricalAxis\"}],\"center\":[{\"id\":\"1723\",\"type\":\"Grid\"},{\"id\":\"1727\",\"type\":\"Grid\"}],\"left\":[{\"id\":\"1724\",\"type\":\"CategoricalAxis\"}],\"min_border_right\":300,\"plot_width\":1200,\"renderers\":[{\"id\":\"1737\",\"type\":\"GlyphRenderer\"}],\"title\":{\"id\":\"1710\",\"type\":\"Title\"},\"toolbar\":{\"id\":\"1730\",\"type\":\"Toolbar\"},\"toolbar_location\":\"below\",\"x_range\":{\"id\":\"1712\",\"type\":\"FactorRange\"},\"x_scale\":{\"id\":\"1716\",\"type\":\"CategoricalScale\"},\"y_range\":{\"id\":\"1714\",\"type\":\"FactorRange\"},\"y_scale\":{\"id\":\"1718\",\"type\":\"CategoricalScale\"}},\"id\":\"1709\",\"subtype\":\"Figure\",\"type\":\"Plot\"},{\"attributes\":{},\"id\":\"1794\",\"type\":\"UnionRenderers\"},{\"attributes\":{},\"id\":\"1725\",\"type\":\"CategoricalTicker\"},{\"attributes\":{\"axis_line_color\":{\"value\":null},\"formatter\":{\"id\":\"1789\",\"type\":\"CategoricalTickFormatter\"},\"major_label_standoff\":16,\"major_label_text_font_size\":{\"value\":\"12pt\"},\"major_tick_line_color\":{\"value\":null},\"ticker\":{\"id\":\"1725\",\"type\":\"CategoricalTicker\"}},\"id\":\"1724\",\"type\":\"CategoricalAxis\"},{\"attributes\":{\"dimension\":1,\"ticker\":{\"id\":\"1725\",\"type\":\"CategoricalTicker\"}},\"id\":\"1727\",\"type\":\"Grid\"},{\"attributes\":{},\"id\":\"1721\",\"type\":\"CategoricalTicker\"},{\"attributes\":{},\"id\":\"1728\",\"type\":\"SaveTool\"},{\"attributes\":{\"callback\":null,\"factors\":[\"dog\",\"Puppies are nice.\",\"I enjoy taking long walks along the beach with my dog.\"]},\"id\":\"1712\",\"type\":\"FactorRange\"},{\"attributes\":{\"active_drag\":\"auto\",\"active_inspect\":\"auto\",\"active_multi\":null,\"active_scroll\":\"auto\",\"active_tap\":\"auto\",\"tools\":[{\"id\":\"1728\",\"type\":\"SaveTool\"},{\"id\":\"1729\",\"type\":\"HoverTool\"}]},\"id\":\"1730\",\"type\":\"Toolbar\"},{\"attributes\":{},\"id\":\"1718\",\"type\":\"CategoricalScale\"},{\"attributes\":{},\"id\":\"1793\",\"type\":\"Selection\"},{\"attributes\":{\"axis_line_color\":{\"value\":null},\"formatter\":{\"id\":\"1791\",\"type\":\"CategoricalTickFormatter\"},\"major_label_orientation\":0.7853981633974483,\"major_label_standoff\":16,\"major_label_text_font_size\":{\"value\":\"12pt\"},\"major_tick_line_color\":{\"value\":null},\"ticker\":{\"id\":\"1721\",\"type\":\"CategoricalTicker\"}},\"id\":\"1720\",\"type\":\"CategoricalAxis\"},{\"attributes\":{\"callback\":null,\"tooltips\":[[\"pair\",\"@embeddings_1 ||| @embeddings_2\"],[\"sim\",\"@sim\"]]},\"id\":\"1729\",\"type\":\"HoverTool\"},{\"attributes\":{},\"id\":\"1716\",\"type\":\"CategoricalScale\"},{\"attributes\":{\"fill_color\":{\"field\":\"sim\",\"transform\":{\"id\":\"1708\",\"type\":\"LinearColorMapper\"}},\"height\":{\"units\":\"data\",\"value\":1},\"line_color\":{\"value\":null},\"width\":{\"units\":\"data\",\"value\":1},\"x\":{\"field\":\"embeddings_1\"},\"y\":{\"field\":\"embeddings_2\"}},\"id\":\"1735\",\"type\":\"Rect\"},{\"attributes\":{\"callback\":null,\"factors\":[\"\\ub098\\ub294 \\ub098\\uc758 \\uc0b0\\ucc45\\uc744 \\ud574\\ubcc0\\uc744 \\ub530\\ub77c \\uae38\\uac8c \\uc0b0\\ucc45\\ud558\\ub294 \\uac83\\uc744 \\uc990\\uae34\\ub2e4.\",\"\\uac15\\uc544\\uc9c0\\uac00 \\uc88b\\ub2e4.\",\"\\uac1c\"]},\"id\":\"1714\",\"type\":\"FactorRange\"},{\"attributes\":{\"fill_alpha\":{\"value\":0.1},\"fill_color\":{\"value\":\"#1f77b4\"},\"height\":{\"units\":\"data\",\"value\":1},\"line_alpha\":{\"value\":0.1},\"line_color\":{\"value\":\"#1f77b4\"},\"width\":{\"units\":\"data\",\"value\":1},\"x\":{\"field\":\"embeddings_1\"},\"y\":{\"field\":\"embeddings_2\"}},\"id\":\"1736\",\"type\":\"Rect\"},{\"attributes\":{\"source\":{\"id\":\"1733\",\"type\":\"ColumnDataSource\"}},\"id\":\"1738\",\"type\":\"CDSView\"},{\"attributes\":{\"data_source\":{\"id\":\"1733\",\"type\":\"ColumnDataSource\"},\"glyph\":{\"id\":\"1735\",\"type\":\"Rect\"},\"hover_glyph\":null,\"muted_glyph\":null,\"nonselection_glyph\":{\"id\":\"1736\",\"type\":\"Rect\"},\"selection_glyph\":null,\"view\":{\"id\":\"1738\",\"type\":\"CDSView\"}},\"id\":\"1737\",\"type\":\"GlyphRenderer\"},{\"attributes\":{\"text\":\"English-Korean Similarity\",\"text_font_size\":{\"value\":\"12pt\"}},\"id\":\"1710\",\"type\":\"Title\"},{\"attributes\":{\"callback\":null,\"data\":{\"embeddings_1\":[\"dog\",\"dog\",\"dog\",\"Puppies are nice.\",\"Puppies are nice.\",\"Puppies are nice.\",\"I enjoy taking long walks along the beach with my dog.\",\"I enjoy taking long walks along the beach with my dog.\",\"I enjoy taking long walks along the beach with my dog.\"],\"embeddings_2\":[\"\\uac1c\",\"\\uac15\\uc544\\uc9c0\\uac00 \\uc88b\\ub2e4.\",\"\\ub098\\ub294 \\ub098\\uc758 \\uc0b0\\ucc45\\uc744 \\ud574\\ubcc0\\uc744 \\ub530\\ub77c \\uae38\\uac8c \\uc0b0\\ucc45\\ud558\\ub294 \\uac83\\uc744 \\uc990\\uae34\\ub2e4.\",\"\\uac1c\",\"\\uac15\\uc544\\uc9c0\\uac00 \\uc88b\\ub2e4.\",\"\\ub098\\ub294 \\ub098\\uc758 \\uc0b0\\ucc45\\uc744 \\ud574\\ubcc0\\uc744 \\ub530\\ub77c \\uae38\\uac8c \\uc0b0\\ucc45\\ud558\\ub294 \\uac83\\uc744 \\uc990\\uae34\\ub2e4.\",\"\\uac1c\",\"\\uac15\\uc544\\uc9c0\\uac00 \\uc88b\\ub2e4.\",\"\\ub098\\ub294 \\ub098\\uc758 \\uc0b0\\ucc45\\uc744 \\ud574\\ubcc0\\uc744 \\ub530\\ub77c \\uae38\\uac8c \\uc0b0\\ucc45\\ud558\\ub294 \\uac83\\uc744 \\uc990\\uae34\\ub2e4.\"],\"index\":[0,1,2,3,4,5,6,7,8],\"sim\":{\"__ndarray__\":\"AAAAgICf6z8AAAAAKEjlPwAAAKBNk+A/AAAAwFZl5D8AAAAAUpnnPwAAAID69uA/AAAAQJua4j8AAADgCQ3jPwAAAIC9gOg/\",\"dtype\":\"float64\",\"shape\":[9]}},\"selected\":{\"id\":\"1793\",\"type\":\"Selection\"},\"selection_policy\":{\"id\":\"1794\",\"type\":\"UnionRenderers\"}},\"id\":\"1733\",\"type\":\"ColumnDataSource\"},{\"attributes\":{},\"id\":\"1789\",\"type\":\"CategoricalTickFormatter\"},{\"attributes\":{},\"id\":\"1791\",\"type\":\"CategoricalTickFormatter\"},{\"attributes\":{\"ticker\":{\"id\":\"1721\",\"type\":\"CategoricalTicker\"}},\"id\":\"1723\",\"type\":\"Grid\"},{\"attributes\":{\"high\":0.8632204532623291,\"low\":0.5179813504219055,\"palette\":[\"#ffffcc\",\"#ffeda0\",\"#fed976\",\"#feb24c\",\"#fd8d3c\",\"#fc4e2a\",\"#e31a1c\",\"#bd0026\",\"#800026\"]},\"id\":\"1708\",\"type\":\"LinearColorMapper\"}],\"root_ids\":[\"1709\"]},\"title\":\"Bokeh Application\",\"version\":\"1.4.0\"}};\n",
+ " var render_items = [{\"docid\":\"1bbe715c-608d-49a6-8927-e818fa752480\",\"roots\":{\"1709\":\"7b449243-0dbd-46b6-8b02-a89fdf92645e\"}}];\n",
+ " root.Bokeh.embed.embed_items_notebook(docs_json, render_items);\n",
+ "\n",
+ " }\n",
+ " if (root.Bokeh !== undefined) {\n",
+ " embed_document(root);\n",
+ " } else {\n",
+ " var attempts = 0;\n",
+ " var timer = setInterval(function(root) {\n",
+ " if (root.Bokeh !== undefined) {\n",
+ " clearInterval(timer);\n",
+ " embed_document(root);\n",
+ " } else {\n",
+ " attempts++;\n",
+ " if (attempts > 100) {\n",
+ " clearInterval(timer);\n",
+ " console.log(\"Bokeh: ERROR: Unable to run BokehJS code because BokehJS library is missing\");\n",
+ " }\n",
+ " }\n",
+ " }, 10, root)\n",
+ " }\n",
+ "})(window);"
+ ],
+ "application/vnd.bokehjs_exec.v0+json": ""
+ },
+ "metadata": {
+ "application/vnd.bokehjs_exec.v0+json": {
+ "id": "1709"
+ },
+ "tags": []
+ },
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "visualize_similarity(en_result, ko_result, english_sentences, korean_sentences, 'English-Korean Similarity')"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "dfTj-JaunFTv"
+ },
+ "source": [
+ "### Chinese-Korean Similarity"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "MndSgKGPnJuF"
+ },
+ "outputs": [
+ {
+ "data": {
+ "application/javascript": [
+ "\n",
+ "(function(root) {\n",
+ " function now() {\n",
+ " return new Date();\n",
+ " }\n",
+ "\n",
+ " var force = true;\n",
+ "\n",
+ " if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n",
+ " root._bokeh_onload_callbacks = [];\n",
+ " root._bokeh_is_loading = undefined;\n",
+ " }\n",
+ "\n",
+ " var JS_MIME_TYPE = 'application/javascript';\n",
+ " var HTML_MIME_TYPE = 'text/html';\n",
+ " var EXEC_MIME_TYPE = 'application/vnd.bokehjs_exec.v0+json';\n",
+ " var CLASS_NAME = 'output_bokeh rendered_html';\n",
+ "\n",
+ " /**\n",
+ " * Render data to the DOM node\n",
+ " */\n",
+ " function render(props, node) {\n",
+ " var script = document.createElement(\"script\");\n",
+ " node.appendChild(script);\n",
+ " }\n",
+ "\n",
+ " /**\n",
+ " * Handle when an output is cleared or removed\n",
+ " */\n",
+ " function handleClearOutput(event, handle) {\n",
+ " var cell = handle.cell;\n",
+ "\n",
+ " var id = cell.output_area._bokeh_element_id;\n",
+ " var server_id = cell.output_area._bokeh_server_id;\n",
+ " // Clean up Bokeh references\n",
+ " if (id != null && id in Bokeh.index) {\n",
+ " Bokeh.index[id].model.document.clear();\n",
+ " delete Bokeh.index[id];\n",
+ " }\n",
+ "\n",
+ " if (server_id !== undefined) {\n",
+ " // Clean up Bokeh references\n",
+ " var cmd = \"from bokeh.io.state import curstate; print(curstate().uuid_to_server['\" + server_id + \"'].get_sessions()[0].document.roots[0]._id)\";\n",
+ " cell.notebook.kernel.execute(cmd, {\n",
+ " iopub: {\n",
+ " output: function(msg) {\n",
+ " var id = msg.content.text.trim();\n",
+ " if (id in Bokeh.index) {\n",
+ " Bokeh.index[id].model.document.clear();\n",
+ " delete Bokeh.index[id];\n",
+ " }\n",
+ " }\n",
+ " }\n",
+ " });\n",
+ " // Destroy server and session\n",
+ " var cmd = \"import bokeh.io.notebook as ion; ion.destroy_server('\" + server_id + \"')\";\n",
+ " cell.notebook.kernel.execute(cmd);\n",
+ " }\n",
+ " }\n",
+ "\n",
+ " /**\n",
+ " * Handle when a new output is added\n",
+ " */\n",
+ " function handleAddOutput(event, handle) {\n",
+ " var output_area = handle.output_area;\n",
+ " var output = handle.output;\n",
+ "\n",
+ " // limit handleAddOutput to display_data with EXEC_MIME_TYPE content only\n",
+ " if ((output.output_type != \"display_data\") || (!output.data.hasOwnProperty(EXEC_MIME_TYPE))) {\n",
+ " return\n",
+ " }\n",
+ "\n",
+ " var toinsert = output_area.element.find(\".\" + CLASS_NAME.split(' ')[0]);\n",
+ "\n",
+ " if (output.metadata[EXEC_MIME_TYPE][\"id\"] !== undefined) {\n",
+ " toinsert[toinsert.length - 1].firstChild.textContent = output.data[JS_MIME_TYPE];\n",
+ " // store reference to embed id on output_area\n",
+ " output_area._bokeh_element_id = output.metadata[EXEC_MIME_TYPE][\"id\"];\n",
+ " }\n",
+ " if (output.metadata[EXEC_MIME_TYPE][\"server_id\"] !== undefined) {\n",
+ " var bk_div = document.createElement(\"div\");\n",
+ " bk_div.innerHTML = output.data[HTML_MIME_TYPE];\n",
+ " var script_attrs = bk_div.children[0].attributes;\n",
+ " for (var i = 0; i < script_attrs.length; i++) {\n",
+ " toinsert[toinsert.length - 1].firstChild.setAttribute(script_attrs[i].name, script_attrs[i].value);\n",
+ " }\n",
+ " // store reference to server id on output_area\n",
+ " output_area._bokeh_server_id = output.metadata[EXEC_MIME_TYPE][\"server_id\"];\n",
+ " }\n",
+ " }\n",
+ "\n",
+ " function register_renderer(events, OutputArea) {\n",
+ "\n",
+ " function append_mime(data, metadata, element) {\n",
+ " // create a DOM node to render to\n",
+ " var toinsert = this.create_output_subarea(\n",
+ " metadata,\n",
+ " CLASS_NAME,\n",
+ " EXEC_MIME_TYPE\n",
+ " );\n",
+ " this.keyboard_manager.register_events(toinsert);\n",
+ " // Render to node\n",
+ " var props = {data: data, metadata: metadata[EXEC_MIME_TYPE]};\n",
+ " render(props, toinsert[toinsert.length - 1]);\n",
+ " element.append(toinsert);\n",
+ " return toinsert\n",
+ " }\n",
+ "\n",
+ " /* Handle when an output is cleared or removed */\n",
+ " events.on('clear_output.CodeCell', handleClearOutput);\n",
+ " events.on('delete.Cell', handleClearOutput);\n",
+ "\n",
+ " /* Handle when a new output is added */\n",
+ " events.on('output_added.OutputArea', handleAddOutput);\n",
+ "\n",
+ " /**\n",
+ " * Register the mime type and append_mime function with output_area\n",
+ " */\n",
+ " OutputArea.prototype.register_mime_type(EXEC_MIME_TYPE, append_mime, {\n",
+ " /* Is output safe? */\n",
+ " safe: true,\n",
+ " /* Index of renderer in `output_area.display_order` */\n",
+ " index: 0\n",
+ " });\n",
+ " }\n",
+ "\n",
+ " // register the mime type if in Jupyter Notebook environment and previously unregistered\n",
+ " if (root.Jupyter !== undefined) {\n",
+ " var events = require('base/js/events');\n",
+ " var OutputArea = require('notebook/js/outputarea').OutputArea;\n",
+ "\n",
+ " if (OutputArea.prototype.mime_types().indexOf(EXEC_MIME_TYPE) == -1) {\n",
+ " register_renderer(events, OutputArea);\n",
+ " }\n",
+ " }\n",
+ "\n",
+ " \n",
+ " if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n",
+ " root._bokeh_timeout = Date.now() + 5000;\n",
+ " root._bokeh_failed_load = false;\n",
+ " }\n",
+ "\n",
+ " var NB_LOAD_WARNING = {'data': {'text/html':\n",
+ " \"
\\n\"+\n",
+ " \"
\\n\"+\n",
+ " \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n",
+ " \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n",
+ " \"
\\n\"+\n",
+ " \"
\\n\"+\n",
+ " \"
re-rerun `output_notebook()` to attempt to load from CDN again, or
\"}};\n",
+ "\n",
+ " function display_loaded() {\n",
+ " var el = document.getElementById(null);\n",
+ " if (el != null) {\n",
+ " el.textContent = \"BokehJS is loading...\";\n",
+ " }\n",
+ " if (root.Bokeh !== undefined) {\n",
+ " if (el != null) {\n",
+ " el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n",
+ " }\n",
+ " } else if (Date.now() < root._bokeh_timeout) {\n",
+ " setTimeout(display_loaded, 100)\n",
+ " }\n",
+ " }\n",
+ "\n",
+ "\n",
+ " function run_callbacks() {\n",
+ " try {\n",
+ " root._bokeh_onload_callbacks.forEach(function(callback) {\n",
+ " if (callback != null)\n",
+ " callback();\n",
+ " });\n",
+ " } finally {\n",
+ " delete root._bokeh_onload_callbacks\n",
+ " }\n",
+ " console.debug(\"Bokeh: all callbacks have finished\");\n",
+ " }\n",
+ "\n",
+ " function load_libs(css_urls, js_urls, callback) {\n",
+ " if (css_urls == null) css_urls = [];\n",
+ " if (js_urls == null) js_urls = [];\n",
+ "\n",
+ " root._bokeh_onload_callbacks.push(callback);\n",
+ " if (root._bokeh_is_loading > 0) {\n",
+ " console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n",
+ " return null;\n",
+ " }\n",
+ " if (js_urls == null || js_urls.length === 0) {\n",
+ " run_callbacks();\n",
+ " return null;\n",
+ " }\n",
+ " console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n",
+ " root._bokeh_is_loading = css_urls.length + js_urls.length;\n",
+ "\n",
+ " function on_load() {\n",
+ " root._bokeh_is_loading--;\n",
+ " if (root._bokeh_is_loading === 0) {\n",
+ " console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n",
+ " run_callbacks()\n",
+ " }\n",
+ " }\n",
+ "\n",
+ " function on_error() {\n",
+ " console.error(\"failed to load \" + url);\n",
+ " }\n",
+ "\n",
+ " for (var i = 0; i < css_urls.length; i++) {\n",
+ " var url = css_urls[i];\n",
+ " const element = document.createElement(\"link\");\n",
+ " element.onload = on_load;\n",
+ " element.onerror = on_error;\n",
+ " element.rel = \"stylesheet\";\n",
+ " element.type = \"text/css\";\n",
+ " element.href = url;\n",
+ " console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n",
+ " document.body.appendChild(element);\n",
+ " }\n",
+ "\n",
+ " for (var i = 0; i < js_urls.length; i++) {\n",
+ " var url = js_urls[i];\n",
+ " var element = document.createElement('script');\n",
+ " element.onload = on_load;\n",
+ " element.onerror = on_error;\n",
+ " element.async = false;\n",
+ " element.src = url;\n",
+ " console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n",
+ " document.head.appendChild(element);\n",
+ " }\n",
+ " };\n",
+ "\n",
+ " function inject_raw_css(css) {\n",
+ " const element = document.createElement(\"style\");\n",
+ " element.appendChild(document.createTextNode(css));\n",
+ " document.body.appendChild(element);\n",
+ " }\n",
+ "\n",
+ " \n",
+ " var js_urls = [\"https://cdn.pydata.org/bokeh/release/bokeh-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-widgets-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-tables-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-gl-1.4.0.min.js\"];\n",
+ " var css_urls = [];\n",
+ " \n",
+ "\n",
+ " var inline_js = [\n",
+ " function(Bokeh) {\n",
+ " Bokeh.set_log_level(\"info\");\n",
+ " },\n",
+ " function(Bokeh) {\n",
+ " \n",
+ " \n",
+ " }\n",
+ " ];\n",
+ "\n",
+ " function run_inline_js() {\n",
+ " \n",
+ " if (root.Bokeh !== undefined || force === true) {\n",
+ " \n",
+ " for (var i = 0; i < inline_js.length; i++) {\n",
+ " inline_js[i].call(root, root.Bokeh);\n",
+ " }\n",
+ " } else if (Date.now() < root._bokeh_timeout) {\n",
+ " setTimeout(run_inline_js, 100);\n",
+ " } else if (!root._bokeh_failed_load) {\n",
+ " console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n",
+ " root._bokeh_failed_load = true;\n",
+ " } else if (force !== true) {\n",
+ " var cell = $(document.getElementById(null)).parents('.cell').data().cell;\n",
+ " cell.output_area.append_execute_result(NB_LOAD_WARNING)\n",
+ " }\n",
+ "\n",
+ " }\n",
+ "\n",
+ " if (root._bokeh_is_loading === 0) {\n",
+ " console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n",
+ " run_inline_js();\n",
+ " } else {\n",
+ " load_libs(css_urls, js_urls, function() {\n",
+ " console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n",
+ " run_inline_js();\n",
+ " });\n",
+ " }\n",
+ "}(window));"
+ ],
+ "application/vnd.bokehjs_load.v0+json": "\n(function(root) {\n function now() {\n return new Date();\n }\n\n var force = true;\n\n if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n root._bokeh_onload_callbacks = [];\n root._bokeh_is_loading = undefined;\n }\n\n \n\n \n if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n root._bokeh_timeout = Date.now() + 5000;\n root._bokeh_failed_load = false;\n }\n\n var NB_LOAD_WARNING = {'data': {'text/html':\n \"
\\n\"+\n \"
\\n\"+\n \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n \"
\\n\"+\n \"
\\n\"+\n \"
re-rerun `output_notebook()` to attempt to load from CDN again, or
"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "U5POcTVNB_dv"
+ },
+ "source": [
+ "# HRNet based model for semantic segmentation\n",
+ "\n",
+ "In this notebook, you will:\n",
+ "\n",
+ "- Choose and load one of the 17 pre-trained HRNet models on different semantic segmentation datasets\n",
+ "- Run inference to extract features from the model backbone and predictions from the model head"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "_XgTpm9ZxoN9"
+ },
+ "outputs": [],
+ "source": [
+ "import tensorflow as tf\n",
+ "import tensorflow_hub as hub\n",
+ "import matplotlib.pyplot as plt\n",
+ "from PIL import Image\n",
+ "import numpy as np"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "UVtEyxDFpKE1"
+ },
+ "source": [
+ "## Loading models from TensorFlow Hub\n",
+ "\n",
+ "Here you can choose the pre-trained HRNet model to load, different models means a different training dataset used. All models have the same architecture, except for the model head, which has a different dimension based on the number of classes contained in the training dataset (dataset_output_classes). For more information about the different datasets we refer to the links above and the [factors of influence dataset collection](https://github.com/google-research/google-research/tree/master/factors_of_influence)."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "y8_ctG55-uTX"
+ },
+ "outputs": [],
+ "source": [
+ "#@title Choose a pre-trained HRNet model to load.\n",
+ "\n",
+ "hrnet_model_name = 'ade20k-hrnetv2-w48/1' #@param [\"ade20k-hrnetv2-w48/1\", \"isprs-hrnetv2-w48/1\", \"vkitti2-hrnetv2-w48/1\", \"vgallery-hrnetv2-w48/1\", \"sunrgbd-hrnetv2-w48/1\", \"suim-hrnetv2-w48/1\", \"scannet-hrnetv2-w48/1\", \"pvoc-hrnetv2-w48/1\", \"msegpcontext-hrnetv2-w48/1\", \"mapillary-hrnetv2-w48/1\", \"kitti-hrnetv2-w48/1\", \"isaid-hrnetv2-w48/1\", \"idd-hrnetv2-w48/1\", \"coco-hrnetv2-w48/1\", \"city-hrnetv2-w48/1\", \"camvid-hrnetv2-w48/1\", \"bdd-hrnetv2-w48/1\"]\n",
+ "\n",
+ "tfhub_model_name = 'https://tfhub.dev/google/HRNet/' + hrnet_model_name\n",
+ "\n",
+ "print('HRNet model selected :', tfhub_model_name)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "T-yHJ5X55kWN"
+ },
+ "outputs": [],
+ "source": [
+ "hrnet_model = hub.load(tfhub_model_name)\n",
+ "\n",
+ "print('HRNet model loaded :', tfhub_model_name)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "pMP_7v9x6kol"
+ },
+ "source": [
+ "## Loading an image and running inference\n",
+ "\n",
+ "This is a demonstration on how to run inference for extracting features and predictions from an image. The image was taken from the scene150 dataset.\n",
+ "\n",
+ "To perform inference on the datasets that were used during training we refer to the [factors of influence dataset collection](https://github.com/google-research/google-research/tree/master/factors_of_influence)."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "GNzjieS66td_"
+ },
+ "outputs": [],
+ "source": [
+ "img_file = tf.keras.utils.get_file(origin=\"https://tensorflow.org/images/bedroom_hrnet_tutorial.jpg\")\n",
+ "img = np.array(Image.open(img_file))/255.0"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "Lp54vD_FZuHw"
+ },
+ "outputs": [],
+ "source": [
+ "plt.imshow(img)\n",
+ "plt.show()\n",
+ "\n",
+ "# Predictions will have shape (batch_size, h, w, dataset_output_classes)\n",
+ "predictions = hrnet_model.predict([img])\n",
+ "plt.imshow(predictions[0,:,:,1])\n",
+ "plt.title('Predictions for class #1')\n",
+ "plt.show() \n",
+ "# Features will have shape (batch_size, h/4, w/4, 720)\n",
+ "features = hrnet_model.get_features([img])\n",
+ "plt.imshow(features[0,:,:,1])\n",
+ "plt.title('Feature #1 out of 720')\n",
+ "plt.show()"
+ ]
+ }
+ ],
+ "metadata": {
+ "colab": {
+ "collapsed_sections": [],
+ "name": "hrnet_semantic_segmentation.ipynb",
+ "toc_visible": true
+ },
+ "kernelspec": {
+ "display_name": "Python 3",
+ "name": "python3"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}
diff --git a/site/en/hub/tutorials/image_classification.ipynb b/site/en/hub/tutorials/image_classification.ipynb
new file mode 100644
index 00000000000..91aadab727e
--- /dev/null
+++ b/site/en/hub/tutorials/image_classification.ipynb
@@ -0,0 +1,463 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "ScitaPqhKtuW"
+ },
+ "source": [
+ "##### Copyright 2021 The TensorFlow Hub Authors.\n",
+ "\n",
+ "Licensed under the Apache License, Version 2.0 (the \"License\");"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "jvztxQ6VsK2k"
+ },
+ "outputs": [],
+ "source": [
+ "# Copyright 2021 The TensorFlow Hub Authors. All Rights Reserved.\n",
+ "#\n",
+ "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+ "# you may not use this file except in compliance with the License.\n",
+ "# You may obtain a copy of the License at\n",
+ "#\n",
+ "# http://www.apache.org/licenses/LICENSE-2.0\n",
+ "#\n",
+ "# Unless required by applicable law or agreed to in writing, software\n",
+ "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+ "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+ "# See the License for the specific language governing permissions and\n",
+ "# limitations under the License.\n",
+ "# =============================================================================="
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "7t7KGfIwHaXz"
+ },
+ "source": [
+ "# Image Classification with TensorFlow Hub\n",
+ "\n",
+ "In this colab, you'll try multiple image classification models from TensorFlow Hub and decide which one is best for your use case.\n",
+ "\n",
+ "Because TF Hub encourages a [consistent input convention](https://www.tensorflow.org/hub/common_saved_model_apis/images#image_input) for models that operate on images, it's easy to experiment with different architectures to find the one that best fits your needs."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "MfBg1C5NB3X0"
+ },
+ "source": [
+ "
"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "gh-LWtlqLtgH"
+ },
+ "source": [
+ "Have you ever seen a beautiful flower and wondered what kind of flower it is? Well, you're not the first, so let's build a way to identify the type of flower from a photo!\n",
+ "\n",
+ "For classifying images, a particular type of *deep neural network*, called a *convolutional neural network* has proved to be particularly powerful. However, modern convolutional neural networks have millions of parameters. Training them from scratch requires a lot of labeled training data and a lot of computing power (hundreds of GPU-hours or more). We only have about three thousand labeled photos and want to spend much less time, so we need to be more clever.\n",
+ "\n",
+ "We will use a technique called *transfer learning* where we take a pre-trained network (trained on about a million general images), use it to extract features, and train a new layer on top for our own task of classifying images of flowers.\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "ORy-KvWXGXBo"
+ },
+ "source": [
+ "## Setup\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "NTrs9zBKJK1c"
+ },
+ "outputs": [],
+ "source": [
+ "import collections\n",
+ "import io\n",
+ "import math\n",
+ "import os\n",
+ "import random\n",
+ "from six.moves import urllib\n",
+ "\n",
+ "from IPython.display import clear_output, Image, display, HTML\n",
+ "\n",
+ "import tensorflow.compat.v1 as tf\n",
+ "tf.disable_v2_behavior()\n",
+ "\n",
+ "import tensorflow_hub as hub\n",
+ "\n",
+ "import numpy as np\n",
+ "import matplotlib.pyplot as plt\n",
+ "import seaborn as sns\n",
+ "import sklearn.metrics as sk_metrics\n",
+ "import time"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "Do-T63G7NCSB"
+ },
+ "source": [
+ "## The flowers dataset\n",
+ "\n",
+ "The flowers dataset consists of images of flowers with 5 possible class labels.\n",
+ "\n",
+ "When training a machine learning model, we split our data into training and test datasets. We will train the model on our training data and then evaluate how well the model performs on data it has never seen - the test set.\n",
+ "\n",
+ "Let's download our training and test examples (it may take a while) and split them into train and test sets.\n",
+ "\n",
+ "Run the following two cells:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "cellView": "both",
+ "id": "HYQr1SILIxSK"
+ },
+ "outputs": [],
+ "source": [
+ "FLOWERS_DIR = './flower_photos'\n",
+ "TRAIN_FRACTION = 0.8\n",
+ "RANDOM_SEED = 2018\n",
+ "\n",
+ "\n",
+ "def download_images():\n",
+ " \"\"\"If the images aren't already downloaded, save them to FLOWERS_DIR.\"\"\"\n",
+ " if not os.path.exists(FLOWERS_DIR):\n",
+ " DOWNLOAD_URL = 'http://download.tensorflow.org/example_images/flower_photos.tgz'\n",
+ " print('Downloading flower images from %s...' % DOWNLOAD_URL)\n",
+ " urllib.request.urlretrieve(DOWNLOAD_URL, 'flower_photos.tgz')\n",
+ " !tar xfz flower_photos.tgz\n",
+ " print('Flower photos are located in %s' % FLOWERS_DIR)\n",
+ "\n",
+ "\n",
+ "def make_train_and_test_sets():\n",
+ " \"\"\"Split the data into train and test sets and get the label classes.\"\"\"\n",
+ " train_examples, test_examples = [], []\n",
+ " shuffler = random.Random(RANDOM_SEED)\n",
+ " is_root = True\n",
+ " for (dirname, subdirs, filenames) in tf.gfile.Walk(FLOWERS_DIR):\n",
+ " # The root directory gives us the classes\n",
+ " if is_root:\n",
+ " subdirs = sorted(subdirs)\n",
+ " classes = collections.OrderedDict(enumerate(subdirs))\n",
+ " label_to_class = dict([(x, i) for i, x in enumerate(subdirs)])\n",
+ " is_root = False\n",
+ " # The sub directories give us the image files for training.\n",
+ " else:\n",
+ " filenames.sort()\n",
+ " shuffler.shuffle(filenames)\n",
+ " full_filenames = [os.path.join(dirname, f) for f in filenames]\n",
+ " label = dirname.split('/')[-1]\n",
+ " label_class = label_to_class[label]\n",
+ " # An example is the image file and it's label class.\n",
+ " examples = list(zip(full_filenames, [label_class] * len(filenames)))\n",
+ " num_train = int(len(filenames) * TRAIN_FRACTION)\n",
+ " train_examples.extend(examples[:num_train])\n",
+ " test_examples.extend(examples[num_train:])\n",
+ "\n",
+ " shuffler.shuffle(train_examples)\n",
+ " shuffler.shuffle(test_examples)\n",
+ " return train_examples, test_examples, classes\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "_9NklpcANhtB"
+ },
+ "outputs": [],
+ "source": [
+ "# Download the images and split the images into train and test sets.\n",
+ "download_images()\n",
+ "TRAIN_EXAMPLES, TEST_EXAMPLES, CLASSES = make_train_and_test_sets()\n",
+ "NUM_CLASSES = len(CLASSES)\n",
+ "\n",
+ "print('\\nThe dataset has %d label classes: %s' % (NUM_CLASSES, CLASSES.values()))\n",
+ "print('There are %d training images' % len(TRAIN_EXAMPLES))\n",
+ "print('there are %d test images' % len(TEST_EXAMPLES))"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "tHF7bHTfnD6S"
+ },
+ "source": [
+ "## Explore the data\n",
+ "\n",
+ "The flowers dataset consists of examples which are labeled images of flowers. Each example contains a JPEG flower image and the class label: what type of flower it is. Let's display a few images together with their labels."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "cellView": "both",
+ "id": "1friUvN6kPYM"
+ },
+ "outputs": [],
+ "source": [
+ "#@title Show some labeled images\n",
+ "def get_label(example):\n",
+ " \"\"\"Get the label (number) for given example.\"\"\"\n",
+ " return example[1]\n",
+ "\n",
+ "def get_class(example):\n",
+ " \"\"\"Get the class (string) of given example.\"\"\"\n",
+ " return CLASSES[get_label(example)]\n",
+ "\n",
+ "def get_encoded_image(example):\n",
+ " \"\"\"Get the image data (encoded jpg) of given example.\"\"\"\n",
+ " image_path = example[0]\n",
+ " return tf.gfile.GFile(image_path, 'rb').read()\n",
+ "\n",
+ "def get_image(example):\n",
+ " \"\"\"Get image as np.array of pixels for given example.\"\"\"\n",
+ " return plt.imread(io.BytesIO(get_encoded_image(example)), format='jpg')\n",
+ "\n",
+ "def display_images(images_and_classes, cols=5):\n",
+ " \"\"\"Display given images and their labels in a grid.\"\"\"\n",
+ " rows = int(math.ceil(len(images_and_classes) / cols))\n",
+ " fig = plt.figure()\n",
+ " fig.set_size_inches(cols * 3, rows * 3)\n",
+ " for i, (image, flower_class) in enumerate(images_and_classes):\n",
+ " plt.subplot(rows, cols, i + 1)\n",
+ " plt.axis('off')\n",
+ " plt.imshow(image)\n",
+ " plt.title(flower_class)\n",
+ "\n",
+ "NUM_IMAGES = 15 #@param {type: 'integer'}\n",
+ "display_images([(get_image(example), get_class(example))\n",
+ " for example in TRAIN_EXAMPLES[:NUM_IMAGES]])"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "Hyjr6PuboTAg"
+ },
+ "source": [
+ "## Build the model\n",
+ "\n",
+ "We will load a [TF-Hub](https://tensorflow.org/hub) image feature vector module, stack a linear classifier on it, and add training and evaluation ops. The following cell builds a TF graph describing the model and its training, but it doesn't run the training (that will be the next step)."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "LbkSRaK_oW5Y"
+ },
+ "outputs": [],
+ "source": [
+ "LEARNING_RATE = 0.01\n",
+ "\n",
+ "tf.reset_default_graph()\n",
+ "\n",
+ "# Load a pre-trained TF-Hub module for extracting features from images. We've\n",
+ "# chosen this particular module for speed, but many other choices are available.\n",
+ "image_module = hub.Module('https://tfhub.dev/google/imagenet/mobilenet_v2_035_128/feature_vector/2')\n",
+ "\n",
+ "# Preprocessing images into tensors with size expected by the image module.\n",
+ "encoded_images = tf.placeholder(tf.string, shape=[None])\n",
+ "image_size = hub.get_expected_image_size(image_module)\n",
+ "\n",
+ "\n",
+ "def decode_and_resize_image(encoded):\n",
+ " decoded = tf.image.decode_jpeg(encoded, channels=3)\n",
+ " decoded = tf.image.convert_image_dtype(decoded, tf.float32)\n",
+ " return tf.image.resize_images(decoded, image_size)\n",
+ "\n",
+ "\n",
+ "batch_images = tf.map_fn(decode_and_resize_image, encoded_images, dtype=tf.float32)\n",
+ "\n",
+ "# The image module can be applied as a function to extract feature vectors for a\n",
+ "# batch of images.\n",
+ "features = image_module(batch_images)\n",
+ "\n",
+ "\n",
+ "def create_model(features):\n",
+ " \"\"\"Build a model for classification from extracted features.\"\"\"\n",
+ " # Currently, the model is just a single linear layer. You can try to add\n",
+ " # another layer, but be careful... two linear layers (when activation=None)\n",
+ " # are equivalent to a single linear layer. You can create a nonlinear layer\n",
+ " # like this:\n",
+ " # layer = tf.layers.dense(inputs=..., units=..., activation=tf.nn.relu)\n",
+ " layer = tf.layers.dense(inputs=features, units=NUM_CLASSES, activation=None)\n",
+ " return layer\n",
+ "\n",
+ "\n",
+ "# For each class (kind of flower), the model outputs some real number as a score\n",
+ "# how much the input resembles this class. This vector of numbers is often\n",
+ "# called the \"logits\".\n",
+ "logits = create_model(features)\n",
+ "labels = tf.placeholder(tf.float32, [None, NUM_CLASSES])\n",
+ "\n",
+ "# Mathematically, a good way to measure how much the predicted probabilities\n",
+ "# diverge from the truth is the \"cross-entropy\" between the two probability\n",
+ "# distributions. For numerical stability, this is best done directly from the\n",
+ "# logits, not the probabilities extracted from them.\n",
+ "cross_entropy = tf.nn.softmax_cross_entropy_with_logits_v2(logits=logits, labels=labels)\n",
+ "cross_entropy_mean = tf.reduce_mean(cross_entropy)\n",
+ "\n",
+ "# Let's add an optimizer so we can train the network.\n",
+ "optimizer = tf.train.GradientDescentOptimizer(learning_rate=LEARNING_RATE)\n",
+ "train_op = optimizer.minimize(loss=cross_entropy_mean)\n",
+ "\n",
+ "# The \"softmax\" function transforms the logits vector into a vector of\n",
+ "# probabilities: non-negative numbers that sum up to one, and the i-th number\n",
+ "# says how likely the input comes from class i.\n",
+ "probabilities = tf.nn.softmax(logits)\n",
+ "\n",
+ "# We choose the highest one as the predicted class.\n",
+ "prediction = tf.argmax(probabilities, 1)\n",
+ "correct_prediction = tf.equal(prediction, tf.argmax(labels, 1))\n",
+ "\n",
+ "# The accuracy will allow us to eval on our test set. \n",
+ "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "0vvhYQ7-3AG_"
+ },
+ "source": [
+ "## Train the network\n",
+ "\n",
+ "Now that our model is built, let's train it and see how it performs on our test set."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "1YnBg7-OS3Dz"
+ },
+ "outputs": [],
+ "source": [
+ "# How long will we train the network (number of batches).\n",
+ "NUM_TRAIN_STEPS = 100 #@param {type: 'integer'}\n",
+ "# How many training examples we use in each step.\n",
+ "TRAIN_BATCH_SIZE = 10 #@param {type: 'integer'}\n",
+ "# How often to evaluate the model performance.\n",
+ "EVAL_EVERY = 10 #@param {type: 'integer'}\n",
+ "\n",
+ "def get_batch(batch_size=None, test=False):\n",
+ " \"\"\"Get a random batch of examples.\"\"\"\n",
+ " examples = TEST_EXAMPLES if test else TRAIN_EXAMPLES\n",
+ " batch_examples = random.sample(examples, batch_size) if batch_size else examples\n",
+ " return batch_examples\n",
+ "\n",
+ "def get_images_and_labels(batch_examples):\n",
+ " images = [get_encoded_image(e) for e in batch_examples]\n",
+ " one_hot_labels = [get_label_one_hot(e) for e in batch_examples]\n",
+ " return images, one_hot_labels\n",
+ "\n",
+ "def get_label_one_hot(example):\n",
+ " \"\"\"Get the one hot encoding vector for the example.\"\"\"\n",
+ " one_hot_vector = np.zeros(NUM_CLASSES)\n",
+ " np.put(one_hot_vector, get_label(example), 1)\n",
+ " return one_hot_vector\n",
+ "\n",
+ "with tf.Session() as sess:\n",
+ " sess.run(tf.global_variables_initializer())\n",
+ " for i in range(NUM_TRAIN_STEPS):\n",
+ " # Get a random batch of training examples.\n",
+ " train_batch = get_batch(batch_size=TRAIN_BATCH_SIZE)\n",
+ " batch_images, batch_labels = get_images_and_labels(train_batch)\n",
+ " # Run the train_op to train the model.\n",
+ " train_loss, _, train_accuracy = sess.run(\n",
+ " [cross_entropy_mean, train_op, accuracy],\n",
+ " feed_dict={encoded_images: batch_images, labels: batch_labels})\n",
+ " is_final_step = (i == (NUM_TRAIN_STEPS - 1))\n",
+ " if i % EVAL_EVERY == 0 or is_final_step:\n",
+ " # Get a batch of test examples.\n",
+ " test_batch = get_batch(batch_size=None, test=True)\n",
+ " batch_images, batch_labels = get_images_and_labels(test_batch)\n",
+ " # Evaluate how well our model performs on the test set.\n",
+ " test_loss, test_accuracy, test_prediction, correct_predicate = sess.run(\n",
+ " [cross_entropy_mean, accuracy, prediction, correct_prediction],\n",
+ " feed_dict={encoded_images: batch_images, labels: batch_labels})\n",
+ " print('Test accuracy at step %s: %.2f%%' % (i, (test_accuracy * 100)))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "ZFUNJxuH2t0V"
+ },
+ "outputs": [],
+ "source": [
+ "def show_confusion_matrix(test_labels, predictions):\n",
+ " \"\"\"Compute confusion matrix and normalize.\"\"\"\n",
+ " confusion = sk_metrics.confusion_matrix(\n",
+ " np.argmax(test_labels, axis=1), predictions)\n",
+ " confusion_normalized = confusion.astype(\"float\") / confusion.sum(axis=1)\n",
+ " axis_labels = list(CLASSES.values())\n",
+ " ax = sns.heatmap(\n",
+ " confusion_normalized, xticklabels=axis_labels, yticklabels=axis_labels,\n",
+ " cmap='Blues', annot=True, fmt='.2f', square=True)\n",
+ " plt.title(\"Confusion matrix\")\n",
+ " plt.ylabel(\"True label\")\n",
+ " plt.xlabel(\"Predicted label\")\n",
+ "\n",
+ "show_confusion_matrix(batch_labels, test_prediction)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "Uu3vo8DK8BdL"
+ },
+ "source": [
+ "## Incorrect predictions\n",
+ "\n",
+ "Let's a take a closer look at the test examples that our model got wrong.\n",
+ "\n",
+ "- Are there any mislabeled examples in our test set?\n",
+ "- Is there any bad data in the test set - images that aren't actually pictures of flowers?\n",
+ "- Are there images where you can understand why the model made a mistake?"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "hqa0V3WN8C9M"
+ },
+ "outputs": [],
+ "source": [
+ "incorrect = [\n",
+ " (example, CLASSES[prediction])\n",
+ " for example, prediction, is_correct in zip(test_batch, test_prediction, correct_predicate)\n",
+ " if not is_correct\n",
+ "]\n",
+ "display_images(\n",
+ " [(get_image(example), \"prediction: {0}\\nlabel:{1}\".format(incorrect_prediction, get_class(example)))\n",
+ " for (example, incorrect_prediction) in incorrect[:20]])"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "YN_s04Il8TvK"
+ },
+ "source": [
+ "## Exercises: Improve the model!\n",
+ "\n",
+ "We've trained a baseline model, now let's try to improve it to achieve better accuracy. (Remember that you'll need to re-run the cells when you make a change.)\n",
+ "\n",
+ "### Exercise 1: Try a different image model.\n",
+ "With TF-Hub, trying a few different image models is simple. Just replace the `\"https://tfhub.dev/google/imagenet/mobilenet_v2_050_128/feature_vector/2\"` handle in the `hub.Module()` call with a handle of different module and rerun all the code. You can see all available image modules at [tfhub.dev](https://tfhub.dev/s?module-type=image-feature-vector). \n",
+ "\n",
+ "A good choice might be one of the other [MobileNet V2 modules](https://tfhub.dev/s?module-type=image-feature-vector&network-architecture=mobilenet-v2). Many of the modules -- including the MobileNet modules -- were trained on the [ImageNet dataset](https://www.tensorflow.org/datasets/catalog/imagenet2012) which contains over 1 million images and 1000 classes. Choosing a network architecture provides a tradeoff between speed and classification accuracy: models like MobileNet or NASNet Mobile are fast and small, more traditional architectures like Inception and ResNet were designed for accuracy.\n",
+ "\n",
+ "For the larger Inception V3 architecture, you can also explore the benefits of pre-training on a domain closer to your own task: it is also available as a [module trained on the iNaturalist dataset](https://tfhub.dev/google/inaturalist/inception_v3/feature_vector/1) of plants and animals.\n",
+ "\n",
+ "### Exercise 2: Add a hidden layer.\n",
+ "Stack a hidden layer between extracted image features and the linear classifier (in function `create_model()` above). To create a non-linear hidden layer with e.g. 100 nodes, use [tf.layers.dense](https://www.tensorflow.org/api_docs/python/tf/compat/v1/layers/dense) with units set to 100 and activation set to `tf.nn.relu`. Does changing the size of the hidden layer affect the test accuracy? Does adding second hidden layer improve the accuracy?\n",
+ "\n",
+ "### Exercise 3: Change hyperparameters.\n",
+ "Does increasing *number of training steps* improves final accuracy? Can you *change the learning rate* to make your model converge more quickly? Does the training *batch size* affect your model's performance?\n",
+ "\n",
+ "### Exercise 4: Try a different optimizer.\n",
+ "\n",
+ "Replace the basic GradientDescentOptimizer with a more sophisticate optimizer, e.g. [AdagradOptimizer](https://www.tensorflow.org/api_docs/python/tf/compat/v1/train/AdagradOptimizer). Does it make a difference to your model training? If you want to learn more about the benefits of different optimization algorithms, check out [this post](http://ruder.io/optimizing-gradient-descent/)."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "kdwVXO1eJS5-"
+ },
+ "source": [
+ "## Want to learn more?\n",
+ "\n",
+ "If you are interested in a more advanced version of this tutorial, check out the [TensorFlow image retraining tutorial](https://www.tensorflow.org/hub/tutorials/image_retraining) which walks you through visualizing the training using TensorBoard, advanced techniques like dataset augmentation by distorting images, and replacing the flowers dataset to learn an image classifier on your own dataset.\n",
+ "\n",
+ "You can learn more about TensorFlow at [tensorflow.org](http://tensorflow.org) and see the TF-Hub API documentation is available at [tensorflow.org/hub](https://www.tensorflow.org/hub/). Find available TensorFlow Hub modules at [tfhub.dev](http://tfhub.dev) including more image feature vector modules and text embedding modules.\n",
+ "\n",
+ "Also check out the [Machine Learning Crash Course](https://developers.google.com/machine-learning/crash-course/) which is Google's fast-paced, practical introduction to machine learning."
+ ]
+ }
+ ],
+ "metadata": {
+ "colab": {
+ "collapsed_sections": [
+ "ScitaPqhKtuW"
+ ],
+ "name": "image_feature_vector.ipynb",
+ "toc_visible": true
+ },
+ "kernelspec": {
+ "display_name": "Python 3",
+ "name": "python3"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}
diff --git a/site/en/hub/tutorials/movenet.ipynb b/site/en/hub/tutorials/movenet.ipynb
new file mode 100644
index 00000000000..f7955a5253b
--- /dev/null
+++ b/site/en/hub/tutorials/movenet.ipynb
@@ -0,0 +1,816 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "toCy3v03Dwx7"
+ },
+ "source": [
+ "##### Copyright 2021 The TensorFlow Hub Authors.\n",
+ "\n",
+ "Licensed under the Apache License, Version 2.0 (the \"License\");"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "QKe-ubNcDvgv"
+ },
+ "outputs": [],
+ "source": [
+ "# Copyright 2021 The TensorFlow Hub Authors. All Rights Reserved.\n",
+ "#\n",
+ "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+ "# you may not use this file except in compliance with the License.\n",
+ "# You may obtain a copy of the License at\n",
+ "#\n",
+ "# http://www.apache.org/licenses/LICENSE-2.0\n",
+ "#\n",
+ "# Unless required by applicable law or agreed to in writing, software\n",
+ "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+ "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+ "# See the License for the specific language governing permissions and\n",
+ "# limitations under the License.\n",
+ "# =============================================================================="
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "KqtQzBCpIJ7Y"
+ },
+ "source": [
+ "# MoveNet: Ultra fast and accurate pose detection model."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "MCmFOosnSkCd"
+ },
+ "source": [
+ "
"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "bC_xJPpQd-LO"
+ },
+ "outputs": [],
+ "source": [
+ "!pip install -q opencv-python\n",
+ "\n",
+ "import os\n",
+ "\n",
+ "import tensorflow.compat.v2 as tf\n",
+ "import tensorflow_hub as hub\n",
+ "\n",
+ "import numpy as np\n",
+ "import cv2\n",
+ "from IPython import display\n",
+ "import math"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "ZxwaK-jf7qkW"
+ },
+ "source": [
+ "## Import TF-Hub model\n",
+ "\n",
+ "This tutorial demonstrates how to use the [S3D MIL-NCE model](https://tfhub.dev/deepmind/mil-nce/s3d/1) from TensorFlow Hub to do **text-to-video retrieval** to find the most similar videos for a given text query.\n",
+ "\n",
+ "The model has 2 signatures, one for generating *video embeddings* and one for generating *text embeddings*. We will use these embedding to find the nearest neighbors in the embedding space."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "nwv4ZQ4qmak5"
+ },
+ "outputs": [],
+ "source": [
+ "# Load the model once from TF-Hub.\n",
+ "hub_handle = 'https://tfhub.dev/deepmind/mil-nce/s3d/1'\n",
+ "hub_model = hub.load(hub_handle)\n",
+ "\n",
+ "def generate_embeddings(model, input_frames, input_words):\n",
+ " \"\"\"Generate embeddings from the model from video frames and input words.\"\"\"\n",
+ " # Input_frames must be normalized in [0, 1] and of the shape Batch x T x H x W x 3\n",
+ " vision_output = model.signatures['video'](tf.constant(tf.cast(input_frames, dtype=tf.float32)))\n",
+ " text_output = model.signatures['text'](tf.constant(input_words))\n",
+ " return vision_output['video_embedding'], text_output['text_embedding']"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "EOZzu9ddekEj"
+ },
+ "outputs": [],
+ "source": [
+ "# @title Define video loading and visualization functions { display-mode: \"form\" }\n",
+ "\n",
+ "# Utilities to open video files using CV2\n",
+ "def crop_center_square(frame):\n",
+ " y, x = frame.shape[0:2]\n",
+ " min_dim = min(y, x)\n",
+ " start_x = (x // 2) - (min_dim // 2)\n",
+ " start_y = (y // 2) - (min_dim // 2)\n",
+ " return frame[start_y:start_y+min_dim,start_x:start_x+min_dim]\n",
+ "\n",
+ "\n",
+ "def load_video(video_url, max_frames=32, resize=(224, 224)):\n",
+ " path = tf.keras.utils.get_file(os.path.basename(video_url)[-128:], video_url)\n",
+ " cap = cv2.VideoCapture(path)\n",
+ " frames = []\n",
+ " try:\n",
+ " while True:\n",
+ " ret, frame = cap.read()\n",
+ " if not ret:\n",
+ " break\n",
+ " frame = crop_center_square(frame)\n",
+ " frame = cv2.resize(frame, resize)\n",
+ " frame = frame[:, :, [2, 1, 0]]\n",
+ " frames.append(frame)\n",
+ "\n",
+ " if len(frames) == max_frames:\n",
+ " break\n",
+ " finally:\n",
+ " cap.release()\n",
+ " frames = np.array(frames)\n",
+ " if len(frames) < max_frames:\n",
+ " n_repeat = int(math.ceil(max_frames / float(len(frames))))\n",
+ " frames = frames.repeat(n_repeat, axis=0)\n",
+ " frames = frames[:max_frames]\n",
+ " return frames / 255.0\n",
+ "\n",
+ "def display_video(urls):\n",
+ " html = '
'\n",
+ " html += '
Video 1
Video 2
Video 3
'\n",
+ " for url in urls:\n",
+ " html += '
'\n",
+ " html += ''.format(url)\n",
+ " html += '
'\n",
+ " html += '
'\n",
+ " return display.HTML(html)\n",
+ "\n",
+ "def display_query_and_results_video(query, urls, scores):\n",
+ " \"\"\"Display a text query and the top result videos and scores.\"\"\"\n",
+ " sorted_ix = np.argsort(-scores)\n",
+ " html = ''\n",
+ " html += '
Input query: {}
'.format(query)\n",
+ " html += 'Results:
'\n",
+ " html += '
'\n",
+ " html += '
Rank #1, Score:{:.2f}
'.format(scores[sorted_ix[0]])\n",
+ " html += '
Rank #2, Score:{:.2f}
'.format(scores[sorted_ix[1]])\n",
+ " html += '
Rank #3, Score:{:.2f}
'.format(scores[sorted_ix[2]])\n",
+ " for i, idx in enumerate(sorted_ix):\n",
+ " url = urls[sorted_ix[i]];\n",
+ " html += '
'\n",
+ " html += ''.format(url)\n",
+ " html += '
'\n",
+ " html += '
'\n",
+ " return html\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "Ime5V4kDewh8"
+ },
+ "outputs": [],
+ "source": [
+ "# @title Load example videos and define text queries { display-mode: \"form\" }\n",
+ "\n",
+ "video_1_url = 'https://upload.wikimedia.org/wikipedia/commons/b/b0/YosriAirTerjun.gif' # @param {type:\"string\"}\n",
+ "video_2_url = 'https://upload.wikimedia.org/wikipedia/commons/e/e6/Guitar_solo_gif.gif' # @param {type:\"string\"}\n",
+ "video_3_url = 'https://upload.wikimedia.org/wikipedia/commons/3/30/2009-08-16-autodrift-by-RalfR-gif-by-wau.gif' # @param {type:\"string\"}\n",
+ "\n",
+ "video_1 = load_video(video_1_url)\n",
+ "video_2 = load_video(video_2_url)\n",
+ "video_3 = load_video(video_3_url)\n",
+ "all_videos = [video_1, video_2, video_3]\n",
+ "\n",
+ "query_1_video = 'waterfall' # @param {type:\"string\"}\n",
+ "query_2_video = 'playing guitar' # @param {type:\"string\"}\n",
+ "query_3_video = 'car drifting' # @param {type:\"string\"}\n",
+ "all_queries_video = [query_1_video, query_2_video, query_3_video]\n",
+ "all_videos_urls = [video_1_url, video_2_url, video_3_url]\n",
+ "display_video(all_videos_urls)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "NCLKv_L_8Anc"
+ },
+ "source": [
+ "## Demonstrate text to video retrieval\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "9oX8ItFUjybi"
+ },
+ "outputs": [],
+ "source": [
+ "# Prepare video inputs.\n",
+ "videos_np = np.stack(all_videos, axis=0)\n",
+ "\n",
+ "# Prepare text input.\n",
+ "words_np = np.array(all_queries_video)\n",
+ "\n",
+ "# Generate the video and text embeddings.\n",
+ "video_embd, text_embd = generate_embeddings(hub_model, videos_np, words_np)\n",
+ "\n",
+ "# Scores between video and text is computed by dot products.\n",
+ "all_scores = np.dot(text_embd, tf.transpose(video_embd))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "d4AwYmODmE9Y"
+ },
+ "outputs": [],
+ "source": [
+ "# Display results.\n",
+ "html = ''\n",
+ "for i, words in enumerate(words_np):\n",
+ " html += display_query_and_results_video(words, all_videos_urls, all_scores[i, :])\n",
+ " html += ' '\n",
+ "display.HTML(html)"
+ ]
+ }
+ ],
+ "metadata": {
+ "colab": {
+ "collapsed_sections": [],
+ "name": "text_to_video_retrieval_with_s3d_milnce.ipynb",
+ "toc_visible": true
+ },
+ "kernelspec": {
+ "display_name": "Python 3",
+ "name": "python3"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}
diff --git a/site/en/hub/tutorials/tf2_arbitrary_image_stylization.ipynb b/site/en/hub/tutorials/tf2_arbitrary_image_stylization.ipynb
new file mode 100644
index 00000000000..3a0cb09113e
--- /dev/null
+++ b/site/en/hub/tutorials/tf2_arbitrary_image_stylization.ipynb
@@ -0,0 +1,375 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "ScitaPqhKtuW"
+ },
+ "source": [
+ "##### Copyright 2019 The TensorFlow Hub Authors.\n",
+ "\n",
+ "Licensed under the Apache License, Version 2.0 (the \"License\");"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "jvztxQ6VsK2k"
+ },
+ "outputs": [],
+ "source": [
+ "# Copyright 2019 The TensorFlow Hub Authors. All Rights Reserved.\n",
+ "#\n",
+ "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+ "# you may not use this file except in compliance with the License.\n",
+ "# You may obtain a copy of the License at\n",
+ "#\n",
+ "# http://www.apache.org/licenses/LICENSE-2.0\n",
+ "#\n",
+ "# Unless required by applicable law or agreed to in writing, software\n",
+ "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+ "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+ "# See the License for the specific language governing permissions and\n",
+ "# limitations under the License.\n",
+ "# =============================================================================="
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "oXlcl8lqBgAD"
+ },
+ "source": [
+ "# Fast Style Transfer for Arbitrary Styles\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "MfBg1C5NB3X0"
+ },
+ "source": [
+ "
"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "Eg62Pmz3o83v"
+ },
+ "source": [
+ "This notebook classifies movie reviews as *positive* or *negative* using the text of the review. This is an example of *binary*—or two-class—classification, an important and widely applicable kind of machine learning problem. \n",
+ "\n",
+ "We'll use the [IMDB dataset](https://www.tensorflow.org/api_docs/python/tf/keras/datasets/imdb) that contains the text of 50,000 movie reviews from the [Internet Movie Database](https://www.imdb.com/). These are split into 25,000 reviews for training and 25,000 reviews for testing. The training and testing sets are *balanced*, meaning they contain an equal number of positive and negative reviews. \n",
+ "\n",
+ "This notebook uses [tf.keras](https://www.tensorflow.org/api_docs/python/tf/keras), a high-level API to build and train models in TensorFlow, and [TensorFlow Hub](https://www.tensorflow.org/hub), a library and platform for transfer learning. For a more advanced text classification tutorial using `tf.keras`, see the [MLCC Text Classification Guide](https://developers.google.com/machine-learning/guides/text-classification/)."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "qrk8NjzhSBh-"
+ },
+ "source": [
+ "### More models\n",
+ "[Here](https://tfhub.dev/s?module-type=text-embedding) you can find more expressive or performant models that you could use to generate the text embedding."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "Q4DN769E2O_R"
+ },
+ "source": [
+ "## Setup"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "2ew7HTbPpCJH"
+ },
+ "outputs": [],
+ "source": [
+ "import numpy as np\n",
+ "\n",
+ "import tensorflow as tf\n",
+ "import tensorflow_hub as hub\n",
+ "import tensorflow_datasets as tfds\n",
+ "\n",
+ "import matplotlib.pyplot as plt\n",
+ "\n",
+ "print(\"Version: \", tf.__version__)\n",
+ "print(\"Eager mode: \", tf.executing_eagerly())\n",
+ "print(\"Hub version: \", hub.__version__)\n",
+ "print(\"GPU is\", \"available\" if tf.config.list_physical_devices('GPU') else \"NOT AVAILABLE\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "iAsKG535pHep"
+ },
+ "source": [
+ "## Download the IMDB dataset\n",
+ "\n",
+ "The IMDB dataset is available on [TensorFlow datasets](https://github.com/tensorflow/datasets). The following code downloads the IMDB dataset to your machine (or the colab runtime):"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "zXXx5Oc3pOmN"
+ },
+ "outputs": [],
+ "source": [
+ "train_data, test_data = tfds.load(name=\"imdb_reviews\", split=[\"train\", \"test\"], \n",
+ " batch_size=-1, as_supervised=True)\n",
+ "\n",
+ "train_examples, train_labels = tfds.as_numpy(train_data)\n",
+ "test_examples, test_labels = tfds.as_numpy(test_data)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "l50X3GfjpU4r"
+ },
+ "source": [
+ "## Explore the data \n",
+ "\n",
+ "Let's take a moment to understand the format of the data. Each example is a sentence representing the movie review and a corresponding label. The sentence is not preprocessed in any way. The label is an integer value of either 0 or 1, where 0 is a negative review, and 1 is a positive review."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "y8qCnve_-lkO"
+ },
+ "outputs": [],
+ "source": [
+ "print(\"Training entries: {}, test entries: {}\".format(len(train_examples), len(test_examples)))"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "RnKvHWW4-lkW"
+ },
+ "source": [
+ "Let's print first 10 examples."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "QtTS4kpEpjbi"
+ },
+ "outputs": [],
+ "source": [
+ "train_examples[:10]"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "IFtaCHTdc-GY"
+ },
+ "source": [
+ "Let's also print the first 10 labels."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "tvAjVXOWc6Mj"
+ },
+ "outputs": [],
+ "source": [
+ "train_labels[:10]"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "LLC02j2g-llC"
+ },
+ "source": [
+ "## Build the model\n",
+ "\n",
+ "The neural network is created by stacking layers—this requires three main architectural decisions:\n",
+ "\n",
+ "* How to represent the text?\n",
+ "* How many layers to use in the model?\n",
+ "* How many *hidden units* to use for each layer?\n",
+ "\n",
+ "In this example, the input data consists of sentences. The labels to predict are either 0 or 1.\n",
+ "\n",
+ "One way to represent the text is to convert sentences into embeddings vectors. We can use a pre-trained text embedding as the first layer, which will have two advantages:\n",
+ "* we don't have to worry about text preprocessing,\n",
+ "* we can benefit from transfer learning.\n",
+ "\n",
+ "For this example we will use a model from [TensorFlow Hub](https://www.tensorflow.org/hub) called [google/nnlm-en-dim50/2](https://tfhub.dev/google/nnlm-en-dim50/2).\n",
+ "\n",
+ "There are two other models to test for the sake of this tutorial:\n",
+ "* [google/nnlm-en-dim50-with-normalization/2](https://tfhub.dev/google/nnlm-en-dim50-with-normalization/2) - same as [google/nnlm-en-dim50/2](https://tfhub.dev/google/nnlm-en-dim50/2), but with additional text normalization to remove punctuation. This can help to get better coverage of in-vocabulary embeddings for tokens on your input text.\n",
+ "* [google/nnlm-en-dim128-with-normalization/2](https://tfhub.dev/google/nnlm-en-dim128-with-normalization/2) - A larger model with an embedding dimension of 128 instead of the smaller 50."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "In2nDpTLkgKa"
+ },
+ "source": [
+ "Let's first create a Keras layer that uses a TensorFlow Hub model to embed the sentences, and try it out on a couple of input examples. Note that the output shape of the produced embeddings is a expected: `(num_examples, embedding_dimension)`."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "_NUbzVeYkgcO"
+ },
+ "outputs": [],
+ "source": [
+ "model = \"https://tfhub.dev/google/nnlm-en-dim50/2\"\n",
+ "hub_layer = hub.KerasLayer(model, input_shape=[], dtype=tf.string, trainable=True)\n",
+ "hub_layer(train_examples[:3])"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "dfSbV6igl1EH"
+ },
+ "source": [
+ "Let's now build the full model:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "xpKOoWgu-llD"
+ },
+ "outputs": [],
+ "source": [
+ "model = tf.keras.Sequential()\n",
+ "model.add(hub_layer)\n",
+ "model.add(tf.keras.layers.Dense(16, activation='relu'))\n",
+ "model.add(tf.keras.layers.Dense(1))\n",
+ "\n",
+ "model.summary()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "6PbKQ6mucuKL"
+ },
+ "source": [
+ "The layers are stacked sequentially to build the classifier:\n",
+ "\n",
+ "1. The first layer is a TensorFlow Hub layer. This layer uses a pre-trained Saved Model to map a sentence into its embedding vector. The model that we are using ([google/nnlm-en-dim50/2](https://tfhub.dev/google/nnlm-en-dim50/2)) splits the sentence into tokens, embeds each token and then combines the embedding. The resulting dimensions are: `(num_examples, embedding_dimension)`.\n",
+ "2. This fixed-length output vector is piped through a fully-connected (`Dense`) layer with 16 hidden units.\n",
+ "3. The last layer is densely connected with a single output node. This outputs logits: the log-odds of the true class, according to the model."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "0XMwnDOp-llH"
+ },
+ "source": [
+ "### Hidden units\n",
+ "\n",
+ "The above model has two intermediate or \"hidden\" layers, between the input and output. The number of outputs (units, nodes, or neurons) is the dimension of the representational space for the layer. In other words, the amount of freedom the network is allowed when learning an internal representation.\n",
+ "\n",
+ "If a model has more hidden units (a higher-dimensional representation space), and/or more layers, then the network can learn more complex representations. However, it makes the network more computationally expensive and may lead to learning unwanted patterns—patterns that improve performance on training data but not on the test data. This is called *overfitting*, and we'll explore it later."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "L4EqVWg4-llM"
+ },
+ "source": [
+ "### Loss function and optimizer\n",
+ "\n",
+ "A model needs a loss function and an optimizer for training. Since this is a binary classification problem and the model outputs a probability (a single-unit layer with a sigmoid activation), we'll use the `binary_crossentropy` loss function. \n",
+ "\n",
+ "This isn't the only choice for a loss function, you could, for instance, choose `mean_squared_error`. But, generally, `binary_crossentropy` is better for dealing with probabilities—it measures the \"distance\" between probability distributions, or in our case, between the ground-truth distribution and the predictions.\n",
+ "\n",
+ "Later, when we are exploring regression problems (say, to predict the price of a house), we will see how to use another loss function called mean squared error.\n",
+ "\n",
+ "Now, configure the model to use an optimizer and a loss function:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "Mr0GP-cQ-llN"
+ },
+ "outputs": [],
+ "source": [
+ "model.compile(optimizer='adam',\n",
+ " loss=tf.losses.BinaryCrossentropy(from_logits=True),\n",
+ " metrics=[tf.metrics.BinaryAccuracy(threshold=0.0, name='accuracy')])"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "hCWYwkug-llQ"
+ },
+ "source": [
+ "## Create a validation set\n",
+ "\n",
+ "When training, we want to check the accuracy of the model on data it hasn't seen before. Create a *validation set* by setting apart 10,000 examples from the original training data. (Why not use the testing set now? Our goal is to develop and tune our model using only the training data, then use the test data just once to evaluate our accuracy)."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "-NpcXY9--llS"
+ },
+ "outputs": [],
+ "source": [
+ "x_val = train_examples[:10000]\n",
+ "partial_x_train = train_examples[10000:]\n",
+ "\n",
+ "y_val = train_labels[:10000]\n",
+ "partial_y_train = train_labels[10000:]"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "35jv_fzP-llU"
+ },
+ "source": [
+ "## Train the model\n",
+ "\n",
+ "Train the model for 40 epochs in mini-batches of 512 samples. This is 40 iterations over all samples in the `x_train` and `y_train` tensors. While training, monitor the model's loss and accuracy on the 10,000 samples from the validation set:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "tXSGrjWZ-llW"
+ },
+ "outputs": [],
+ "source": [
+ "history = model.fit(partial_x_train,\n",
+ " partial_y_train,\n",
+ " epochs=40,\n",
+ " batch_size=512,\n",
+ " validation_data=(x_val, y_val),\n",
+ " verbose=1)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "9EEGuDVuzb5r"
+ },
+ "source": [
+ "## Evaluate the model\n",
+ "\n",
+ "And let's see how the model performs. Two values will be returned. Loss (a number which represents our error, lower values are better), and accuracy."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "zOMKywn4zReN"
+ },
+ "outputs": [],
+ "source": [
+ "results = model.evaluate(test_examples, test_labels)\n",
+ "\n",
+ "print(results)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "z1iEXVTR0Z2t"
+ },
+ "source": [
+ "This fairly naive approach achieves an accuracy of about 87%. With more advanced approaches, the model should get closer to 95%."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "5KggXVeL-llZ"
+ },
+ "source": [
+ "## Create a graph of accuracy and loss over time\n",
+ "\n",
+ "`model.fit()` returns a `History` object that contains a dictionary with everything that happened during training:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "VcvSXvhp-llb"
+ },
+ "outputs": [],
+ "source": [
+ "history_dict = history.history\n",
+ "history_dict.keys()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "nRKsqL40-lle"
+ },
+ "source": [
+ "There are four entries: one for each monitored metric during training and validation. We can use these to plot the training and validation loss for comparison, as well as the training and validation accuracy:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "nGoYf2Js-lle"
+ },
+ "outputs": [],
+ "source": [
+ "acc = history_dict['accuracy']\n",
+ "val_acc = history_dict['val_accuracy']\n",
+ "loss = history_dict['loss']\n",
+ "val_loss = history_dict['val_loss']\n",
+ "\n",
+ "epochs = range(1, len(acc) + 1)\n",
+ "\n",
+ "# \"bo\" is for \"blue dot\"\n",
+ "plt.plot(epochs, loss, 'bo', label='Training loss')\n",
+ "# b is for \"solid blue line\"\n",
+ "plt.plot(epochs, val_loss, 'b', label='Validation loss')\n",
+ "plt.title('Training and validation loss')\n",
+ "plt.xlabel('Epochs')\n",
+ "plt.ylabel('Loss')\n",
+ "plt.legend()\n",
+ "\n",
+ "plt.show()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "6hXx-xOv-llh"
+ },
+ "outputs": [],
+ "source": [
+ "plt.clf() # clear figure\n",
+ "\n",
+ "plt.plot(epochs, acc, 'bo', label='Training acc')\n",
+ "plt.plot(epochs, val_acc, 'b', label='Validation acc')\n",
+ "plt.title('Training and validation accuracy')\n",
+ "plt.xlabel('Epochs')\n",
+ "plt.ylabel('Accuracy')\n",
+ "plt.legend()\n",
+ "\n",
+ "plt.show()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "oFEmZ5zq-llk"
+ },
+ "source": [
+ "In this plot, the dots represent the training loss and accuracy, and the solid lines are the validation loss and accuracy.\n",
+ "\n",
+ "Notice the training loss *decreases* with each epoch and the training accuracy *increases* with each epoch. This is expected when using a gradient descent optimization—it should minimize the desired quantity on every iteration.\n",
+ "\n",
+ "This isn't the case for the validation loss and accuracy—they seem to peak after about twenty epochs. This is an example of overfitting: the model performs better on the training data than it does on data it has never seen before. After this point, the model over-optimizes and learns representations *specific* to the training data that do not *generalize* to test data.\n",
+ "\n",
+ "For this particular case, we could prevent overfitting by simply stopping the training after twenty or so epochs. Later, you'll see how to do this automatically with a callback."
+ ]
+ }
+ ],
+ "metadata": {
+ "colab": {
+ "collapsed_sections": [],
+ "name": "tf2_text_classification.ipynb",
+ "toc_visible": true
+ },
+ "kernelspec": {
+ "display_name": "Python 3",
+ "name": "python3"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}
diff --git a/site/en/hub/tutorials/tf_hub_delf_module.ipynb b/site/en/hub/tutorials/tf_hub_delf_module.ipynb
new file mode 100644
index 00000000000..b6dec2eae00
--- /dev/null
+++ b/site/en/hub/tutorials/tf_hub_delf_module.ipynb
@@ -0,0 +1,372 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "RUymE2l9GZfO"
+ },
+ "source": [
+ "##### Copyright 2018 The TensorFlow Hub Authors.\n",
+ "\n",
+ "Licensed under the Apache License, Version 2.0 (the \"License\");"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "cellView": "code",
+ "id": "JMyTNwSJGGWg"
+ },
+ "outputs": [],
+ "source": [
+ "# Copyright 2018 The TensorFlow Hub Authors. All Rights Reserved.\n",
+ "#\n",
+ "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+ "# you may not use this file except in compliance with the License.\n",
+ "# You may obtain a copy of the License at\n",
+ "#\n",
+ "# http://www.apache.org/licenses/LICENSE-2.0\n",
+ "#\n",
+ "# Unless required by applicable law or agreed to in writing, software\n",
+ "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+ "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+ "# See the License for the specific language governing permissions and\n",
+ "# limitations under the License.\n",
+ "# =============================================================================="
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "0DmDwGPOGfaQ"
+ },
+ "source": [
+ "# How to match images using DELF and TensorFlow Hub\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "MfBg1C5NB3X0"
+ },
+ "source": [
+ "
"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "ndG8MjmJeicp"
+ },
+ "source": [
+ "# Fine-tuning Wav2Vec2 with an LM head\n",
+ "\n",
+ "In this notebook, we will load the pre-trained wav2vec2 model from [TFHub](https://tfhub.dev) and will fine-tune it on [LibriSpeech dataset](https://huggingface.co/datasets/librispeech_asr) by appending Language Modeling head (LM) over the top of our pre-trained model. The underlying task is to build a model for **Automatic Speech Recognition** i.e. given some speech, the model should be able to transcribe it into text."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "rWk8nL6Ui-_0"
+ },
+ "source": [
+ "## Setting Up\n",
+ "\n",
+ "Before running this notebook, please ensure that you are on GPU runtime (`Runtime` > `Change runtime type` > `GPU`). The following cell will install [`gsoc-wav2vec2`](https://github.com/vasudevgupta7/gsoc-wav2vec2) package & its dependencies."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "seqTlMyeZvM4"
+ },
+ "outputs": [],
+ "source": [
+ "!pip3 install -q git+https://github.com/vasudevgupta7/gsoc-wav2vec2@main\n",
+ "!sudo apt-get install -y libsndfile1-dev\n",
+ "!pip3 install -q SoundFile"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "wvuJL8-f0zn5"
+ },
+ "source": [
+ "## Model setup using `TFHub`\n",
+ "\n",
+ "We will start by importing some libraries/modules."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "M3_fgx4eZvM7"
+ },
+ "outputs": [],
+ "source": [
+ "import os\n",
+ "\n",
+ "import tensorflow as tf\n",
+ "import tensorflow_hub as hub\n",
+ "from wav2vec2 import Wav2Vec2Config\n",
+ "\n",
+ "config = Wav2Vec2Config()\n",
+ "\n",
+ "print(\"TF version:\", tf.__version__)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "y0rVUxyWsS5f"
+ },
+ "source": [
+ "First, we will download our model from TFHub & will wrap our model signature with [`hub.KerasLayer`](https://www.tensorflow.org/hub/api_docs/python/hub/KerasLayer) to be able to use this model like any other Keras layer. Fortunately, `hub.KerasLayer` can do both in just 1 line.\n",
+ "\n",
+ "**Note:** When loading model with `hub.KerasLayer`, model becomes a bit opaque but sometimes we need finer controls over the model, then we can load the model with `tf.keras.models.load_model(...)`."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "NO6QRC7KZvM9"
+ },
+ "outputs": [],
+ "source": [
+ "pretrained_layer = hub.KerasLayer(\"https://tfhub.dev/vasudevgupta7/wav2vec2/1\", trainable=True)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "pCputyVBv2e9"
+ },
+ "source": [
+ "You can refer to this [script](https://github.com/vasudevgupta7/gsoc-wav2vec2/blob/main/src/export2hub.py) in case you are interested in the model exporting script. Object `pretrained_layer` is the freezed version of [`Wav2Vec2Model`](https://github.com/vasudevgupta7/gsoc-wav2vec2/blob/main/src/wav2vec2/modeling.py). These pre-trained weights were converted from HuggingFace PyTorch [pre-trained weights](https://huggingface.co/facebook/wav2vec2-base) using [this script](https://github.com/vasudevgupta7/gsoc-wav2vec2/blob/main/src/convert_torch_to_tf.py).\n",
+ "\n",
+ "Originally, wav2vec2 was pre-trained with a masked language modelling approach with the objective to identify the true quantized latent speech representation for a masked time step. You can read more about the training objective in the paper- [wav2vec 2.0: A Framework for Self-Supervised Learning of Speech Representations](https://arxiv.org/abs/2006.11477)."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "SseDnCr7hyhC"
+ },
+ "source": [
+ "Now, we will be defining a few constants and hyper-parameters which will be useful in the next few cells. `AUDIO_MAXLEN` is intentionally set to `246000` as the model signature only accepts static sequence length of `246000`."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "eiILuMBERxlO"
+ },
+ "outputs": [],
+ "source": [
+ "AUDIO_MAXLEN = 246000\n",
+ "LABEL_MAXLEN = 256\n",
+ "BATCH_SIZE = 2"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "1V4gTgGLgXvO"
+ },
+ "source": [
+ "In the following cell, we will wrap `pretrained_layer` & a dense layer (LM head) with the [Keras's Functional API](https://www.tensorflow.org/guide/keras/functional)."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "a3CUN1KEB10Q"
+ },
+ "outputs": [],
+ "source": [
+ "inputs = tf.keras.Input(shape=(AUDIO_MAXLEN,))\n",
+ "hidden_states = pretrained_layer(inputs)\n",
+ "outputs = tf.keras.layers.Dense(config.vocab_size)(hidden_states)\n",
+ "\n",
+ "model = tf.keras.Model(inputs=inputs, outputs=outputs)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "5zDXuoMXhDMo"
+ },
+ "source": [
+ "The dense layer (defined above) is having an output dimension of `vocab_size` as we want to predict probabilities of each token in the vocabulary at each time step."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "oPp18ZHRtnq-"
+ },
+ "source": [
+ "## Setting up training state"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "ATQy1ZK3vFr7"
+ },
+ "source": [
+ "In TensorFlow, model weights are built only when `model.call` or `model.build` is called for the first time, so the following cell will build the model weights for us. Further, we will be running `model.summary()` to check the total number of trainable parameters."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "ZgL5wyaXZvM-"
+ },
+ "outputs": [],
+ "source": [
+ "model(tf.random.uniform(shape=(BATCH_SIZE, AUDIO_MAXLEN)))\n",
+ "model.summary()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "EQxxA4Fevp7m"
+ },
+ "source": [
+ "Now, we need to define the `loss_fn` and optimizer to be able to train the model. The following cell will do that for us. We will be using the `Adam` optimizer for simplicity. `CTCLoss` is a common loss type that is used for tasks (like `ASR`) where input sub-parts can't be easily aligned with output sub-parts. You can read more about CTC-loss from this amazing [blog post](https://distill.pub/2017/ctc/).\n",
+ "\n",
+ "\n",
+ "`CTCLoss` (from [`gsoc-wav2vec2`](https://github.com/vasudevgupta7/gsoc-wav2vec2) package) accepts 3 arguments: `config`, `model_input_shape` & `division_factor`. If `division_factor=1`, then loss will simply get summed, so pass `division_factor` accordingly to get mean over batch."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "glDepVEHZvM_"
+ },
+ "outputs": [],
+ "source": [
+ "from wav2vec2 import CTCLoss\n",
+ "\n",
+ "LEARNING_RATE = 5e-5\n",
+ "\n",
+ "loss_fn = CTCLoss(config, (BATCH_SIZE, AUDIO_MAXLEN), division_factor=BATCH_SIZE)\n",
+ "optimizer = tf.keras.optimizers.Adam(LEARNING_RATE)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "1mvTuOXpwsQe"
+ },
+ "source": [
+ "## Loading & Pre-processing data\n",
+ "\n",
+ "Let's now download the LibriSpeech dataset from the [official website](http://www.openslr.org/12) and set it up."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "I4kIEC77cBCM"
+ },
+ "outputs": [],
+ "source": [
+ "!wget https://www.openslr.org/resources/12/dev-clean.tar.gz -P ./data/train/\n",
+ "!tar -xf ./data/train/dev-clean.tar.gz -C ./data/train/"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "LsQpmpn6jrMI"
+ },
+ "source": [
+ "**Note:** We are using `dev-clean` configuration as this notebook is just for demonstration purposes, so we need a small amount of data. Complete training data can be easily downloaded from [LibriSpeech website](http://www.openslr.org/12)."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "ynxAjtGHGFpM"
+ },
+ "outputs": [],
+ "source": [
+ "ls ./data/train/"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "yBMiORo0xJD0"
+ },
+ "source": [
+ "Our dataset lies in the LibriSpeech directory. Let's explore these files."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "jkIu_Wt4ZvNA"
+ },
+ "outputs": [],
+ "source": [
+ "data_dir = \"./data/train/LibriSpeech/dev-clean/2428/83705/\"\n",
+ "all_files = os.listdir(data_dir)\n",
+ "\n",
+ "flac_files = [f for f in all_files if f.endswith(\".flac\")]\n",
+ "txt_files = [f for f in all_files if f.endswith(\".txt\")]\n",
+ "\n",
+ "print(\"Transcription files:\", txt_files, \"\\nSound files:\", flac_files)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "XEObi_Apk3ZD"
+ },
+ "source": [
+ "Alright, so each sub-directory has many `.flac` files and a `.txt` file. The `.txt` file contains text transcriptions for all the speech samples (i.e. `.flac` files) present in that sub-directory."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "WYW6WKJflO2e"
+ },
+ "source": [
+ "We can load this text data as follows:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "cEBKxQblHPwq"
+ },
+ "outputs": [],
+ "source": [
+ "def read_txt_file(f):\n",
+ " with open(f, \"r\") as f:\n",
+ " samples = f.read().split(\"\\n\")\n",
+ " samples = {s.split()[0]: \" \".join(s.split()[1:]) for s in samples if len(s.split()) > 2}\n",
+ " return samples"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "Ldkf_ceb0_YW"
+ },
+ "source": [
+ "Similarly, we will define a function for loading a speech sample from a `.flac` file.\n",
+ "\n",
+ "`REQUIRED_SAMPLE_RATE` is set to `16000` as wav2vec2 was pre-trained with `16K` frequency and it's recommended to fine-tune it without any major change in data distribution due to frequency."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "YOJ3OzPsTyXv"
+ },
+ "outputs": [],
+ "source": [
+ "import soundfile as sf\n",
+ "\n",
+ "REQUIRED_SAMPLE_RATE = 16000\n",
+ "\n",
+ "def read_flac_file(file_path):\n",
+ " with open(file_path, \"rb\") as f:\n",
+ " audio, sample_rate = sf.read(f)\n",
+ " if sample_rate != REQUIRED_SAMPLE_RATE:\n",
+ " raise ValueError(\n",
+ " f\"sample rate (={sample_rate}) of your files must be {REQUIRED_SAMPLE_RATE}\"\n",
+ " )\n",
+ " file_id = os.path.split(file_path)[-1][:-len(\".flac\")]\n",
+ " return {file_id: audio}"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "2sxDN8P4nWkW"
+ },
+ "source": [
+ "Now, we will pick some random samples & will try to visualize them."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "HI5J-2Dfm_wT"
+ },
+ "outputs": [],
+ "source": [
+ "from IPython.display import Audio\n",
+ "import random\n",
+ "\n",
+ "file_id = random.choice([f[:-len(\".flac\")] for f in flac_files])\n",
+ "flac_file_path, txt_file_path = os.path.join(data_dir, f\"{file_id}.flac\"), os.path.join(data_dir, \"2428-83705.trans.txt\")\n",
+ "\n",
+ "print(\"Text Transcription:\", read_txt_file(txt_file_path)[file_id], \"\\nAudio:\")\n",
+ "Audio(filename=flac_file_path)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "M8jJ7Ed81p_A"
+ },
+ "source": [
+ "Now, we will combine all the speech & text samples and will define the function (in next cell) for that purpose."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "MI-5YCzaTsei"
+ },
+ "outputs": [],
+ "source": [
+ "def fetch_sound_text_mapping(data_dir):\n",
+ " all_files = os.listdir(data_dir)\n",
+ "\n",
+ " flac_files = [os.path.join(data_dir, f) for f in all_files if f.endswith(\".flac\")]\n",
+ " txt_files = [os.path.join(data_dir, f) for f in all_files if f.endswith(\".txt\")]\n",
+ "\n",
+ " txt_samples = {}\n",
+ " for f in txt_files:\n",
+ " txt_samples.update(read_txt_file(f))\n",
+ "\n",
+ " speech_samples = {}\n",
+ " for f in flac_files:\n",
+ " speech_samples.update(read_flac_file(f))\n",
+ "\n",
+ " assert len(txt_samples) == len(speech_samples)\n",
+ "\n",
+ " samples = [(speech_samples[file_id], txt_samples[file_id]) for file_id in speech_samples.keys() if len(speech_samples[file_id]) < AUDIO_MAXLEN]\n",
+ " return samples"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "mx95Lxvu0nT4"
+ },
+ "source": [
+ "It's time to have a look at a few samples ..."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "_Ls7X_jqIz4R"
+ },
+ "outputs": [],
+ "source": [
+ "samples = fetch_sound_text_mapping(data_dir)\n",
+ "samples[:5]"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "TUjhSWfsnlCL"
+ },
+ "source": [
+ "Note: We are loading this data into memory as we working with a small amount of dataset in this notebook. But for training on the complete dataset (~300 GBs), you will have to load data lazily. You can refer to [this script](https://github.com/vasudevgupta7/gsoc-wav2vec2/blob/main/src/data_utils.py) to know more on that."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "xg8Zia1kzw0J"
+ },
+ "source": [
+ "Let's pre-process the data now !!!\n",
+ "\n",
+ "We will first define the tokenizer & processor using `gsoc-wav2vec2` package. Then, we will do very simple pre-processing. `processor` will normalize raw speech w.r.to frames axis and `tokenizer` will convert our model outputs into the string (using the defined vocabulary) & will take care of the removal of special tokens (depending on your tokenizer configuration)."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "gaat_hMLNVHF"
+ },
+ "outputs": [],
+ "source": [
+ "from wav2vec2 import Wav2Vec2Processor\n",
+ "tokenizer = Wav2Vec2Processor(is_tokenizer=True)\n",
+ "processor = Wav2Vec2Processor(is_tokenizer=False)\n",
+ "\n",
+ "def preprocess_text(text):\n",
+ " label = tokenizer(text)\n",
+ " return tf.constant(label, dtype=tf.int32)\n",
+ "\n",
+ "def preprocess_speech(audio):\n",
+ " audio = tf.constant(audio, dtype=tf.float32)\n",
+ " return processor(tf.transpose(audio))"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "GyKl8QP-zRFC"
+ },
+ "source": [
+ "Now, we will define the python generator to call the preprocessing functions we defined in above cells."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "PoQrRalwMpQ6"
+ },
+ "outputs": [],
+ "source": [
+ "def inputs_generator():\n",
+ " for speech, text in samples:\n",
+ " yield preprocess_speech(speech), preprocess_text(text)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "7Vlm3ySFULsG"
+ },
+ "source": [
+ "## Setting up `tf.data.Dataset`\n",
+ "\n",
+ "Following cell will setup `tf.data.Dataset` object using its `.from_generator(...)` method. We will be using the `generator` object, we defined in the above cell.\n",
+ "\n",
+ "**Note:** For distributed training (especially on TPUs), `.from_generator(...)` doesn't work currently and it is recommended to train on data stored in `.tfrecord` format (Note: The TFRecords should ideally be stored inside a GCS Bucket in order for the TPUs to work to the fullest extent).\n",
+ "\n",
+ "You can refer to [this script](https://github.com/vasudevgupta7/gsoc-wav2vec2/blob/main/src/make_tfrecords.py) for more details on how to convert LibriSpeech data into tfrecords."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "LbQ_dMwGO62h"
+ },
+ "outputs": [],
+ "source": [
+ "output_signature = (\n",
+ " tf.TensorSpec(shape=(None), dtype=tf.float32),\n",
+ " tf.TensorSpec(shape=(None), dtype=tf.int32),\n",
+ ")\n",
+ "\n",
+ "dataset = tf.data.Dataset.from_generator(inputs_generator, output_signature=output_signature)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "HXBbNsRyPyw3"
+ },
+ "outputs": [],
+ "source": [
+ "BUFFER_SIZE = len(flac_files)\n",
+ "SEED = 42\n",
+ "\n",
+ "dataset = dataset.shuffle(BUFFER_SIZE, seed=SEED)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "9DAUmns3pXfr"
+ },
+ "source": [
+ "We will pass the dataset into multiple batches, so let's prepare batches in the following cell. Now, all the sequences in a batch should be padded to a constant length. We will use the`.padded_batch(...)` method for that purpose."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "Okhko1IWRida"
+ },
+ "outputs": [],
+ "source": [
+ "dataset = dataset.padded_batch(BATCH_SIZE, padded_shapes=(AUDIO_MAXLEN, LABEL_MAXLEN), padding_values=(0.0, 0))"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "A45CjQG5qSbV"
+ },
+ "source": [
+ "Accelerators (like GPUs/TPUs) are very fast and often data-loading (& pre-processing) becomes the bottleneck during training as the data-loading part happens on CPUs. This can increase the training time significantly especially when there is a lot of online pre-processing involved or data is streamed online from GCS buckets. To handle those issues, `tf.data.Dataset` offers the `.prefetch(...)` method. This method helps in preparing the next few batches in parallel (on CPUs) while the model is making predictions (on GPUs/TPUs) on the current batch."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "f-bKu2YjRior"
+ },
+ "outputs": [],
+ "source": [
+ "dataset = dataset.prefetch(tf.data.AUTOTUNE)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "Lqk2cs6LxVIh"
+ },
+ "source": [
+ "Since this notebook is made for demonstration purposes, we will be taking first `num_train_batches` and will perform training over only that. You are encouraged to train on the whole dataset though. Similarly, we will evaluate only `num_val_batches`."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "z6GO5oYUxXtz"
+ },
+ "outputs": [],
+ "source": [
+ "num_train_batches = 10\n",
+ "num_val_batches = 4\n",
+ "\n",
+ "train_dataset = dataset.take(num_train_batches)\n",
+ "val_dataset = dataset.skip(num_train_batches).take(num_val_batches)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "CzAOI78tky08"
+ },
+ "source": [
+ "## Model training\n",
+ "\n",
+ "For training our model, we will be directly calling `.fit(...)` method after compiling our model with `.compile(...)`."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "vuBY2sZElgwg"
+ },
+ "outputs": [],
+ "source": [
+ "model.compile(optimizer, loss=loss_fn)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "qswxafSl0HjO"
+ },
+ "source": [
+ "The above cell will set up our training state. Now we can initiate training with the `.fit(...)` method."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "vtuSfnj1l-I_"
+ },
+ "outputs": [],
+ "source": [
+ "history = model.fit(train_dataset, validation_data=val_dataset, epochs=3)\n",
+ "history.history"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "ySvp8r2E1q_V"
+ },
+ "source": [
+ "Let's save our model with `.save(...)` method to be able to perform inference later. You can also export this SavedModel to TFHub by following [TFHub documentation](https://www.tensorflow.org/hub/publish)."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "C0KEYcwydwjF"
+ },
+ "outputs": [],
+ "source": [
+ "save_dir = \"finetuned-wav2vec2\"\n",
+ "model.save(save_dir, include_optimizer=False)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "MkOpp9rZ211t"
+ },
+ "source": [
+ "Note: We are setting `include_optimizer=False` as we want to use this model for inference only."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "SJfPlTgezD0i"
+ },
+ "source": [
+ "## Evaluation\n",
+ "\n",
+ "Now we will be computing Word Error Rate over the validation dataset\n",
+ "\n",
+ "**Word error rate** (WER) is a common metric for measuring the performance of an automatic speech recognition system. The WER is derived from the Levenshtein distance, working at the word level. Word error rate can then be computed as: WER = (S + D + I) / N = (S + D + I) / (S + D + C) where S is the number of substitutions, D is the number of deletions, I is the number of insertions, C is the number of correct words, N is the number of words in the reference (N=S+D+C). This value indicates the percentage of words that were incorrectly predicted. \n",
+ "\n",
+ "You can refer to [this paper](https://www.isca-speech.org/archive_v0/interspeech_2004/i04_2765.html) to learn more about WER."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "Io_91Y7-r3xu"
+ },
+ "source": [
+ "We will use `load_metric(...)` function from [HuggingFace datasets](https://huggingface.co/docs/datasets/) library. Let's first install the `datasets` library using `pip` and then define the `metric` object."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "GW9F_oVDU1TZ"
+ },
+ "outputs": [],
+ "source": [
+ "!pip3 install -q datasets\n",
+ "\n",
+ "from datasets import load_metric\n",
+ "metric = load_metric(\"wer\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "ssWXWc7CZvNB"
+ },
+ "outputs": [],
+ "source": [
+ "@tf.function(jit_compile=True)\n",
+ "def eval_fwd(batch):\n",
+ " logits = model(batch, training=False)\n",
+ " return tf.argmax(logits, axis=-1)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "NFh1myg1x4ua"
+ },
+ "source": [
+ "It's time to run the evaluation on validation data now."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "EQTFVjZghckJ"
+ },
+ "outputs": [],
+ "source": [
+ "from tqdm.auto import tqdm\n",
+ "\n",
+ "for speech, labels in tqdm(val_dataset, total=num_val_batches):\n",
+ " predictions = eval_fwd(speech)\n",
+ " predictions = [tokenizer.decode(pred) for pred in predictions.numpy().tolist()]\n",
+ " references = [tokenizer.decode(label, group_tokens=False) for label in labels.numpy().tolist()]\n",
+ " metric.add_batch(references=references, predictions=predictions)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "WWCc8qBesv3e"
+ },
+ "source": [
+ "We are using the `tokenizer.decode(...)` method for decoding our predictions and labels back into the text and will add them to the metric for `WER` computation later."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "XI_URj8Wtb2g"
+ },
+ "source": [
+ "Now, let's calculate the metric value in following cell:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "a83wekLgWMod"
+ },
+ "outputs": [],
+ "source": [
+ "metric.compute()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "c_cD1OgVEjl4"
+ },
+ "source": [
+ "**Note:** Here metric value doesn't make any sense as the model is trained on very small data and ASR-like tasks often require a large amount of data to learn a mapping from speech to text. You should probably train on large data to get some good results. This notebook gives you a template to fine-tune a pre-trained speech model."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "G14o706kdTE1"
+ },
+ "source": [
+ "## Inference\n",
+ "\n",
+ "Now that we are satisfied with the training process & have saved the model in `save_dir`, we will see how this model can be used for inference.\n",
+ "\n",
+ "First, we will load our model using `tf.keras.models.load_model(...)`."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "wrTrExiUdaED"
+ },
+ "outputs": [],
+ "source": [
+ "finetuned_model = tf.keras.models.load_model(save_dir)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "luodSroz20SR"
+ },
+ "source": [
+ "Let's download some speech samples for performing inference. You can replace the following sample with your speech sample also."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "HUE0shded6Ej"
+ },
+ "outputs": [],
+ "source": [
+ "!wget https://github.com/vasudevgupta7/gsoc-wav2vec2/raw/main/data/SA2.wav"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "ycBjU_U53FjL"
+ },
+ "source": [
+ "Now, we will read the speech sample using `soundfile.read(...)` and pad it to `AUDIO_MAXLEN` to satisfy the model signature. Then we will normalize that speech sample using the `Wav2Vec2Processor` instance & will feed it into the model."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "z7CARje4d5_H"
+ },
+ "outputs": [],
+ "source": [
+ "import numpy as np\n",
+ "\n",
+ "speech, _ = sf.read(\"SA2.wav\")\n",
+ "speech = np.pad(speech, (0, AUDIO_MAXLEN - len(speech)))\n",
+ "speech = tf.expand_dims(processor(tf.constant(speech)), 0)\n",
+ "\n",
+ "outputs = finetuned_model(speech)\n",
+ "outputs"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "lUSttSPa30qP"
+ },
+ "source": [
+ "Let's decode numbers back into text sequence using the `Wav2Vec2tokenizer` instance, we defined above."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "RYdJqxQ4llgI"
+ },
+ "outputs": [],
+ "source": [
+ "predictions = tf.argmax(outputs, axis=-1)\n",
+ "predictions = [tokenizer.decode(pred) for pred in predictions.numpy().tolist()]\n",
+ "predictions"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "7DXC757bztJc"
+ },
+ "source": [
+ "This prediction is quite random as the model was never trained on large data in this notebook (as this notebook is not meant for doing complete training). You will get good predictions if you train this model on complete LibriSpeech dataset.\n",
+ "\n",
+ "Finally, we have reached an end to this notebook. But it's not an end of learning TensorFlow for speech-related tasks, this [repository](https://github.com/tulasiram58827/TTS_TFLite) contains some more amazing tutorials. In case you encountered any bug in this notebook, please create an issue [here](https://github.com/vasudevgupta7/gsoc-wav2vec2/issues)."
+ ]
+ }
+ ],
+ "metadata": {
+ "accelerator": "GPU",
+ "colab": {
+ "collapsed_sections": [
+ "rWk8nL6Ui-_0",
+ "wvuJL8-f0zn5",
+ "oPp18ZHRtnq-",
+ "1mvTuOXpwsQe",
+ "7Vlm3ySFULsG",
+ "CzAOI78tky08",
+ "SJfPlTgezD0i",
+ "G14o706kdTE1"
+ ],
+ "name": "wav2vec2_saved_model_finetuning.ipynb",
+ "toc_visible": true
+ },
+ "kernelspec": {
+ "display_name": "Python 3",
+ "name": "python3"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}
diff --git a/site/en/hub/tutorials/wiki40b_lm.ipynb b/site/en/hub/tutorials/wiki40b_lm.ipynb
new file mode 100644
index 00000000000..ad94ce0aab8
--- /dev/null
+++ b/site/en/hub/tutorials/wiki40b_lm.ipynb
@@ -0,0 +1,451 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "Oxb_tjw13y4G"
+ },
+ "source": [
+ "##### Copyright 2019 The TensorFlow Hub Authors.\n",
+ "\n",
+ "Licensed under the Apache License, Version 2.0 (the \"License\");"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "EAkh2aBJLg6q"
+ },
+ "outputs": [],
+ "source": [
+ "# Copyright 2019 The TensorFlow Hub Authors. All Rights Reserved.\n",
+ "#\n",
+ "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+ "# you may not use this file except in compliance with the License.\n",
+ "# You may obtain a copy of the License at\n",
+ "#\n",
+ "# http://www.apache.org/licenses/LICENSE-2.0\n",
+ "#\n",
+ "# Unless required by applicable law or agreed to in writing, software\n",
+ "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+ "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+ "# See the License for the specific language governing permissions and\n",
+ "# limitations under the License.\n",
+ "# =============================================================================="
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "owAopeOtirc9"
+ },
+ "source": [
+ "# Wiki40B Language Models\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "T-nCyGRri-KO"
+ },
+ "source": [
+ "
"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "8eY9jkGpjf3d"
+ },
+ "source": [
+ "Generate Wikipedia-like text using the **Wiki40B language models** from [TensorFlow Hub](https://tfhub.dev)!\n",
+ "\n",
+ "This notebook illustrates how to:\n",
+ "* Load the 41 monolingual and 2 multilingual language models that are part of the [Wiki40b-LM collection](https://tfhub.dev/google/collections/wiki40b-lm/1) on TF-Hub\n",
+ "* Use the models to obtain perplexity, per layer activations, and word embeddings for a given piece of text\n",
+ "* Generate text token-by-token from a piece of seed text\n",
+ "\n",
+ "The language models are trained on the newly published, cleaned-up [Wiki40B dataset](https://www.tensorflow.org/datasets/catalog/wiki40b) available on TensorFlow Datasets. The training setup is based on the paper [“Wiki-40B: Multilingual Language Model Dataset”](https://research.google/pubs/pub49029/)."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "wK2YnrEhLjDf"
+ },
+ "source": [
+ "## Setup"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "cellView": "form",
+ "id": "sv2CmI7BdaML"
+ },
+ "outputs": [],
+ "source": [
+ "#@title Installing Dependencies\n",
+ "!pip install --quiet \"tensorflow-text==2.11.*\""
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "cellView": "form",
+ "id": "8uSkaQ-Vdon2"
+ },
+ "outputs": [],
+ "source": [
+ "#@title Imports\n",
+ "import numpy as np\n",
+ "import tensorflow.compat.v1 as tf\n",
+ "import tensorflow_hub as hub\n",
+ "import tensorflow_text as tf_text\n",
+ "\n",
+ "tf.disable_eager_execution()\n",
+ "tf.logging.set_verbosity(tf.logging.WARN)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "d2MvP-cyL-BN"
+ },
+ "source": [
+ "## Choose Language\n",
+ "\n",
+ "Let's choose **which language model** to load from TF-Hub and the **length of text** to be generated. \n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "cellView": "form",
+ "id": "33zYlSXwMA_o"
+ },
+ "outputs": [],
+ "source": [
+ "#@title { run: \"auto\" }\n",
+ "language = \"en\" #@param [\"en\", \"ar\", \"zh-cn\", \"zh-tw\", \"nl\", \"fr\", \"de\", \"it\", \"ja\", \"ko\", \"pl\", \"pt\", \"ru\", \"es\", \"th\", \"tr\", \"bg\", \"ca\", \"cs\", \"da\", \"el\", \"et\", \"fa\", \"fi\", \"he\", \"hi\", \"hr\", \"hu\", \"id\", \"lt\", \"lv\", \"ms\", \"no\", \"ro\", \"sk\", \"sl\", \"sr\", \"sv\", \"tl\", \"uk\", \"vi\", \"multilingual-64k\", \"multilingual-128k\"]\n",
+ "hub_module = \"https://tfhub.dev/google/wiki40b-lm-{}/1\".format(language)\n",
+ "max_gen_len = 20 #@param\n",
+ "\n",
+ "print(\"Using the {} model to generate sequences of max length {}.\".format(hub_module, max_gen_len))"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "dgw2qW4xZbMj"
+ },
+ "source": [
+ "## Build the Model\n",
+ "\n",
+ "Okay, now that we've configured which pre-trained model to use, let's configure it to generate text up to `max_gen_len`. We will need to load the language model from TF-Hub, feed in a piece of starter text, and then iteratively feed in tokens as they are generated."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "cellView": "form",
+ "id": "pUypKuc3Mlpa"
+ },
+ "outputs": [],
+ "source": [
+ "#@title Load the language model pieces\n",
+ "g = tf.Graph()\n",
+ "n_layer = 12\n",
+ "model_dim = 768\n",
+ "\n",
+ "with g.as_default():\n",
+ " text = tf.placeholder(dtype=tf.string, shape=(1,))\n",
+ "\n",
+ " # Load the pretrained model from TF-Hub\n",
+ " module = hub.Module(hub_module)\n",
+ "\n",
+ " # Get the word embeddings, activations at each layer, negative log likelihood\n",
+ " # of the text, and calculate the perplexity.\n",
+ " embeddings = module(dict(text=text), signature=\"word_embeddings\", as_dict=True)[\"word_embeddings\"]\n",
+ " activations = module(dict(text=text), signature=\"activations\", as_dict=True)[\"activations\"]\n",
+ " neg_log_likelihood = module(dict(text=text), signature=\"neg_log_likelihood\", as_dict=True)[\"neg_log_likelihood\"]\n",
+ " ppl = tf.exp(tf.reduce_mean(neg_log_likelihood, axis=1))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "cellView": "form",
+ "id": "ZOS2Z2n0MsuC"
+ },
+ "outputs": [],
+ "source": [
+ "#@title Construct the per-token generation graph\n",
+ "def feedforward_step(module, inputs, mems):\n",
+ " \"\"\"Generate one step.\"\"\"\n",
+ " # Set up the input dict for one step of generation\n",
+ " inputs = tf.dtypes.cast(inputs, tf.int64)\n",
+ " generation_input_dict = dict(input_tokens=inputs)\n",
+ " mems_dict = {\"mem_{}\".format(i): mems[i] for i in range(n_layer)}\n",
+ " generation_input_dict.update(mems_dict)\n",
+ "\n",
+ " # Generate the tokens from the language model\n",
+ " generation_outputs = module(generation_input_dict, signature=\"prediction\", as_dict=True)\n",
+ "\n",
+ " # Get the probabilities and the inputs for the next steps\n",
+ " probs = generation_outputs[\"probs\"]\n",
+ " new_mems = [generation_outputs[\"new_mem_{}\".format(i)] for i in range(n_layer)]\n",
+ "\n",
+ " return probs, new_mems"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "cellView": "form",
+ "id": "S9ss6amQMyVY"
+ },
+ "outputs": [],
+ "source": [
+ "#@title Build the statically unrolled graph for `max_gen_len` tokens\n",
+ "with g.as_default():\n",
+ " # Tokenization with the sentencepiece model.\n",
+ " token_ids = module(dict(text=text), signature=\"tokenization\", as_dict=True)[\"token_ids\"]\n",
+ " inputs_np = token_ids\n",
+ " # Generate text by statically unrolling the computational graph\n",
+ " mems_np = [np.zeros([1, 0, model_dim], dtype=np.float32) for _ in range(n_layer)]\n",
+ "\n",
+ " # Generate up to `max_gen_len` tokens\n",
+ " sampled_ids = []\n",
+ " for step in range(max_gen_len):\n",
+ " probs, mems_np = feedforward_step(module, inputs_np, mems_np)\n",
+ " sampled_id = tf.random.categorical(tf.math.log(probs[0]), num_samples=1, dtype=tf.int32)\n",
+ " sampled_id = tf.squeeze(sampled_id)\n",
+ " sampled_ids.append(sampled_id)\n",
+ " inputs_np = tf.reshape(sampled_id, [1, 1])\n",
+ "\n",
+ " # Transform the ids into text\n",
+ " sampled_ids = tf.expand_dims(sampled_ids, axis=0)\n",
+ " generated_text = module(dict(token_ids=sampled_ids), signature=\"detokenization\", as_dict=True)[\"text\"]\n",
+ "\n",
+ " init_op = tf.group([tf.global_variables_initializer(), tf.tables_initializer()])"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "K5SYcRrxM7vS"
+ },
+ "source": [
+ "## Generate some text\n",
+ "\n",
+ "Let's generate some text! We'll set a text `seed` to prompt the language model.\n",
+ "\n",
+ "You can use one of the **predefined** seeds or _optionally_ **enter your own**. This text will be used as seed for the language model to help prompt the language model for what to generate next.\n",
+ "\n",
+ "You can use the following special tokens precede special parts of the generated article. Use **`_START_ARTICLE_`** to indicate the beginning of the article, **`_START_SECTION_`** to indicate the beginning of a section, and **`_START_PARAGRAPH_`** to generate text in the article\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "cellView": "form",
+ "id": "GmZxv7bzMIcL"
+ },
+ "outputs": [],
+ "source": [
+ "#@title Predefined Seeds\n",
+ "lang_to_seed = {\"en\": \"\\n_START_ARTICLE_\\n1882 Prince Edward Island general election\\n_START_PARAGRAPH_\\nThe 1882 Prince Edward Island election was held on May 8, 1882 to elect members of the House of Assembly of the province of Prince Edward Island, Canada.\",\n",
+ " \"ar\": \"\\n_START_ARTICLE_\\nأوليفيا كوك\\n_START_SECTION_\\nنشأتها والتعلي \\n_START_PARAGRAPH_\\nولدت أوليفيا كوك في أولدهام في مانشستر الكبرى لأسرة تتكون من أب يعمل كظابط شرطة، وأمها تعمل كممثلة مبيعات. عندما كانت صغيرة بدأت تأخذ دروساً في الباليه الجمباز. وفي المدرسة شاركت في المسرحيات المدرسية، إضافةً إلى عملها في مسرح سندريلا . وفي سن الرابعة عشر عاماً، حصلت على وكيلة لها في مانشستر وهي وقعت عقداً مع وكالة الفنانين المبدعين في مانشستر،\",\n",
+ " \"zh-cn\": \"\\n_START_ARTICLE_\\n上尾事件\\n_START_SECTION_\\n日本国铁劳资关系恶化\\n_START_PARAGRAPH_\\n由于日本国铁财政恶化，管理层开始重整人手安排，令工会及员工感到受威胁。但日本国铁作为公营企业，其雇员均受公营企业等劳资关系法规管——该法第17条规定公营企业员工不得发动任何罢工行为。为了规避该法例\",\n",
+ " \"zh-tw\": \"\\n_START_ARTICLE_\\n乌森\\n_START_PARAGRAPH_\\n烏森（法語：Houssen，發音：[usən]；德語：Hausen；阿爾薩斯語：Hüse）是法國上萊茵省的一個市鎮，位於該省北部，屬於科爾馬-里博維萊區（Colmar-Ribeauvillé）第二科爾馬縣（Colmar-2）。該市鎮總面積6.7平方公里，2009年時的人口為\",\n",
+ " \"nl\": \"\\n_START_ARTICLE_\\n1001 vrouwen uit de Nederlandse geschiedenis\\n_START_SECTION_\\nSelectie van vrouwen\\n_START_PARAGRAPH_\\nDe 'oudste' biografie in het boek is gewijd aan de beschermheilige\",\n",
+ " \"fr\": \"\\n_START_ARTICLE_\\nꝹ\\n_START_SECTION_\\nUtilisation\\n_START_PARAGRAPH_\\nLe d insulaire est utilisé comme lettre additionnelle dans l’édition de 1941 du recueil de chroniques galloises Brut y Tywysogion\",\n",
+ " \"de\": \"\\n_START_ARTICLE_\\nÜnal Demirkıran\\n_START_SECTION_\\nLaufbahn\\n_START_PARAGRAPH_\\nDemirkıran debütierte als junges Talent am 25. September 1999 im Auswärtsspiel des SSV Ulm 1846 bei Werder Bremen (2:2) in der Bundesliga, als er kurz\",\n",
+ " \"it\": \"\\n_START_ARTICLE_\\n28th Street (linea IRT Lexington Avenue)\\n_START_SECTION_\\nStoria\\n_START_PARAGRAPH_\\nLa stazione, i cui lavori di costruzione ebbero inizio nel 1900, venne aperta il 27 ottobre 1904, come\",\n",
+ " \"ja\": \"\\n_START_ARTICLE_\\nしのぶ・まさみshow'05 恋してラララ\\n_START_SECTION_\\n概要\\n_START_PARAGRAPH_\\n『上海ルーキーSHOW』の打ち切り後に放送された年末特番で、同番組MCの大竹しのぶと久本雅美が恋愛にまつわるテーマでトークや音楽企画を展開していた。基本は女\",\n",
+ " \"ko\": \"\\n_START_ARTICLE_\\n녹턴, Op. 9 (쇼팽)\\n_START_SECTION_\\n녹턴 3번 나장조\\n_START_PARAGRAPH_\\n쇼팽의 녹턴 3번은 세도막 형식인 (A-B-A)형식을 취하고 있다. 첫 부분은 알레그레토(Allegretto)의 빠르기가 지시되어 있으며 물 흐르듯이 부드럽게 전개되나\",\n",
+ " \"pl\": \"\\n_START_ARTICLE_\\nAK-176\\n_START_SECTION_\\nHistoria\\n_START_PARAGRAPH_\\nPod koniec lat 60 XX w. w ZSRR dostrzeżono potrzebę posiadania lekkiej armaty uniwersalnej średniego kalibru o stosunkowo dużej mocy ogniowej, która\",\n",
+ " \"pt\": \"\\n_START_ARTICLE_\\nÁcido ribonucleico\\n_START_SECTION_\\nIntermediário da transferência de informação\\n_START_PARAGRAPH_\\nEm 1957 Elliot Volkin e Lawrence Astrachan fizeram uma observação significativa. Eles descobriram que uma das mais marcantes mudanças\",\n",
+ " \"ru\": \"\\n_START_ARTICLE_\\nАрнольд, Ремо\\n_START_SECTION_\\nКлубная карьера\\n_START_PARAGRAPH_\\nАрнольд перешёл в академию «Люцерна» в 12 лет. С 2014 года выступал за вторую команду, где провёл пятнадцать встреч. С сезона 2015/2016 находится в составе основной команды. 27 сентября 2015 года дебютировал\",\n",
+ " \"es\": \"\\n_START_ARTICLE_\\n(200012) 2007 LK20\\n_START_SECTION_\\nDesignación y nombre\\n_START_PARAGRAPH_\\nDesignado provisionalmente como 2007 LK20.\\n_START_SECTION_\\nCaracterísticas orbitales\\n_START_PARAGRAPH_\\n2007 LK20\",\n",
+ " \"th\": \"\\n_START_ARTICLE_\\nการนัดหยุดเรียนเพื่อภูมิอากาศ\\n_START_SECTION_\\nเกรียตา ทืนแบร์ย\\n_START_PARAGRAPH_\\nวันที่ 20 สิงหาคม 2561 เกรียตา ทืนแบร์ย นักกิจกรรมภูมิอากาศชาวสวีเดน ซึ่งขณะนั้นศึกษาอยู่ในชั้นเกรด 9 (เทียบเท่ามัธยมศึกษาปีที่ 3) ตัดสินใจไม่เข้าเรียนจนกระทั่งการเลือกตั้งทั่วไปในประเทศสวีเดนปี\",\n",
+ " \"tr\": \"\\n_START_ARTICLE_\\nİsrail'in Muhafazakar Dostları\\n_START_SECTION_\\nFaaliyetleri\\n_START_PARAGRAPH_\\nGrubun 2005 stratejisi ile aşağıdaki faaliyet alanları tespit edilmiştir:_NEWLINE_İsrail'i destekleme\",\n",
+ " \"bg\": \"\\n_START_ARTICLE_\\nАвтомобил с повишена проходимост\\n_START_SECTION_\\nОсобености на конструкцията\\n_START_PARAGRAPH_\\nВ исторически план леки автомобили с висока проходимост се произвеждат и имат военно\",\n",
+ " \"ca\": \"\\n_START_ARTICLE_\\nAuchy-la-Montagne\\n_START_SECTION_\\nPoblació\\n_START_PARAGRAPH_\\nEl 2007 la població de fet d'Auchy-la-Montagne era de 469 persones. Hi havia 160 famílies de les quals 28\",\n",
+ " \"cs\": \"\\n_START_ARTICLE_\\nŘemeslo\\n_START_PARAGRAPH_\\nŘemeslo je určitý druh manuální dovednosti, provozovaný za účelem obživy, resp. vytváření zisku. Pro řemeslné práce je charakteristický vysoký podíl ruční práce, spojený s používáním specializovaných nástrojů a pomůcek. Řemeslné práce\",\n",
+ " \"da\": \"\\n_START_ARTICLE_\\nÖrenäs slot\\n_START_PARAGRAPH_\\nÖrenäs slot (svensk: Örenäs slott) er et slot nær Glumslöv i Landskrona stad tæt på Øresunds-kysten i Skåne i Sverige._NEWLINE_Örenäs ligger\",\n",
+ " \"el\": \"\\n_START_ARTICLE_\\nΆλβαρο Ρεκόμπα\\n_START_SECTION_\\nΒιογραφικά στοιχεία\\n_START_PARAGRAPH_\\nΟ Άλβαρο Ρεκόμπα γεννήθηκε στις 17 Μαρτίου 1976 στο Μοντεβίδεο της Ουρουγουάης από\",\n",
+ " \"et\": \"\\n_START_ARTICLE_\\nAus deutscher Geistesarbeit\\n_START_PARAGRAPH_\\nAus deutscher Geistesarbeit (alapealkiri Wochenblatt für wissenschaftliche und kulturelle Fragen der Gegenwart) oli ajakiri, mis 1924–1934 ilmus Tallinnas. Ajakirja andis 1932–1934\",\n",
+ " \"fa\": \"\\n_START_ARTICLE_\\nتفسیر بغوی\\n_START_PARAGRAPH_\\nایرانی حسین بن مسعود بغوی است. این کتاب خلاصه ای از تفسیر الکشف و البیان عن تفسیر القرآن ابواسحاق احمد ثعلبی میباشد. این کتاب در ۴ جلد موجود میباش\",\n",
+ " \"fi\": \"\\n_START_ARTICLE_\\nBovesin verilöyly\\n_START_SECTION_\\nVerilöyly\\n_START_PARAGRAPH_\\n19. syyskuuta 1943 partisaaniryhmä saapui Bovesiin tarkoituksenaan ostaa leipää kylästä. Kylässä sattui olemaan kaksi SS-miestä, jotka\",\n",
+ " \"he\": \"\\n_START_ARTICLE_\\nאוגדה 85\\n_START_SECTION_\\nהיסטוריה\\n_START_PARAGRAPH_\\nהאוגדה הוקמה בהתחלה כמשלט העמקים בשנות השבעים. בשנות השמונים הפכה להיות אוגדה מרחבית עם שתי\",\n",
+ " \"hi\": \"\\n_START_ARTICLE_\\nऑडी\\n_START_SECTION_\\nऑडी इंडिया\\n_START_PARAGRAPH_\\nऑडी इंडिया की स्थापना मार्च 2007 में फोक्सवैगन ग्रुप सेल्स इंडिया के एक विभाजन के रूप में की गई थी। दुनिया भर में 110\",\n",
+ " \"hr\": \"\\n_START_ARTICLE_\\nČimariko (jezična porodica)\\n_START_PARAGRAPH_\\nChimarikan.-porodica sjevernoameričkih indijanskih jezika koja prema Powersu obuhvaća jezike Indijanaca Chimariko (Chemaŕeko) sa rijeke Trinity i Chimalakwe\",\n",
+ " \"hu\": \"\\n_START_ARTICLE_\\nÁllami Politikai Igazgatóság\\n_START_PARAGRAPH_\\nAz Állami Politikai Igazgatóság (rövidítve: GPU, oroszul: Государственное политическое управление), majd később Egyesített Állami Politikai Igazgatóság Szovjet-Oroszország\",\n",
+ " \"id\": \"\\n_START_ARTICLE_\\n(257195) 2008 QY41\\n_START_SECTION_\\nPembentukan\\n_START_PARAGRAPH_\\nSeperti asteroid secara keseluruhan, asteroid ini terbentuk dari nebula matahari primordial sebagai pecahan planetisimal, sesuatu di\",\n",
+ " \"lt\": \"\\n_START_ARTICLE_\\nŠavijos–Uardigo regionas\\n_START_SECTION_\\nGeografija\\n_START_PARAGRAPH_\\nŠavijos-Uardigo regionas yra Atlanto vandenynu pakrantės lygumoje\",\n",
+ " \"lv\": \"\\n_START_ARTICLE_\\nApatīts\\n_START_SECTION_\\nĪpašības\\n_START_PARAGRAPH_\\nApatīta kopējā ķīmiskā formula ir Ca₁₀(PO₄)₆(OH,F,Cl)₂, ir trīs atšķirīgi apatīta veidi: apatīts: Ca₁₀(PO₄)₆(OH)₂, fluorapatīts Ca₁₀(PO₄)₆(F)₂ un hlorapatīts: Ca₁₀(PO₄)₆(Cl)₂. Pēc sastāva\",\n",
+ " \"ms\": \"\\n_START_ARTICLE_\\nEdward C. Prescott\\n_START_PARAGRAPH_\\nEdward Christian Prescott (lahir 26 Disember 1940) ialah seorang ahli ekonomi Amerika. Beliau menerima Hadiah Peringatan Nobel dalam Sains Ekonomi pada tahun 2004, berkongsi\",\n",
+ " \"no\": \"\\n_START_ARTICLE_\\nAl-Minya\\n_START_SECTION_\\nEtymologi\\n_START_PARAGRAPH_\\nDet er sprikende forklaringer på bynavnet. Det kan komme fra gammelegyptisk Men'at Khufu, i betydning byen hvor Khufu ble ammet, noe som knytter byen til farao Khufu (Keops), som\",\n",
+ " \"ro\": \"\\n_START_ARTICLE_\\nDealurile Cernăuțiului\\n_START_PARAGRAPH_\\nDealurile Cernăuțiului sunt un lanț deluros striat, care se întinde în partea centrală a interfluviului dintre Prut și Siret, în cadrul regiunii Cernăuți din\",\n",
+ " \"sk\": \"\\n_START_ARTICLE_\\n10. peruť RAAF\\n_START_PARAGRAPH_\\n10. peruť RAAF je námorná hliadkovacia peruť kráľovských austrálskych vzdušných síl (Royal Australian Air Force – RAAF) založená na základni Edinburgh v Južnej Austrálii ako súčasť 92\",\n",
+ " \"sl\": \"\\n_START_ARTICLE_\\n105 Artemida\\n_START_SECTION_\\nOdkritje\\n_START_PARAGRAPH_\\nAsteroid je 16. septembra 1868 odkril James Craig Watson (1838 – 1880). Poimenovan je po Artemidi, boginji Lune iz grške\",\n",
+ " \"sr\": \"\\n_START_ARTICLE_\\nЉанос Морелос 1. Сексион (Истапангахоја)\\n_START_SECTION_\\nСтановништво\\n_START_PARAGRAPH_\\nПрема подацима из 2010. године у насељу је живело 212\",\n",
+ " \"sv\": \"\\n_START_ARTICLE_\\nÖstra Torps landskommun\\n_START_SECTION_\\nAdministrativ historik\\n_START_PARAGRAPH_\\nKommunen bildades i Östra Torps socken i Vemmenhögs härad i Skåne när 1862 års kommunalförordningar trädde i kraft. _NEWLINE_Vid kommunreformen\",\n",
+ " \"tl\": \"\\n_START_ARTICLE_\\nBésame Mucho\\n_START_PARAGRAPH_\\nAng Bésame Mucho ay isang awit na nasa Kastila. Isinulat ito ng Mehikanang si Consuelo Velázquez noong 1940, bago sumapit ang kanyang ika-16 na\",\n",
+ " \"uk\": \"\\n_START_ARTICLE_\\nІслам та інші релігії\\n_START_PARAGRAPH_\\nПротягом багатовікової ісламської історії мусульманські правителі, ісламські вчені і звичайні мусульмани вступали у різні відносини з представниками інших релігій. Стиль цих\",\n",
+ " \"vi\": \"\\n_START_ARTICLE_\\nĐường tỉnh 316\\n_START_PARAGRAPH_\\nĐường tỉnh 316 hay tỉnh lộ 316, viết tắt ĐT316 hay TL316, là đường tỉnh ở các huyện Thanh Sơn, Thanh Thủy, Tam Nông tỉnh Phú Thọ ._NEWLINE_ĐT316 bắt đầu từ xã Tinh Nhuệ\",\n",
+ " \"multilingual-64k\": \"\\n_START_ARTICLE_\\n1882 Prince Edward Island general election\\n_START_PARAGRAPH_\\nThe 1882 Prince Edward Island election was held on May 8, 1882 to elect members of the House of Assembly of the province of Prince Edward Island, Canada.\",\n",
+ " \"multilingual-128k\": \"\\n_START_ARTICLE_\\n1882 Prince Edward Island general election\\n_START_PARAGRAPH_\\nThe 1882 Prince Edward Island election was held on May 8, 1882 to elect members of the House of Assembly of the province of Prince Edward Island, Canada.\"}\n",
+ "\n",
+ "seed = lang_to_seed[language]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "cellView": "form",
+ "id": "mZDGsSyUM_Mg"
+ },
+ "outputs": [],
+ "source": [
+ "#@title Enter your own seed (Optional).\n",
+ "user_seed = \"\" #@param { type: \"string\" }\n",
+ "if user_seed.strip():\n",
+ " seed = user_seed.strip()\n",
+ "\n",
+ "# The seed must start with \"_START_ARTICLE_\" or the generated text will be gibberish\n",
+ "START_ARTICLE = \"_START_ARTICLE_\"\n",
+ "if START_ARTICLE not in seed:\n",
+ " seed = \"\\n{}\\n{}\".format(START_ARTICLE, seed)\n",
+ "\n",
+ "print(\"Generating text from seed:\\n{}\".format(seed))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "cellView": "form",
+ "id": "5dMuShi3XuLd"
+ },
+ "outputs": [],
+ "source": [
+ "#@title Initialize session.\n",
+ "with tf.Session(graph=g).as_default() as session:\n",
+ " session.run(init_op)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "cellView": "form",
+ "id": "aS53xjmbbw0Z"
+ },
+ "outputs": [],
+ "source": [
+ "#@title Generate text\n",
+ "\n",
+ "with session.as_default():\n",
+ " results = session.run([embeddings, neg_log_likelihood, ppl, activations, token_ids, generated_text], feed_dict={text: [seed]})\n",
+ " embeddings_result, neg_log_likelihood_result, ppl_result, activations_result, token_ids_result, generated_text_result = results\n",
+ " generated_text_output = generated_text_result[0].decode('utf-8')\n",
+ "\n",
+ "print(generated_text_output)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "tjQf3N1wdND0"
+ },
+ "source": [
+ "We can also look at the other outputs of the model - the perplexity, the token ids, the intermediate activations, and the embeddings"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "pGfw3CQWNC_n"
+ },
+ "outputs": [],
+ "source": [
+ "ppl_result"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "FLlgJObFNEmj"
+ },
+ "outputs": [],
+ "source": [
+ "token_ids_result"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "5SaH36M-NGXc"
+ },
+ "outputs": [],
+ "source": [
+ "activations_result.shape"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "k9Eb_DPfQdUu"
+ },
+ "outputs": [],
+ "source": [
+ "embeddings_result"
+ ]
+ }
+ ],
+ "metadata": {
+ "colab": {
+ "collapsed_sections": [],
+ "name": "wiki40b_lm.ipynb",
+ "toc_visible": true
+ },
+ "kernelspec": {
+ "display_name": "Python 3",
+ "name": "python3"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}
diff --git a/site/en/hub/tutorials/yamnet.ipynb b/site/en/hub/tutorials/yamnet.ipynb
new file mode 100644
index 00000000000..e6c9fbca5a1
--- /dev/null
+++ b/site/en/hub/tutorials/yamnet.ipynb
@@ -0,0 +1,359 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "laa9tRjJ59bl"
+ },
+ "source": [
+ "##### Copyright 2020 The TensorFlow Hub Authors.\n",
+ "\n",
+ "Licensed under the Apache License, Version 2.0 (the \"License\");"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "cellView": "form",
+ "id": "T4ZHtBpK6Dom"
+ },
+ "outputs": [],
+ "source": [
+ "#@title Copyright 2020 The TensorFlow Hub Authors. All Rights Reserved.\n",
+ "#\n",
+ "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+ "# you may not use this file except in compliance with the License.\n",
+ "# You may obtain a copy of the License at\n",
+ "#\n",
+ "# http://www.apache.org/licenses/LICENSE-2.0\n",
+ "#\n",
+ "# Unless required by applicable law or agreed to in writing, software\n",
+ "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+ "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+ "# See the License for the specific language governing permissions and\n",
+ "# limitations under the License.\n",
+ "# =============================================================================="
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "hk5u_9KN1m-t"
+ },
+ "source": [
+ "
# Requires the latest pip
pip install --upgrade pip
- # Current stable release for CPU and GPU
+ # Current stable release for CPU
pip install tensorflow
+ # Current stable release for GPU (Linux / WSL2)
+ pip install tensorflow[and-cuda]
# Or try the preview build (unstable)
pip install tf-nightly
diff --git a/site/en/install/docker.md b/site/en/install/docker.md
index 30942924688..376ca0820a7 100644
--- a/site/en/install/docker.md
+++ b/site/en/install/docker.md
@@ -1,45 +1,43 @@
# Docker
-[Docker](https://docs.docker.com/install/){:.external} uses *containers* to
+[Docker](https://docs.docker.com/install/) uses *containers* to
create virtual environments that isolate a TensorFlow installation from the rest
of the system. TensorFlow programs are run *within* this virtual environment that
can share resources with its host machine (access directories, use the GPU,
connect to the Internet, etc.). The
-[TensorFlow Docker images](https://hub.docker.com/r/tensorflow/tensorflow/){:.external}
+[TensorFlow Docker images](https://hub.docker.com/r/tensorflow/tensorflow/)
are tested for each release.
-Docker is the easiest way to enable TensorFlow [GPU support](./gpu.md) on Linux since only the
-[NVIDIA® GPU driver](https://github.com/NVIDIA/nvidia-docker/wiki/Frequently-Asked-Questions#how-do-i-install-the-nvidia-driver){:.external}
+Docker is the easiest way to enable TensorFlow [GPU support](./pip.md) on Linux since only the
+[NVIDIA® GPU driver](https://github.com/NVIDIA/nvidia-docker/wiki/Frequently-Asked-Questions#how-do-i-install-the-nvidia-driver)
is required on the *host* machine (the *NVIDIA® CUDA® Toolkit* does not need to
be installed).
## TensorFlow Docker requirements
-1. [Install Docker](https://docs.docker.com/install/){:.external} on
+1. [Install Docker](https://docs.docker.com/install/) on
your local *host* machine.
-2. For GPU support on Linux, [install NVIDIA Docker support](https://github.com/NVIDIA/nvidia-docker){:.external}.
+2. For GPU support on Linux, [install NVIDIA Docker support](https://github.com/NVIDIA/nvidia-container-toolkit).
* Take note of your Docker version with `docker -v`. Versions __earlier than__ 19.03 require nvidia-docker2 and the `--runtime=nvidia` flag. On versions __including and after__ 19.03, you will use the `nvidia-container-toolkit` package and the `--gpus all` flag. Both options are documented on the page linked above.
Note: To run the `docker` command without `sudo`, create the `docker` group and
add your user. For details, see the
-[post-installation steps for Linux](https://docs.docker.com/install/linux/linux-postinstall/){:.external}.
+[post-installation steps for Linux](https://docs.docker.com/install/linux/linux-postinstall/).
## Download a TensorFlow Docker image
The official TensorFlow Docker images are located in the
-[tensorflow/tensorflow](https://hub.docker.com/r/tensorflow/tensorflow/){:.external}
-Docker Hub repository. Image releases [are tagged](https://hub.docker.com/r/tensorflow/tensorflow/tags/){:.external}
+[tensorflow/tensorflow](https://hub.docker.com/r/tensorflow/tensorflow/)
+Docker Hub repository. Image releases [are tagged](https://hub.docker.com/r/tensorflow/tensorflow/tags/)
using the following format:
| Tag | Description |
|-------------|----------------------------------------------------------------------------------------------------------------------|
| `latest` | The latest release of TensorFlow CPU binary image. Default. |
| `nightly` | Nightly builds of the TensorFlow image. (Unstable.) |
-| *`version`* | Specify the *version* of the TensorFlow binary image, for example\: *2.1.0* |
-| `devel` | Nightly builds of a TensorFlow `master` development environment. Includes TensorFlow source code. |
-| `custom-op` | Special experimental image for developing TF custom ops. More info [here](https://github.com/tensorflow/custom-op). |
+| *`version`* | Specify the *version* of the TensorFlow binary image, for example\: *2.8.3* |
Each base *tag* has variants that add or change functionality:
@@ -66,7 +64,7 @@ To start a TensorFlow-configured container, use the following command form:
docker run [-it] [--rm] [-p hostPort:containerPort] tensorflow/tensorflow[:tag] [command]
-For details, see the [docker run reference](https://docs.docker.com/engine/reference/run/){:.external}.
+For details, see the [docker run reference](https://docs.docker.com/engine/reference/run/).
### Examples using CPU-only images
@@ -100,7 +98,7 @@ docker run -it --rm -v $PWD:/tmp -w /tmp tensorflow/tensorflow python ./script.p
Permission issues can arise when files created within a container are exposed to
the host. It's usually best to edit files on the host system.
-Start a [Jupyter Notebook](https://jupyter.org/){:.external} server using
+Start a [Jupyter Notebook](https://jupyter.org/) server using
TensorFlow's nightly build:
@@ -114,13 +112,13 @@ Follow the instructions and open the URL in your host web browser:
## GPU support
Docker is the easiest way to run TensorFlow on a GPU since the *host* machine
-only requires the [NVIDIA® driver](https://github.com/NVIDIA/nvidia-docker/wiki/Frequently-Asked-Questions#how-do-i-install-the-nvidia-driver){:.external}
+only requires the [NVIDIA® driver](https://github.com/NVIDIA/nvidia-docker/wiki/Frequently-Asked-Questions#how-do-i-install-the-nvidia-driver)
(the *NVIDIA® CUDA® Toolkit* is not required).
-Install the [Nvidia Container Toolkit](https://github.com/NVIDIA/nvidia-docker/blob/master/README.md#quickstart){:.external}
+Install the [Nvidia Container Toolkit](https://github.com/NVIDIA/nvidia-docker/blob/master/README.md#quickstart)
to add NVIDIA® GPU support to Docker. `nvidia-container-runtime` is only
available for Linux. See the `nvidia-container-runtime`
-[platform support FAQ](https://github.com/NVIDIA/nvidia-docker/wiki/Frequently-Asked-Questions#platform-support){:.external}
+[platform support FAQ](https://github.com/NVIDIA/nvidia-docker/wiki/Frequently-Asked-Questions#platform-support)
for details.
Check if a GPU is available:
@@ -132,7 +130,7 @@ lspci | grep -i nvidia
Verify your `nvidia-docker` installation:
-docker run --gpus all --rm nvidia/cuda nvidia-smi
+docker run --rm --runtime=nvidia --gpus all ubuntu nvidia-smi
Note: `nvidia-docker` v2 uses `--runtime=nvidia` instead of `--gpus all`. `nvidia-docker` v1 uses the `nvidia-docker` alias,
diff --git a/site/en/install/errors.md b/site/en/install/errors.md
index 158b165f492..938ba8b454f 100644
--- a/site/en/install/errors.md
+++ b/site/en/install/errors.md
@@ -1,8 +1,8 @@
# Build and install error messages
-TensorFlow uses [GitHub issues](https://github.com/tensorflow/tensorflow/issues){:.external},
-[Stack Overflow](https://stackoverflow.com/questions/tagged/tensorflow){:.external} and
-[TensorFlow Forum](https://discuss.tensorflow.org/c/general-discussion/6){:.external}
+TensorFlow uses [GitHub issues](https://github.com/tensorflow/tensorflow/issues),
+[Stack Overflow](https://stackoverflow.com/questions/tagged/tensorflow) and
+[TensorFlow Forum](https://discuss.tensorflow.org/c/general-discussion/6)
to track, document, and discuss build and installation problems.
The following list links error messages to a solution or discussion. If you find
@@ -61,7 +61,7 @@ CodedInputStream::SetTotalBytesLimit() in google/protobuf/io/coded_stream.h.
diff --git a/site/en/install/gpu_plugins.md b/site/en/install/gpu_plugins.md
index 290cd48cf76..39e3cf09b29 100644
--- a/site/en/install/gpu_plugins.md
+++ b/site/en/install/gpu_plugins.md
@@ -4,9 +4,9 @@ Note: This page is for non-NVIDIA® GPU devices. For NVIDIA® GPU support, go to
the [Install TensorFlow with pip](./pip.md) guide.
TensorFlow's
-[pluggable device](https://github.com/tensorflow/community/blob/master/rfcs/20200624-pluggable-device-for-tensorflow.md){.external}
-architecture adds new device support as separate plug-in packages
-that are installed alongside the official TensorFlow package.
+[pluggable device](https://github.com/tensorflow/community/blob/master/rfcs/20200624-pluggable-device-for-tensorflow.md)
+architecture adds new device support as separate plug-in packages that are
+installed alongside the official TensorFlow package.
The mechanism requires no device-specific changes in the TensorFlow code. It
relies on C APIs to communicate with the TensorFlow binary in a stable manner.
@@ -57,12 +57,24 @@ run() # PluggableDevices also work with tf.function and graph mode.
Metal `PluggableDevice` for macOS GPUs:
-* [Getting started guide](https://developer.apple.com/metal/tensorflow-plugin/){:.external}.
+* Works with TF 2.5 or later.
+* [Getting started guide](https://developer.apple.com/metal/tensorflow-plugin/).
* For questions and feedback, please visit the
- [Apple Developer Forum](https://developer.apple.com/forums/tags/tensorflow-metal){:.external}.
+ [Apple Developer Forum](https://developer.apple.com/forums/tags/tensorflow-metal).
DirectML `PluggableDevice` for Windows and WSL (preview):
-* [PyPI wheel](https://pypi.org/project/tensorflow-directml-plugin/){:.external}.
-* [GitHub repo](https://github.com/microsoft/tensorflow-directml-plugin){.external}.
-* For questions, feedback or to raise issues, please visit the [Issues page of `tensorflow-directml-plugin` on GitHub](https://github.com/microsoft/tensorflow-directml-plugin/issues){:.external}.
+* Works with `tensorflow-cpu` package, version 2.10 or later.
+* [PyPI wheel](https://pypi.org/project/tensorflow-directml-plugin/).
+* [GitHub repo](https://github.com/microsoft/tensorflow-directml-plugin).
+* For questions, feedback or to raise issues, please visit the
+ [Issues page of `tensorflow-directml-plugin` on GitHub](https://github.com/microsoft/tensorflow-directml-plugin/issues).
+
+Intel® Extension for TensorFlow `PluggableDevice` for Linux and WSL:
+
+* Works with TF 2.10 or later.
+* [Getting started guide](https://intel.github.io/intel-extension-for-tensorflow/latest/get_started.html)
+* [PyPI wheel](https://pypi.org/project/intel-extension-for-tensorflow/).
+* [GitHub repo](https://github.com/intel/intel-extension-for-tensorflow).
+* For questions, feedback, or to raise issues, please visit the
+ [Issues page of `intel-extension-for-tensorflow` on GitHub](https://github.com/intel/intel-extension-for-tensorflow/issues).
diff --git a/site/en/install/lang_c.ipynb b/site/en/install/lang_c.ipynb
index da4bbf03fc6..788a5e6c891 100644
--- a/site/en/install/lang_c.ipynb
+++ b/site/en/install/lang_c.ipynb
@@ -127,30 +127,48 @@
"\n",
"
"
]
},
{
@@ -174,8 +192,8 @@
"outputs": [],
"source": [
"%%bash\n",
- "FILENAME=libtensorflow-cpu-linux-x86_64-2.8.0.tar.gz\n",
- "wget -q --no-check-certificate https://storage.googleapis.com/tensorflow/libtensorflow/${FILENAME}\n",
+ "FILENAME=libtensorflow-cpu-linux-x86_64.tar.gz\n",
+ "wget -q --no-check-certificate https://storage.googleapis.com/tensorflow/versions/2.18.1/${FILENAME}\n",
"sudo tar -C /usr/local -xzf ${FILENAME}"
]
},
@@ -344,15 +362,15 @@
"## Build from source\n",
"\n",
"TensorFlow is open source. Read\n",
- "[the instructions](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/tools/lib_package/README.md){:.external}\n",
+ "[the instructions](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/tools/lib_package/README.md)\n",
"to build TensorFlow's C library from source code."
]
}
],
"metadata": {
"colab": {
- "collapsed_sections": [],
"name": "lang_c.ipynb",
+ "provenance": [],
"toc_visible": true
},
"kernelspec": {
diff --git a/site/en/install/lang_java_legacy.md b/site/en/install/lang_java_legacy.md
index af177dc0950..37341c36659 100644
--- a/site/en/install/lang_java_legacy.md
+++ b/site/en/install/lang_java_legacy.md
@@ -1,7 +1,7 @@
# Install TensorFlow for Java
Warning: TensorFlow for Java is deprecated and will be removed in a future
-version of TensorFlow once the replacement is stable.
+version of TensorFlow once [the replacement](https://www.tensorflow.org/jvm) is stable.
TensorFlow provides a
[Java API](https://www.tensorflow.org/api_docs/java/reference/org/tensorflow/package-summary)—
@@ -27,7 +27,7 @@ To use TensorFlow on Android see [TensorFlow Lite](https://tensorflow.org/lite)
## TensorFlow with Apache Maven
-To use TensorFlow with [Apache Maven](https://maven.apache.org){:.external},
+To use TensorFlow with [Apache Maven](https://maven.apache.org),
add the dependency to the project's `pom.xml` file:
```xml
@@ -40,7 +40,7 @@ add the dependency to the project's `pom.xml` file:
### GPU support
-If your system has [GPU support](./gpu.md), add the following TensorFlow
+If your system has [GPU support](./pip.md), add the following TensorFlow
dependencies to the project's `pom.xml` file:
```xml
@@ -167,11 +167,11 @@ system and processor support:
Note: On Windows, the native library (`tensorflow_jni.dll`) requires
`msvcp140.dll` at runtime. See the
[Windows build from source](./source_windows.md) guide to install the
-[Visual C++ 2019 Redistributable](https://visualstudio.microsoft.com/vs/){:.external}.
+[Visual C++ 2019 Redistributable](https://visualstudio.microsoft.com/vs/).
### Compile
-Using the `HelloTensorFlow.java` file from the [previous example](#example),
+Using the `HelloTensorFlow.java` file from the [previous example](#example-program),
compile a program that uses TensorFlow. Make sure the `libtensorflow.jar` is
accessible to your `classpath`:
@@ -203,5 +203,5 @@ Success: TensorFlow for Java is configured.
## Build from source
TensorFlow is open source. Read
-[the instructions](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/java/README.md){:.external}
+[the instructions](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/java/README.md)
to build TensorFlow's Java and native libraries from source code.
diff --git a/site/en/install/pip.md b/site/en/install/pip.md
index 6bc4a61e5c8..a9e4bf4bf74 100644
--- a/site/en/install/pip.md
+++ b/site/en/install/pip.md
@@ -1,69 +1,121 @@
+
# Install TensorFlow with pip
+
This guide is for the latest stable version of TensorFlow. For the
-preview build *(nightly)*, please use the pip package named
+preview build *(nightly)*, use the pip package named
`tf-nightly`. Refer to [these tables](./source#tested_build_configurations) for
-older TensorFlow version requirements. For TensorFlow 1.x users, please refer to
-the [migration guide](../guide/migrate) to upgrade to TensorFlow 2.
+older TensorFlow version requirements. For the CPU-only build, use the pip
+package named `tensorflow-cpu`.
-Here is a lookup table for the install commands. Scroll down for the
+Here are the quick versions of the install commands. Scroll down for the
step-by-step instructions.
* {Linux}
- ```bash
- conda install -c conda-forge cudatoolkit=11.2 cudnn=8.1.0
- export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$CONDA_PREFIX/lib/
- python3 -m pip install tensorflow
- # Verify install:
- python3 -c "import tensorflow as tf; print(tf.config.list_physical_devices('GPU'))"
- ```
+ Note: Starting with TensorFlow `2.10`, Linux CPU-builds for Aarch64/ARM64
+ processors are built, maintained, tested and released by a third party:
+ [AWS](https://aws.amazon.com/).
+ Installing the [`tensorflow`](https://pypi.org/project/tensorflow/)
+ package on an ARM machine installs AWS's
+ [`tensorflow-cpu-aws`](https://pypi.org/project/tensorflow-cpu-aws/) package.
+ They are provided as-is. Tensorflow will use reasonable efforts to maintain
+ the availability and integrity of this pip package. There may be delays if
+ the third party fails to release the pip package. See
+ [this blog post](https://blog.tensorflow.org/2022/09/announcing-tensorflow-official-build-collaborators.html)
+ for more information about this collaboration.
+
+ ```bash
+ python3 -m pip install 'tensorflow[and-cuda]'
+ # Verify the installation:
+ python3 -c "import tensorflow as tf; print(tf.config.list_physical_devices('GPU'))"
+ ```
* {MacOS}
- ```bash
- # Currently, we do not have official GPU support for MacOS.
- python3 -m pip install tensorflow
- # Verify install:
- python3 -c "import tensorflow as tf; print(tf.reduce_sum(tf.random.normal([1000, 1000])))"
- ```
+ ```bash
+ # There is currently no official GPU support for MacOS.
+ python3 -m pip install tensorflow
+ # Verify the installation:
+ python3 -c "import tensorflow as tf; print(tf.reduce_sum(tf.random.normal([1000, 1000])))"
+ ```
+
+* {Windows Native}
+
+ Caution: TensorFlow `2.10` was the **last** TensorFlow release that
+ supported GPU on native-Windows.
+ Starting with TensorFlow `2.11`, you will need to install
+ [TensorFlow in WSL2](https://tensorflow.org/install/pip#windows-wsl2),
+ or install `tensorflow` or `tensorflow-cpu` and, optionally, try the
+ [TensorFlow-DirectML-Plugin](https://github.com/microsoft/tensorflow-directml-plugin#tensorflow-directml-plugin-)
+
+ ```bash
+ conda install -c conda-forge cudatoolkit=11.2 cudnn=8.1.0
+ # Anything above 2.10 is not supported on the GPU on Windows Native
+ python -m pip install "tensorflow<2.11"
+ # Verify the installation:
+ python -c "import tensorflow as tf; print(tf.config.list_physical_devices('GPU'))"
+ ```
-* {Windows}
+* {Windows WSL2}
- ```bash
- conda install -c conda-forge cudatoolkit=11.2 cudnn=8.1.0
- python3 -m pip install tensorflow
- # Verify install:
- python3 -c "import tensorflow as tf; print(tf.config.list_physical_devices('GPU'))"
- ```
+ Note: TensorFlow with GPU access is supported for WSL2 on Windows 10 19044 or
+ higher. This corresponds to Windows 10 version 21H2, the November 2021
+ update. You can get the latest update from here:
+ [Download Windows 10](https://www.microsoft.com/software-download/windows10).
+ For instructions, see
+ [Install WSL2](https://docs.microsoft.com/windows/wsl/install)
+ and
+ [NVIDIA’s setup docs](https://docs.nvidia.com/cuda/wsl-user-guide/index.html)
+ for CUDA in WSL.
+
+ ```bash
+ python3 -m pip install tensorflow[and-cuda]
+ # Verify the installation:
+ python3 -c "import tensorflow as tf; print(tf.config.list_physical_devices('GPU'))"
+ ```
* {CPU}
- ```bash
- python3 -m pip install tensorflow
- # Verify install:
- python3 -c "import tensorflow as tf; print(tf.reduce_sum(tf.random.normal([1000, 1000])))"
- ```
+ Note: Starting with TensorFlow `2.10`, Windows CPU-builds for x86/x64
+ processors are built, maintained, tested and released by a third party:
+ [Intel](https://www.intel.com/).
+ Installing the Windows-native [`tensorflow`](https://pypi.org/project/tensorflow/)
+ or [`tensorflow-cpu`](https://pypi.org/project/tensorflow-cpu/)
+ package installs Intel's
+ [`tensorflow-intel`](https://pypi.org/project/tensorflow-intel/)
+ package. These packages are provided as-is. Tensorflow will use reasonable
+ efforts to maintain the availability and integrity of this pip package.
+ There may be delays if the third party fails to release the pip package. See
+ [this blog post](https://blog.tensorflow.org/2022/09/announcing-tensorflow-official-build-collaborators.html)
+ for more information about this
+ collaboration.
+
+ ```bash
+ python3 -m pip install tensorflow
+ # Verify the installation:
+ python3 -c "import tensorflow as tf; print(tf.reduce_sum(tf.random.normal([1000, 1000])))"
+ ```
* {Nightly}
- ```bash
- python3 -m pip install tf-nightly
- # Verify install:
- python3 -c "import tensorflow as tf; print(tf.reduce_sum(tf.random.normal([1000, 1000])))"
- ```
+ ```bash
+ python3 -m pip install tf-nightly
+ # Verify the installation:
+ python3 -c "import tensorflow as tf; print(tf.reduce_sum(tf.random.normal([1000, 1000])))"
+ ```
## Hardware requirements
Note: TensorFlow binaries use
-[AVX instructions](https://en.wikipedia.org/wiki/Advanced_Vector_Extensions#CPUs_with_AVX){:.external}
+[AVX instructions](https://en.wikipedia.org/wiki/Advanced_Vector_Extensions#CPUs_with_AVX)
which may not run on older CPUs.
The following GPU-enabled devices are supported:
* NVIDIA® GPU card with CUDA® architectures 3.5, 5.0, 6.0, 7.0, 7.5, 8.0 and
higher. See the list of
- [CUDA®-enabled GPU cards](https://developer.nvidia.com/cuda-gpus){:.external}.
+ [CUDA®-enabled GPU cards](https://developer.nvidia.com/cuda-gpus).
* For GPUs with unsupported CUDA® architectures, or to avoid JIT compilation
from PTX, or to use different versions of the NVIDIA® libraries, see the
[Linux build from source](./source.md) guide.
@@ -71,7 +123,7 @@ The following GPU-enabled devices are supported:
architecture; therefore, TensorFlow fails to load on older GPUs when
`CUDA_FORCE_PTX_JIT=1` is set. (See
[Application Compatibility](https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#application-compatibility)
- for details.) {:.external}
+ for details.)
Note: The error message "Status: device kernel image is invalid" indicates that
the TensorFlow package does not contain PTX for your architecture. You can
@@ -80,79 +132,61 @@ enable compute capabilities by [building TensorFlow from source](./source.md).
## System requirements
* Ubuntu 16.04 or higher (64-bit)
-* macOS 10.12.6 (Sierra) or higher (64-bit) *(no GPU support)*
-* Windows 7 or higher (64-bit)
+* macOS 12.0 (Monterey) or higher (64-bit) *(no GPU support)*
+* Windows Native - Windows 7 or higher (64-bit) *(no GPU support after TF 2.10)*
+* Windows WSL2 - Windows 10 19044 or higher (64-bit)
Note: GPU support is available for Ubuntu and Windows with CUDA®-enabled cards.
## Software requirements
-* Python 3.7–3.10
-* pip version 19.0 or higher for Linux (requires `manylinux2010` support) and
- Windows, version 20.3 or higher for macOS
-* Windows Requires
- [Microsoft Visual C++ Redistributable for Visual Studio 2015, 2017 and 2019](https://support.microsoft.com/help/2977003/the-latest-supported-visual-c-downloads){:.external}
+* Python 3.9–3.12
+* pip version 19.0 or higher for Linux (requires `manylinux2014` support) and
+ Windows. pip version 20.3 or higher for macOS.
+* Windows Native Requires
+ [Microsoft Visual C++ Redistributable for Visual Studio 2015, 2017 and 2019](https://learn.microsoft.com/en-us/cpp/windows/latest-supported-vc-redist)
+
The following NVIDIA® software are only required for GPU support.
-* [NVIDIA® GPU drivers](https://www.nvidia.com/drivers){:.external}
- version 450.80.02 or higher.
-* [CUDA® Toolkit 11.2](https://developer.nvidia.com/cuda-toolkit-archive){:.external}.
-* [cuDNN SDK 8.1.0](https://developer.nvidia.com/cudnn){:.external}.
+* [NVIDIA® GPU drivers](https://www.nvidia.com/drivers)
+ * >= 525.60.13 for Linux
+ * >= 528.33 for WSL on Windows
+* [CUDA® Toolkit 12.3](https://developer.nvidia.com/cuda-toolkit-archive).
+* [cuDNN SDK 8.9.7](https://developer.nvidia.com/cudnn).
* *(Optional)*
- [TensorRT](https://docs.nvidia.com/deeplearning/tensorrt/archives/index.html#trt_7){:.external}
+ [TensorRT](https://docs.nvidia.com/deeplearning/tensorrt/archives/index.html#trt_7)
to improve latency and throughput for inference.
## Step-by-step instructions
-
* {Linux}
- We only officially support Ubuntu. However, the following instructions may
- also work for other Linux distros.
+ ### 1. System requirements
- We recommend using
- [Miniconda](https://docs.conda.io/en/latest/miniconda.html){:.external} to
- create a separate environment to avoid changing any installed software in
- your system. This is also the easiest way to install the required software,
- especially for the GPU setup.
+ * Ubuntu 16.04 or higher (64-bit)
- ### 1. Install Miniconda
+ TensorFlow only officially supports Ubuntu. However, the following
+ instructions may also work for other Linux distros.
- You can use the following command to install Miniconda. During installation,
- you may need to press enter and type "yes".
+ Note: Starting with TensorFlow `2.10`, Linux CPU-builds for Aarch64/ARM64
+ processors are built, maintained, tested and released by a third party:
+ [AWS](https://aws.amazon.com/).
+ Installing the [`tensorflow`](https://pypi.org/project/tensorflow/)
+ package on an ARM machine installs AWS's
+ [`tensorflow-cpu-aws`](https://pypi.org/project/tensorflow-cpu-aws/) package.
+ They are provided as-is. Tensorflow will use reasonable efforts to maintain
+ the availability and integrity of this pip package. There may be delays if
+ the third party fails to release the pip package. See
+ [this blog post](https://blog.tensorflow.org/2022/09/announcing-tensorflow-official-build-collaborators.html)
+ for more information about this collaboration.
- ```bash
- curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -o Miniconda3-latest-Linux-x86_64.sh
- bash Miniconda3-latest-Linux-x86_64.sh
- ```
-
- You may need to restart your terminal or `source ~/.bashrc` to enable the
- `conda` command. Use `conda -V` to test if it is installed successfully.
+ ### 2. GPU setup
- ### 2. Create a conda environment
-
- Create a new conda environment named `tf` with the following command.
+ You can skip this section if you only run TensorFlow on the CPU.
- ```bash
- conda create --name tf python=3.9
- ```
-
- You can deactivate and activate it with the following commands.
-
- ```bash
- conda deactivate
- conda activate tf
- ```
-
- Please make sure it is activated for the rest of the installation.
-
- ### 3. GPU setup
-
- You can skip this section if you only run TensorFlow on CPU.
-
- First, we need to install
- [NVIDIA GPU driver](https://www.nvidia.com/Download/index.aspx){:.external}
+ Install the
+ [NVIDIA GPU driver](https://www.nvidia.com/Download/index.aspx)
if you have not. You can use the following command to verify it is
installed.
@@ -160,28 +194,24 @@ The following NVIDIA® software are only required for GPU support.
nvidia-smi
```
- Then, we install the CUDA, cuDNN with conda.
+ ### 3. Create a virtual environment with [venv](https://docs.python.org/3/library/venv.html){:.external}
+
+ The venv module is part of Python’s standard library and is the officially recommended way to create virtual environments.
+
+ Navigate to your desired virtual environments directory and create a new venv environment named `tf` with the following command.
```bash
- conda install -c conda-forge cudatoolkit=11.2 cudnn=8.1.0
+ python3 -m venv tf
```
- Configure the system paths. You can do it with following command everytime
- your start a new terminal after activating your conda environment.
+ You can activate it with the following command.
```bash
- export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$CONDA_PREFIX/lib/
- ```
-
- However, for your convenience, we recommend automating it with the following
- commands. The system paths will be automatically configured when you
- activate this conda environment.
-
- ```bash
- mkdir -p $CONDA_PREFIX/etc/conda/activate.d
- echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$CONDA_PREFIX/lib/' > $CONDA_PREFIX/etc/conda/activate.d/env_vars.sh
+ source tf/bin/activate
```
+ Make sure that the virtual environment is activated for the rest of the installation.
+
### 4. Install TensorFlow
TensorFlow requires a recent version of pip, so upgrade your pip
@@ -193,15 +223,16 @@ The following NVIDIA® software are only required for GPU support.
Then, install TensorFlow with pip.
- Note: Do not install with conda. It may not have the latest stable
- version. We recommend using pip since TensorFlow is only
- officially released to PyPI.
-
```bash
+ # For GPU users
+ pip install tensorflow[and-cuda]
+ # For CPU users
pip install tensorflow
```
- ### 5. Verify install
+ **Note:** Do not install TensorFlow with `conda`. It may not have the latest stable version. `pip` is recommended since TensorFlow is only officially released to PyPI.
+
+ ### 6. Verify the installation
Verify the CPU setup:
@@ -218,71 +249,65 @@ The following NVIDIA® software are only required for GPU support.
```
If a list of GPU devices is returned, you've installed TensorFlow
- successfully.
+ successfully. **If not continue to the next step**.
-* {MacOS}
+ ### 6. [GPU only] Virtual environment configuration
- Note: For users of Apple M1 computers, to get native performance, you'll
- want to follow the instructions found
- [here](https://developer.apple.com/metal/tensorflow-plugin/){:.external}.
- Conda has shown to have the smoothest install. Some TensorFlow binaries
- (specifically, ones with custom C++ extensions like TensorFlow Decision
- Forests, object detection, and TFX) are not compiled against M1 targets. If
- you need those libraries, you will have to use TensorFlow with x86 emulation
- and Rosetta.
+ If the GPU test in the last section was unsuccessful, the most likely cause is that components aren't being detected,
+ and/or conflict with the existing system CUDA installation. So you need to add some symbolic links to fix this.
- Currently, we do not have official GPU support for running TensorFlow on
- MacOS. The following is instructions are for running on CPU.
-
- ### 1. Check Python version
+ * Create symbolic links to NVIDIA shared libraries:
+
+ ```bash
+ pushd $(dirname $(python -c 'print(__import__("tensorflow").__file__)'))
+ ln -svf ../nvidia/*/lib/*.so* .
+ popd
+ ```
+
+ * Create a symbolic link to ptxas:
- Check if your Python environment is already configured:
+ ```bash
+ ln -sf $(find $(dirname $(dirname $(python -c "import nvidia.cuda_nvcc;
+ print(nvidia.cuda_nvcc.__file__)"))/*/bin/) -name ptxas -print -quit) $VIRTUAL_ENV/bin/ptxas
+ ```
- Note: Requires Python 3.7–3.10, and pip >= 20.3 for MacOS.
+ Verify the GPU setup:
```bash
- python3 --version
- python3 -m pip --version
+ python3 -c "import tensorflow as tf; print(tf.config.list_physical_devices('GPU'))"
```
- If you have the correct version of Python and pip, you may skip the next two
- steps and go to "4. Install TensorFlow". However, we still recommend not
- skipping the steps. Use
- [Miniconda](https://docs.conda.io/en/latest/miniconda.html){:.external} to
- install Python and pip. It create a separate environment to avoid
- changing any installed software in your system.
- ### 2. Install Miniconda
- You can use the following command to install Miniconda. During installation,
- you may need to press enter and type "yes".
- ```bash
- curl https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh -o Miniconda3-latest-MacOSX-x86_64.sh
- bash Miniconda3-latest-MacOSX-x86_64.sh
- ```
+* {MacOS}
- You may need to restart your terminal or `source ~/.bashrc` to enable the
- `conda` command. Use `conda -V` to test if it is installed successfully.
+ ### 1. System requirements
- ### 3. Create a conda environment
+ * macOS 10.12.6 (Sierra) or higher (64-bit)
- Create a new conda environment named `tf` with the following command.
+ Note: While TensorFlow supports Apple Silicon (M1), packages that include
+ custom C++ extensions for TensorFlow also need to be compiled for Apple M1.
+ Some packages, like
+ [tensorflow_decision_forests](https://www.tensorflow.org/decision_forests)
+ publish M1-compatible versions, but many packages don't. To use those
+ libraries, you will have to use TensorFlow with x86 emulation and Rosetta.
- ```bash
- conda create --name tf python=3.9
- ```
+ Currently there is no official GPU support for running TensorFlow on
+ MacOS. The following instructions are for running on CPU.
- You can deactivate and activate it with the following commands.
+ ### 2. Check Python version
+
+ Check if your Python environment is already configured:
+
+ Note: Requires Python 3.9–3.11, and pip >= 20.3 for MacOS.
```bash
- conda deactivate
- conda activate tf
+ python3 --version
+ python3 -m pip --version
```
- Please make sure it is activated for the rest of the installation.
-
- ### 4. Install TensorFlow
+ ### 3. Install TensorFlow
TensorFlow requires a recent version of pip, so upgrade your pip
installation to be sure you're running the latest version.
@@ -293,15 +318,11 @@ The following NVIDIA® software are only required for GPU support.
Then, install TensorFlow with pip.
- Note: Do not install with conda. It may not have the latest stable
- version. We recommend using pip since TensorFlow is only
- officially released to PyPI.
-
```bash
pip install tensorflow
```
- ### 5. Verify install
+ ### 4. Verify the installation
```bash
python3 -c "import tensorflow as tf; print(tf.reduce_sum(tf.random.normal([1000, 1000])))"
@@ -309,17 +330,34 @@ The following NVIDIA® software are only required for GPU support.
If a tensor is returned, you've installed TensorFlow successfully.
-* {Windows}
+* {Windows Native}
- Note: Experimental support for WSL2 on Windows 10 19044 or higher with GPU
- access is now available. This corresponds to Windows 10 version
- 21H2, the November 2021 update. You can get the latest update from here:
- [Download Windows 10](https://www.microsoft.com/en-us/software-download/windows10){:.external}.
- For instructions, please see
- [NVIDIA’s setup docs](https://docs.nvidia.com/cuda/wsl-user-guide/index.html){:.external}
- for CUDA in WSL.
+ Caution: TensorFlow `2.10` was the **last** TensorFlow release that
+ supported GPU on native-Windows.
+ Starting with TensorFlow `2.11`, you will need to install
+ [TensorFlow in WSL2](https://tensorflow.org/install/pip#windows-[wsl2]),
+ or install `tensorflow-cpu` and, optionally, try the
+ [TensorFlow-DirectML-Plugin](https://github.com/microsoft/tensorflow-directml-plugin#tensorflow-directml-plugin-)
+
+ ## 1. System requirements
+
+ * Windows 7 or higher (64-bit)
- ### 1. Install Microsoft Visual C++ Redistributable
+ Note: Starting with TensorFlow `2.10`, Windows CPU-builds for x86/x64
+ processors are built, maintained, tested and released by a third party:
+ [Intel](https://www.intel.com/).
+ Installing the windows-native [`tensorflow`](https://pypi.org/project/tensorflow/)
+ or [`tensorflow-cpu`](https://pypi.org/project/tensorflow-cpu/)
+ package installs Intel's
+ [`tensorflow-intel`](https://pypi.org/project/tensorflow-intel/)
+ package. These packages are provided as-is. Tensorflow will use reasonable
+ efforts to maintain the availability and integrity of this pip package.
+ There may be delays if the third party fails to release the pip package. See
+ [this blog post](https://blog.tensorflow.org/2022/09/announcing-tensorflow-official-build-collaborators.html)
+ for more information about this
+ collaboration.
+
+ ### 2. Install Microsoft Visual C++ Redistributable
Install the *Microsoft Visual C++ Redistributable for Visual Studio 2015,
2017, and 2019*. Starting with the TensorFlow 2.1.0 version, the
@@ -328,28 +366,28 @@ The following NVIDIA® software are only required for GPU support.
with *Visual Studio 2019* but can be installed separately:
1. Go to the
- [Microsoft Visual C++ downloads](https://support.microsoft.com/help/2977003/the-latest-supported-visual-c-downloads){:.external}.
+ [Microsoft Visual C++ downloads](https://support.microsoft.com/help/2977003/the-latest-supported-visual-c-downloads).
2. Scroll down the page to the *Visual Studio 2015, 2017 and 2019* section.
3. Download and install the *Microsoft Visual C++ Redistributable for
Visual Studio 2015, 2017 and 2019* for your platform.
Make sure
- [long paths are enabled](https://superuser.com/questions/1119883/windows-10-enable-ntfs-long-paths-policy-option-missing){:.external}
+ [long paths are enabled](https://superuser.com/questions/1119883/windows-10-enable-ntfs-long-paths-policy-option-missing)
on Windows.
- ### 2. Install Miniconda
+ ### 3. Install Miniconda
- We recommend using
- [Miniconda](https://docs.conda.io/en/latest/miniconda.html){:.external} to
- create a separate environment to avoid changing any installed software in
- your system. This is also the easiest way to install the required software,
- especially for the GPU setup.
+ [Miniconda](https://docs.conda.io/en/latest/miniconda.html)
+ is the recommended approach for installing TensorFlow with GPU support.
+ It creates a separate environment to avoid changing any installed
+ software in your system. This is also the easiest way to install the
+ required software especially for the GPU setup.
Download the
- [Miniconda Windows Installer](https://repo.anaconda.com/miniconda/Miniconda3-latest-Windows-x86_64.exe){:.external}.
+ [Miniconda Windows Installer](https://repo.anaconda.com/miniconda/Miniconda3-latest-Windows-x86_64.exe).
Double-click the downloaded file and follow the instructions on the screen.
- ### 3. Create a conda environment
+ ### 4. Create a conda environment
Create a new conda environment named `tf` with the following command.
@@ -364,23 +402,23 @@ The following NVIDIA® software are only required for GPU support.
conda activate tf
```
- Please make sure it is activated for the rest of the installation.
+ Make sure it is activated for the rest of the installation.
- ### 4. GPU setup
+ ### 5. GPU setup
You can skip this section if you only run TensorFlow on CPU.
- First, we need to install
- [NVIDIA GPU driver](https://www.nvidia.com/Download/index.aspx){:.external}
+ First install
+ [NVIDIA GPU driver](https://www.nvidia.com/Download/index.aspx)
if you have not.
- Then, we install the CUDA, cuDNN with conda.
+ Then install the CUDA, cuDNN with conda.
```bash
conda install -c conda-forge cudatoolkit=11.2 cudnn=8.1.0
```
- ### 5. Install TensorFlow
+ ### 6. Install TensorFlow
TensorFlow requires a recent version of pip, so upgrade your pip
installation to be sure you're running the latest version.
@@ -391,15 +429,79 @@ The following NVIDIA® software are only required for GPU support.
Then, install TensorFlow with pip.
- Note: Do not install with conda. It may not have the latest stable
- version. We recommend using pip since TensorFlow is only
- officially released to PyPI.
+ Note: Do not install TensorFlow with conda. It may not have the latest stable
+ version. pip is recommended since TensorFlow is only officially released to
+ PyPI.
+
+ ```bash
+ # Anything above 2.10 is not supported on the GPU on Windows Native
+ pip install "tensorflow<2.11"
+ ```
+
+ ### 7. Verify the installation
+
+ Verify the CPU setup:
+
+ ```bash
+ python -c "import tensorflow as tf; print(tf.reduce_sum(tf.random.normal([1000, 1000])))"
+ ```
+
+ If a tensor is returned, you've installed TensorFlow successfully.
+
+ Verify the GPU setup:
```bash
+ python -c "import tensorflow as tf; print(tf.config.list_physical_devices('GPU'))"
+ ```
+
+ If a list of GPU devices is returned, you've installed TensorFlow
+ successfully.
+
+* {Windows WSL2}
+
+ ### 1. System requirements
+
+ * Windows 10 19044 or higher (64-bit). This corresponds to Windows 10
+ version 21H2, the November 2021 update.
+
+ See the following documents to:
+
+ * [Download the latest Windows 10 update](https://www.microsoft.com/software-download/windows10).
+ * [Install WSL2](https://docs.microsoft.com/windows/wsl/install)
+ * [Setup NVIDIA® GPU support in WSL2](https://docs.nvidia.com/cuda/wsl-user-guide/index.html)
+
+ ### 2. GPU setup
+
+ You can skip this section if you only run TensorFlow on the CPU.
+
+ Install the
+ [NVIDIA GPU driver](https://www.nvidia.com/Download/index.aspx)
+ if you have not. You can use the following command to verify it is
+ installed.
+
+ ```bash
+ nvidia-smi
+ ```
+
+ ### 3. Install TensorFlow
+
+ TensorFlow requires a recent version of pip, so upgrade your pip
+ installation to be sure you're running the latest version.
+
+ ```bash
+ pip install --upgrade pip
+ ```
+
+ Then, install TensorFlow with pip.
+
+ ```bash
+ # For GPU users
+ pip install tensorflow[and-cuda]
+ # For CPU users
pip install tensorflow
```
- ### 6. Verify install
+ ### 4. Verify the installation
Verify the CPU setup:
@@ -418,6 +520,7 @@ The following NVIDIA® software are only required for GPU support.
If a list of GPU devices is returned, you've installed TensorFlow
successfully.
+
## Package location
A few installation mechanisms require the URL of the TensorFlow Python package.
@@ -425,90 +528,131 @@ The value you specify depends on your Python version.
diff --git a/site/en/install/source.md b/site/en/install/source.md
index 0ba98b99ac5..dc847f017e9 100644
--- a/site/en/install/source.md
+++ b/site/en/install/source.md
@@ -25,9 +25,6 @@ Install the following build tools to configure your development environment.
-/usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)"
-export PATH="/usr/local/opt/python/libexec/bin:$PATH"
-# if you are on macOS 10.12 (Sierra) use `export PATH="/usr/local/bin:/usr/local/sbin:$PATH"`brew install python
@@ -37,13 +34,12 @@ Install the TensorFlow *pip* package dependencies (if using a virtual
environment, omit the `--user` argument):
Note: A `pip` version >19.0 is required to install the TensorFlow 2 `.whl`
package. Additional required dependencies are listed in the
-setup.py
+setup.py.tpl
file under `REQUIRED_PACKAGES`.
### Install Bazel
@@ -55,31 +51,82 @@ ease of use, add Bazelisk as the `bazel` executable in your `PATH`.
If Bazelisk is not available, you can manually
[install Bazel](https://bazel.build/install). Make
-sure to install a supported Bazel version: any version between
-`_TF_MIN_BAZEL_VERSION` and `_TF_MAX_BAZEL_VERSION` as specified in
-`tensorflow/configure.py`.
+sure to install the correct Bazel version from TensorFlow's
+[.bazelversion](https://github.com/tensorflow/tensorflow/blob/master/.bazelversion)
+file.
+
+### Install Clang (recommended, Linux only)
+
+Clang is a C/C++/Objective-C compiler that is compiled in C++ based on LLVM. It
+is the default compiler to build TensorFlow starting with TensorFlow 2.13. The
+current supported version is LLVM/Clang 17.
+
+[LLVM Debian/Ubuntu nightly packages](https://apt.llvm.org) provide an automatic
+installation script and packages for manual installation on Linux. Make sure you
+run the following command if you manually add llvm apt repository to your
+package sources:
+
+
+
+Now that `/usr/lib/llvm-17/bin/clang` is the actual path to clang in this case.
+
+Alternatively, you can download and unpack the pre-built
+[Clang + LLVM 17](https://github.com/llvm/llvm-project/releases/tag/llvmorg-17.0.2).
+
+Below is an example of steps you can take to set up the downloaded Clang + LLVM
+17 binaries on Debian/Ubuntu operating systems:
+
+1. Change to the desired destination directory: `cd `
+
+1. Load and extract an archive file...(suitable to your architecture):
+
+ wget https://github.com/llvm/llvm-project/releases/download/llvmorg-17.0.2/clang+llvm-17.0.2-x86_64-linux-gnu-ubuntu-22.04.tar.xz
+
+ tar -xvf clang+llvm-17.0.2-x86_64-linux-gnu-ubuntu-22.04.tar.xz
+
+
+
+1. Copy the extracted contents (directories and files) to `/usr` (you may need
+ sudo permissions, and the correct directory may vary by distribution). This
+ effectively installs Clang and LLVM, and adds it to the path. You should not
+ have to replace anything, unless you have a previous installation, in which
+ case you should replace the files:
+
+
+1. Now that `/usr/bin/clang` is the actual path to your new clang. You can run
+ the `./configure` script or manually set environment variables `CC` and
+ `BAZEL_COMPILER` to this path.
### Install GPU support (optional, Linux only)
There is *no* GPU support for macOS.
-Read the [GPU support](./gpu.md) guide to install the drivers and additional
+Read the [GPU support](./pip.md) guide to install the drivers and additional
software required to run TensorFlow on a GPU.
Note: It is easier to set up one of TensorFlow's GPU-enabled [Docker images](#docker_linux_builds).
### Download the TensorFlow source code
-Use [Git](https://git-scm.com/){:.external} to clone the
-[TensorFlow repository](https://github.com/tensorflow/tensorflow){:.external}:
+Use [Git](https://git-scm.com/) to clone the
+[TensorFlow repository](https://github.com/tensorflow/tensorflow):
-The repo defaults to the `master` development branch. You can also checkout a
-[release branch](https://github.com/tensorflow/tensorflow/releases){:.external}
+The repo defaults to the `master` development branch. You can also check out a
+[release branch](https://github.com/tensorflow/tensorflow/releases)
to build:
@@ -89,16 +136,21 @@ git checkout branch_name # r2.2, r2.3, etc.
## Configure the build
-Configure your system build by running the `./configure` at the root of your
-TensorFlow source tree. This script prompts you for the location of TensorFlow
-dependencies and asks for additional build configuration options (compiler
-flags, for example).
+TensorFlow builds are configured by the `.bazelrc` file in the repository's
+root directory. The `./configure` or `./configure.py` scripts can be used to
+adjust common settings.
+
+Please run the `./configure` script from the repository's root directory. This
+script will prompt you for the location of TensorFlow dependencies and asks for
+additional build configuration options (compiler flags, for example). Refer to
+the _Sample session_ section for details.
./configure
-If using a virtual environment, `python configure.py` prioritizes paths
+There is also a python version of this script, `./configure.py`. If using a
+virtual environment, `python configure.py` prioritizes paths
within the environment, whereas `./configure` prioritizes paths outside
the environment. In both cases you can change the default.
@@ -111,65 +163,47 @@ session may differ):
View sample configuration session
./configure
-You have bazel 3.0.0 installed.
-Please specify the location of python. [Default is /usr/bin/python3]:
+You have bazel 6.1.0 installed.
+Please specify the location of python. [Default is /Library/Frameworks/Python.framework/Versions/3.9/bin/python3]:
Found possible Python library paths:
- /usr/lib/python3/dist-packages
- /usr/local/lib/python3.6/dist-packages
-Please input the desired Python library path to use. Default is [/usr/lib/python3/dist-packages]
-
-Do you wish to build TensorFlow with OpenCL SYCL support? [y/N]:
-No OpenCL SYCL support will be enabled for TensorFlow.
+ /Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/site-packages
+Please input the desired Python library path to use. Default is [/Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/site-packages]
-Do you wish to build TensorFlow with ROCm support? [y/N]:
+Do you wish to build TensorFlow with ROCm support? [y/N]:
No ROCm support will be enabled for TensorFlow.
-Do you wish to build TensorFlow with CUDA support? [y/N]: Y
-CUDA support will be enabled for TensorFlow.
-
-Do you wish to build TensorFlow with TensorRT support? [y/N]:
-No TensorRT support will be enabled for TensorFlow.
-
-Found CUDA 10.1 in:
- /usr/local/cuda-10.1/targets/x86_64-linux/lib
- /usr/local/cuda-10.1/targets/x86_64-linux/include
-Found cuDNN 7 in:
- /usr/lib/x86_64-linux-gnu
- /usr/include
-
-
-Please specify a list of comma-separated CUDA compute capabilities you want to build with.
-You can find the compute capability of your device at: https://developer.nvidia.com/cuda-gpus. Each capability can be specified as "x.y" or "compute_xy" to include both virtual and binary GPU code, or as "sm_xy" to only include the binary code.
-Please note that each additional compute capability significantly increases your build time and binary size, and that TensorFlow only supports compute capabilities >= 3.5 [Default is: 3.5,7.0]: 6.1
-
+Do you wish to build TensorFlow with CUDA support? [y/N]:
+No CUDA support will be enabled for TensorFlow.
-Do you want to use clang as CUDA compiler? [y/N]:
-nvcc will be used as CUDA compiler.
+Do you want to use Clang to build TensorFlow? [Y/n]:
+Clang will be used to compile TensorFlow.
-Please specify which gcc should be used by nvcc as the host compiler. [Default is /usr/bin/gcc]:
+Please specify the path to clang executable. [Default is /usr/lib/llvm-16/bin/clang]:
+You have Clang 16.0.4 installed.
-Please specify optimization flags to use during compilation when bazel option "--config=opt" is specified [Default is -march=native -Wno-sign-compare]:
+Please specify optimization flags to use during compilation when bazel option "--config=opt" is specified [Default is -Wno-sign-compare]:
-Would you like to interactively configure ./WORKSPACE for Android builds? [y/N]:
+Would you like to interactively configure ./WORKSPACE for Android builds? [y/N]: n
Not configuring the WORKSPACE for Android builds.
+Do you wish to build TensorFlow with iOS support? [y/N]: n
+No iOS support will be enabled for TensorFlow.
+
Preconfigured Bazel build configs. You can use any of the below by adding "--config=<>" to your build command. See .bazelrc for more details.
--config=mkl # Build with MKL support.
+ --config=mkl_aarch64 # Build with oneDNN and Compute Library for the Arm Architecture (ACL).
--config=monolithic # Config for mostly static monolithic build.
- --config=ngraph # Build with Intel nGraph support.
--config=numa # Build with NUMA support.
--config=dynamic_kernels # (Experimental) Build kernels into separate shared objects.
- --config=v2 # Build TensorFlow 2.x instead of 1.x.
+ --config=v1 # Build with TensorFlow 1 API instead of TF 2 API.
Preconfigured Bazel build configs to DISABLE default on features:
- --config=noaws # Disable AWS S3 filesystem support.
--config=nogcp # Disable GCP support.
- --config=nohdfs # Disable HDFS support.
--config=nonccl # Disable NVIDIA NCCL support.
-Configuration finished
+
@@ -177,7 +211,14 @@ Configuration finished
#### GPU support
-For [GPU support](./gpu.md), set `cuda=Y` during configuration and specify the
+##### from v.2.18.0
+For [GPU support](./pip.md), set `cuda=Y` during configuration and specify the
+versions of CUDA and cuDNN if required. Bazel will download CUDA and CUDNN
+packages automatically or point to CUDA/CUDNN/NCCL redistributions on local file
+system if required.
+
+##### before v.2.18.0
+For [GPU support](./pip.md), set `cuda=Y` during configuration and specify the
versions of CUDA and cuDNN. If your system has multiple versions of CUDA or
cuDNN installed, explicitly set the version instead of relying on the default.
`./configure` creates symbolic links to your system's CUDA libraries—so if you
@@ -189,7 +230,7 @@ building.
For compilation optimization flags, the default (`-march=native`) optimizes the
generated code for your machine's CPU type. However, if building TensorFlow for
a different CPU type, consider a more specific optimization flag. Check the
-[GCC manual](https://gcc.gnu.org/onlinedocs/gcc-4.5.3/gcc/i386-and-x86_002d64-Options.html){:.external}
+[GCC manual](https://gcc.gnu.org/onlinedocs/gcc-4.5.3/gcc/i386-and-x86_002d64-Options.html)
for examples.
#### Preconfigured configurations
@@ -201,44 +242,13 @@ There are some preconfigured build configs available that can be added to the
[CONTRIBUTING.md](https://github.com/tensorflow/tensorflow/blob/master/CONTRIBUTING.md)
for details.
* `--config=mkl` —Support for the
- [Intel® MKL-DNN](https://github.com/intel/mkl-dnn){:.external}.
+ [Intel® MKL-DNN](https://github.com/intel/mkl-dnn).
* `--config=monolithic` —Configuration for a mostly static, monolithic build.
-* `--config=v1` —Build TensorFlow 1.x instead of 2.x.
-
-Note: Starting with TensorFlow 1.6, binaries use AVX instructions which may not
-run on older CPUs.
-
-
-## Build the pip package
-### TensorFlow 2.x
-[Install Bazel](https://docs.bazel.build/versions/master/install.html) and use
-`bazel build` to create the TensorFlow 2.x package with *CPU-only* support:
+## Build and install the pip package
-
-
-Note: GPU support can be enabled with `cuda=Y` during the `./configure` stage.
-
-### GPU support
-
-To build a TensorFlow package builder with GPU support:
-
-
-
-### Bazel build options
+#### Bazel build options
Refer to the Bazel
[command-line reference](https://bazel.build/reference/command-line-reference)
@@ -248,35 +258,39 @@ for
Building TensorFlow from source can use a lot of RAM. If your system is
memory-constrained, limit Bazel's RAM usage with: `--local_ram_resources=2048`.
-The [official TensorFlow packages](./pip.md) are built with a GCC 7.3
-toolchain that complies with the manylinux2010 package standard.
-
-For GCC 5 and later, compatibility with the older ABI can be built using:
-`--cxxopt="-D_GLIBCXX_USE_CXX11_ABI=0"`. ABI compatibility ensures that custom
-ops built against the official TensorFlow package continue to work with the
-GCC 5 built package.
+The [official TensorFlow packages](./pip.md) are built with a Clang toolchain
+that complies with the manylinux2014 package standard.
### Build the package
-The `bazel build` command creates an executable named `build_pip_package`—this
-is the program that builds the `pip` package. Run the executable as shown
-below to build a `.whl` package in the `/tmp/tensorflow_pkg` directory.
+To build pip package, you need to specify `--repo_env=WHEEL_NAME` flag.
+depending on the provided name, package will be created, e.g:
-To build from a release branch:
+To build tensorflow CPU package:
+
-Although it is possible to build both CUDA and non-CUDA configurations under the
-same source tree, it's recommended to run `bazel clean` when switching between
-these two configurations in the same source tree.
+As a result, generated wheel will be located in
+
### Install the package
@@ -284,7 +298,7 @@ The filename of the generated `.whl` file depends on the TensorFlow version and
your platform. Use `pip install` to install the package, for example:
Success: TensorFlow is now installed.
@@ -296,7 +310,7 @@ TensorFlow's Docker development images are an easy way to set up an environment
to build Linux packages from source. These images already contain the source
code and dependencies required to build TensorFlow. Go to the TensorFlow
[Docker guide](./docker.md) for installation instructions and the
-[list of available image tags](https://hub.docker.com/r/tensorflow/tensorflow/tags/){:.external}.
+[list of available image tags](https://hub.docker.com/r/tensorflow/tensorflow/tags/).
### CPU-only
@@ -332,20 +346,20 @@ docker run -it -w /tensorflow -v /path/to/tensorflow:/tensorflow -v $
With the source tree set up, build the TensorFlow package within the container's
virtual environment:
-1. Configure the build—this prompts the user to answer build configuration
- questions.
-2. Build the tool used to create the *pip* package.
-3. Run the tool to create the *pip* package.
-4. Adjust the ownership permissions of the file for outside the container.
+1. Optional: Configure the build—this prompts the user to answer build
+ configuration questions.
+2. Build the *pip* package.
+3. Adjust the ownership permissions of the file for outside the container.
Install and verify the package within the container:
@@ -353,7 +367,7 @@ Install and verify the package within the container:
pip uninstall tensorflow # remove current version
-pip install /mnt/tensorflow-version-tags.whl
+pip install bazel-bin/tensorflow/tools/pip_package/wheel_house/tensorflow-version-tags.whlcd /tmp # don't import from source directorypython -c "import tensorflow as tf; print(tf.__version__)"
@@ -366,12 +380,15 @@ On your host machine, the TensorFlow *pip* package is in the current directory
### GPU support
+Note: Starting from Tensorflow v.2.18.0 the wheels can be built from
+source on a machine without GPUs and without NVIDIA driver installed.
+
Docker is the easiest way to build GPU support for TensorFlow since the *host*
machine only requires the
-[NVIDIA® driver](https://github.com/NVIDIA/nvidia-docker/wiki/Frequently-Asked-Questions#how-do-i-install-the-nvidia-driver){:.external}
+[NVIDIA® driver](https://github.com/NVIDIA/nvidia-docker/wiki/Frequently-Asked-Questions#how-do-i-install-the-nvidia-driver)
(the *NVIDIA® CUDA® Toolkit* doesn't have to be installed). Refer to the
-[GPU support guide](./gpu.md) and the TensorFlow [Docker guide](./docker.md) to
-set up [nvidia-docker](https://github.com/NVIDIA/nvidia-docker){:.external}
+[GPU support guide](./pip.md) and the TensorFlow [Docker guide](./docker.md) to
+set up [nvidia-docker](https://github.com/NVIDIA/nvidia-docker)
(Linux only).
The following example downloads the TensorFlow `:devel-gpu` image and uses
@@ -389,13 +406,15 @@ Then, within the container's virtual environment, build the TensorFlow package
with GPU support:
Install and verify the package within the container and check for a GPU:
@@ -403,7 +422,7 @@ Install and verify the package within the container and check for a GPU:
pip uninstall tensorflow # remove current version
-pip install /mnt/tensorflow-version-tags.whl
+pip install bazel-bin/tensorflow/tools/pip_package/wheel_house/tensorflow-version-tags.whlcd /tmp # don't import from source directorypython -c "import tensorflow as tf; print(\"Num GPUs Available: \", len(tf.config.list_physical_devices('GPU')))"
@@ -420,6 +439,17 @@ Success: TensorFlow is now installed.
Version
Python version
Compiler
Build tools
+
tensorflow-2.20.0
3.9-3.13
Clang 18.1.8
Bazel 7.4.1
+
tensorflow-2.19.0
3.9-3.12
Clang 18.1.8
Bazel 6.5.0
+
tensorflow-2.18.0
3.9-3.12
Clang 17.0.6
Bazel 6.5.0
+
tensorflow-2.17.0
3.9-3.12
Clang 17.0.6
Bazel 6.5.0
+
tensorflow-2.16.1
3.9-3.12
Clang 17.0.6
Bazel 6.5.0
+
tensorflow-2.15.0
3.9-3.11
Clang 16.0.0
Bazel 6.1.0
+
tensorflow-2.14.0
3.9-3.11
Clang 16.0.0
Bazel 6.1.0
+
tensorflow-2.13.0
3.8-3.11
Clang 16.0.0
Bazel 5.3.0
+
tensorflow-2.12.0
3.8-3.11
GCC 9.3.1
Bazel 5.3.0
+
tensorflow-2.11.0
3.7-3.10
GCC 9.3.1
Bazel 5.3.0
+
tensorflow-2.10.0
3.7-3.10
GCC 9.3.1
Bazel 5.1.1
tensorflow-2.9.0
3.7-3.10
GCC 9.3.1
Bazel 5.0.0
tensorflow-2.8.0
3.7-3.10
GCC 7.3.1
Bazel 4.2.1
tensorflow-2.7.0
3.7-3.9
GCC 7.3.1
Bazel 3.7.2
@@ -452,6 +482,17 @@ Success: TensorFlow is now installed.
Version
Python version
Compiler
Build tools
cuDNN
CUDA
+
tensorflow-2.20.0
3.9-3.13
Clang 18.1.8
Bazel 7.4.1
9.3
12.5
+
tensorflow-2.19.0
3.9-3.12
Clang 18.1.8
Bazel 6.5.0
9.3
12.5
+
tensorflow-2.18.0
3.9-3.12
Clang 17.0.6
Bazel 6.5.0
9.3
12.5
+
tensorflow-2.17.0
3.9-3.12
Clang 17.0.6
Bazel 6.5.0
8.9
12.3
+
tensorflow-2.16.1
3.9-3.12
Clang 17.0.6
Bazel 6.5.0
8.9
12.3
+
tensorflow-2.15.0
3.9-3.11
Clang 16.0.0
Bazel 6.1.0
8.9
12.2
+
tensorflow-2.14.0
3.9-3.11
Clang 16.0.0
Bazel 6.1.0
8.7
11.8
+
tensorflow-2.13.0
3.8-3.11
Clang 16.0.0
Bazel 5.3.0
8.6
11.8
+
tensorflow-2.12.0
3.8-3.11
GCC 9.3.1
Bazel 5.3.0
8.6
11.8
+
tensorflow-2.11.0
3.7-3.10
GCC 9.3.1
Bazel 5.3.0
8.1
11.2
+
tensorflow-2.10.0
3.7-3.10
GCC 9.3.1
Bazel 5.1.1
8.1
11.2
tensorflow-2.9.0
3.7-3.10
GCC 9.3.1
Bazel 5.0.0
8.1
11.2
tensorflow-2.8.0
3.7-3.10
GCC 7.3.1
Bazel 4.2.1
8.1
11.2
tensorflow-2.7.0
3.7-3.9
GCC 7.3.1
Bazel 3.7.2
8.1
11.2
@@ -486,6 +527,13 @@ Success: TensorFlow is now installed.
Version
Python version
Compiler
Build tools
+
tensorflow-2.16.1
3.9-3.12
Clang from Xcode 13.6
Bazel 6.5.0
+
tensorflow-2.15.0
3.9-3.11
Clang from xcode 10.15
Bazel 6.1.0
+
tensorflow-2.14.0
3.9-3.11
Clang from xcode 10.15
Bazel 6.1.0
+
tensorflow-2.13.0
3.8-3.11
Clang from xcode 10.15
Bazel 5.3.0
+
tensorflow-2.12.0
3.8-3.11
Clang from xcode 10.15
Bazel 5.3.0
+
tensorflow-2.11.0
3.7-3.10
Clang from xcode 10.14
Bazel 5.3.0
+
tensorflow-2.10.0
3.7-3.10
Clang from xcode 10.14
Bazel 5.1.1
tensorflow-2.9.0
3.7-3.10
Clang from xcode 10.14
Bazel 5.0.0
tensorflow-2.8.0
3.7-3.10
Clang from xcode 10.14
Bazel 4.2.1
tensorflow-2.7.0
3.7-3.9
Clang from xcode 10.11
Bazel 3.7.2
diff --git a/site/en/install/source_windows.md b/site/en/install/source_windows.md
index e4259229691..efc0f7a9286 100644
--- a/site/en/install/source_windows.md
+++ b/site/en/install/source_windows.md
@@ -1,6 +1,6 @@
# Build from source on Windows
-Build a TensorFlow *pip* package from source and install it on Windows.
+Build a TensorFlow *pip* package from the source and install it on Windows.
Note: We already provide well-tested, pre-built
[TensorFlow packages](./pip.md) for Windows systems.
@@ -13,13 +13,14 @@ environment.
### Install Python and the TensorFlow package dependencies
Install a
-[Python 3.7+ 64-bit release for Windows](https://www.python.org/downloads/windows/){:.external}.
+[Python 3.9+ 64-bit release for Windows](https://www.python.org/downloads/windows/).
Select *pip* as an optional feature and add it to your `%PATH%` environmental
variable.
Install the TensorFlow *pip* package dependencies:
@@ -41,38 +42,53 @@ Add the location of the Bazel executable to your `%PATH%` environment variable.
### Install MSYS2
-[Install MSYS2](https://www.msys2.org/){:.external} for the bin tools needed to
+[Install MSYS2](https://www.msys2.org/) for the bin tools needed to
build TensorFlow. If MSYS2 is installed to `C:\msys64`, add
`C:\msys64\usr\bin` to your `%PATH%` environment variable. Then, using `cmd.exe`,
run:
-### Install Visual C++ Build Tools 2019
+Note: Clang will be the preferred compiler to build TensorFlow CPU wheels on the Windows Platform starting with TF 2.16.1 The currently supported version is LLVM/clang 17.0.6.
-Install the *Visual C++ build tools 2019*. This comes with *Visual Studio 2019*
+Note: To build with Clang on Windows, it is required to install both LLVM and Visual C++ Build tools as although Windows uses clang-cl.exe as the compiler, Visual C++ Build tools are needed to link to Visual C++ libraries
+
+### Install Visual C++ Build Tools 2022
+
+Install the *Visual C++ build tools 2022*. This comes with *Visual Studio Community 2022*
but can be installed separately:
1. Go to the
- [Visual Studio downloads](https://visualstudio.microsoft.com/downloads/){:.external},
-2. Select *Redistributables and Build Tools*,
+ [Visual Studio downloads](https://visualstudio.microsoft.com/downloads/),
+2. Select *Tools for Visual Studio or Other Tools, Framework and Redistributables*,
3. Download and install:
- - *Microsoft Visual C++ 2019 Redistributable*
- - *Microsoft Build Tools 2019*
+ - *Build Tools for Visual Studio 2022*
+ - *Microsoft Visual C++ Redistributables for Visual Studio 2022*
+
+Note: TensorFlow is tested against the *Visual Studio Community 2022*.
+
+### Install LLVM
+
+1. Go to the
+ [LLVM downloads](https://github.com/llvm/llvm-project/releases/),
+2. Download and install Windows-compatible LLVM in C:/Program Files/LLVM e.g., LLVM-17.0.6-win64.exe
-Note: TensorFlow is tested against the *Visual Studio 2019*.
### Install GPU support (optional)
See the Windows [GPU support](./gpu.md) guide to install the drivers and
additional software required to run TensorFlow on a GPU.
+Note: GPU support on native-Windows is only available for 2.10 or earlier versions, starting in TF 2.11, CUDA build is not supported for Windows. For using TensorFlow GPU on Windows, you will need to build/install TensorFlow in WSL2 or use tensorflow-cpu with TensorFlow-DirectML-Plugin
+
### Download the TensorFlow source code
-Use [Git](https://git-scm.com/){:.external} to clone the
-[TensorFlow repository](https://github.com/tensorflow/tensorflow){:.external}
+Use [Git](https://git-scm.com/) to clone the
+[TensorFlow repository](https://github.com/tensorflow/tensorflow)
(`git` is installed with MSYS2):
@@ -80,8 +96,8 @@ Use [Git](https://git-scm.com/){:.external} to clone the
cd tensorflow
-The repo defaults to the `master` development branch. You can also checkout a
-[release branch](https://github.com/tensorflow/tensorflow/releases){:.external}
+The repo defaults to the `master` development branch. You can also check out a
+[release branch](https://github.com/tensorflow/tensorflow/releases)
to build:
@@ -91,11 +107,38 @@ git checkout branch_name # r1.9, r1.10, etc.
Key Point: If you're having build problems on the latest development branch, try
a release branch that is known to work.
+## Optional: Environmental Variable Set Up
+Run the following commands before running the build command to avoid issues with package creation:
+(If the below commands were set up while installing the packages, please ignore them). Run `set` to check if all the paths were set correctly, run `echo %Environmental Variable%` e.g., `echo %BAZEL_VC%` to check the path set up for a specific Environmental Variable
+
+ Python path set up issue [tensorflow:issue#59943](https://github.com/tensorflow/tensorflow/issues/59943),[tensorflow:issue#9436](https://github.com/tensorflow/tensorflow/issues/9436),[tensorflow:issue#60083](https://github.com/tensorflow/tensorflow/issues/60083)
+
+
+
+Bazel/MSVC/CLANG path set up issue [tensorflow:issue#54578](https://github.com/tensorflow/tensorflow/issues/54578)
+
+
+set BAZEL_SH=C:/msys64/usr/bin/bash.exe
+set BAZEL_VS=C:/Program Files/Microsoft Visual Studio/2022/BuildTools
+set BAZEL_VC=C:/Program Files/Microsoft Visual Studio/2022/BuildTools/VC
+set Bazel_LLVM=C:/Program Files/LLVM (explicitly tell Bazel where LLVM is installed by BAZEL_LLVM, needed while using CLANG)
+set PATH=C:/Program Files/LLVM/bin;%PATH% (Optional, needed while using CLANG as Compiler)
+
+
+## Optional: Configure the build
-## Configure the build
+TensorFlow builds are configured by the `.bazelrc` file in the repository's
+root directory. The `./configure` or `./configure.py` scripts can be used to
+adjust common settings.
-Configure your system build by running the following at the root of your
-TensorFlow source tree:
+If you need to change the configuration, run the `./configure` script from
+the repository's root directory.
python ./configure.py
-Starting local Bazel server and connecting to it...
-................
-You have bazel 0.15.0 installed.
-Please specify the location of python. [Default is C:\python36\python.exe]:
+You have bazel 6.5.0 installed.
+Please specify the location of python. [Default is C:\Python311\python.exe]:
Found possible Python library paths:
- C:\python36\lib\site-packages
-Please input the desired Python library path to use. Default is [C:\python36\lib\site-packages]
-
-Do you wish to build TensorFlow with CUDA support? [y/N]: Y
-CUDA support will be enabled for TensorFlow.
+C:\Python311\lib\site-packages
+Please input the desired Python library path to use. Default is [C:\Python311\lib\site-packages]
-Please specify the CUDA SDK version you want to use. [Leave empty to default to CUDA 9.0]:
+Do you wish to build TensorFlow with ROCm support? [y/N]:
+No ROCm support will be enabled for TensorFlow.
-Please specify the location where CUDA 9.0 toolkit is installed. Refer to README.md for more details. [Default is C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v9.0]:
+WARNING: Cannot build with CUDA support on Windows.
+Starting in TF 2.11, CUDA build is not supported for Windows. To use TensorFlow GPU on Windows, you will need to build/install TensorFlow in WSL2.
-Please specify the cuDNN version you want to use. [Leave empty to default to cuDNN 7.0]: 7.0
+Do you want to use Clang to build TensorFlow? [Y/n]:
+Add "--config=win_clang" to compile TensorFlow with CLANG.
-Please specify the location where cuDNN 7 library is installed. Refer to README.md for more details. [Default is C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v9.0]: C:\tools\cuda
+Please specify the path to clang executable. [Default is C:\Program Files\LLVM\bin\clang.EXE]:
-Please specify a list of comma-separated Cuda compute capabilities you want to build with.
-You can find the compute capability of your device at: https://developer.nvidia.com/cuda-gpus.
-Please note that each additional compute capability significantly increases your build time and binary size. [Default is: 3.5,7.0]: 3.7
+You have Clang 17.0.6 installed.
Please specify optimization flags to use during compilation when bazel option "--config=opt" is specified [Default is /arch:AVX]:
Would you like to override eigen strong inline for some C++ compilation to reduce the compilation time? [Y/n]:
Eigen strong inline overridden.
-Configuration finished
+Would you like to interactively configure ./WORKSPACE for Android builds? [y/N]:
+Not configuring the WORKSPACE for Android builds.
+
+Preconfigured Bazel build configs. You can use any of the below by adding "--config=<>" to your build command. See .bazelrc for more details.
+ --config=mkl # Build with MKL support.
+ --config=mkl_aarch64 # Build with oneDNN and Compute Library for the Arm Architecture (ACL).
+ --config=monolithic # Config for mostly static monolithic build.
+ --config=numa # Build with NUMA support.
+ --config=dynamic_kernels # (Experimental) Build kernels into separate shared objects.
+ --config=v1 # Build with TensorFlow 1 API instead of TF 2 API.
+Preconfigured Bazel build configs to DISABLE default on features:
+ --config=nogcp # Disable GCP support.
+ --config=nonccl # Disable NVIDIA NCCL support.
-### Configuration options
-
-For [GPU support](./gpu.md), specify the versions of CUDA and cuDNN. If your
-system has multiple versions of CUDA or cuDNN installed, explicitly set the
-version instead of relying on the default. `./configure.py` creates symbolic
-links to your system's CUDA libraries—so if you update your CUDA library paths,
-this configuration step must be run again before building.
-
-Note: Starting with TensorFlow 1.6, binaries use AVX instructions which may not
-run on older CPUs.
-
-Warning: TF-TRT Windows support is provided experimentally. No guarantee is made
-regarding functionality or engineering support. Use at your own risk.
+## Build and install the pip package
-## Build the pip package
+The pip package is built in two steps. A `bazel build` command creates a
+"package-builder" program. You then run the package-builder to create the
+package.
-### TensorFlow 2.x
+### Build the package-builder
tensorflow:master repo has been updated to build 2.x by default.
[Install Bazel](https://docs.bazel.build/versions/master/install.html) and use
-`bazel build ` to create the TensorFlow package.
+`bazel build ` to create the TensorFlow package-builder.
+#### CPU-only
-### TensorFlow 1.x
-
-To build the 1.x version of TensorFlow from master, use
-`bazel build --config=v1` to create a TensorFlow 1.x package.
+Use `bazel` to make the TensorFlow package builder with CPU-only support:
+##### Build with MSVC
-#### CPU-only
-
-Use `bazel` to make the TensorFlow package builder with CPU-only support:
+##### Build with CLANG
+Use --config=`win_clang` to build TenorFlow with the CLANG Compiler:
#### GPU support
+Note: GPU support on native-Windows is only available for 2.10 or earlier versions, starting in TF 2.11, CUDA build is not supported for Windows. For using TensorFlow GPU on Windows, you will need to build/install TensorFlow in WSL2 or use tensorflow-cpu with TensorFlow-DirectML-Plugin
+
To make the TensorFlow package builder with GPU support:
+Commands to clean the bazel cache to resolve errors due to invalid or outdated cached data, bazel clean with --expunge flag removes files permanently
+
+
+bazel clean
+bazel clean --expunge
+
+
#### Bazel build options
-Use this option when building to avoid issue with package creation:
+Use this option when building to avoid issues with package creation:
[tensorflow:issue#22390](https://github.com/tensorflow/tensorflow/issues/22390)
@@ -216,30 +264,37 @@ to suppress nvcc warning messages.
### Build the package
-The `bazel build` command creates an executable named `build_pip_package`—this
-is the program that builds the `pip` package. For example, the following builds
-a `.whl` package in the `C:/tmp/tensorflow_pkg` directory:
+To build a pip package, you need to specify the --repo_env=WHEEL_NAME flag.
+Depending on the provided name, the package will be created. For example:
-
-bazel-bin\tensorflow\tools\pip_package\build_pip_package C:/tmp/tensorflow_pkg
+To build tensorflow CPU package:
+
-Although it is possible to build both CUDA and non-CUDA configs under the
-same source tree, we recommend running `bazel clean` when switching between
-these two configurations in the same source tree.
### Install the package
The filename of the generated `.whl` file depends on the TensorFlow version and
-your platform. Use `pip3 install` to install the package, for example:
+your platform. Use `pip install` to install the package, for example:
-
Success: TensorFlow is now installed.
-
## Build using the MSYS shell
TensorFlow can also be built using the MSYS shell. Make the changes listed
@@ -261,12 +316,12 @@ considered a Unix absolute path since it starts with a slash.)
Add the Bazel and Python installation directories to your `$PATH` environmental
variable. If Bazel is installed to `C:\tools\bazel.exe`, and Python to
-`C:\Python36\python.exe`, set your `PATH` with:
+`C:\Python\python.exe`, set your `PATH` with:
# Use Unix-style with ':' as separator
export PATH="/c/tools:$PATH"
-export PATH="/c/Python36:$PATH"
+export PATH="/c/path/to/Python:$PATH"
For GPU support, add the CUDA and cuDNN bin directories to your `$PATH`:
@@ -277,6 +332,8 @@ For GPU support, add the CUDA and cuDNN bin directories to your `$PATH`:
export PATH="/c/tools/cuda/bin:$PATH"
+Note: Starting in TF 2.11, CUDA build is not supported for Windows. For using TensorFlow GPU on Windows, you will need to build/install TensorFlow in WSL2 or use tensorflow-cpu with TensorFlow-DirectML-Plugin
+
## Tested build configurations
@@ -284,6 +341,16 @@ For GPU support, add the CUDA and cuDNN bin directories to your `$PATH`:
Version
Python version
Compiler
Build tools
+
tensorflow-2.20.0
3.9-3.13
CLANG 18.1.4
Bazel 7.4.1
+
tensorflow-2.19.0
3.9-3.12
CLANG 18.1.4
Bazel 6.5.0
+
tensorflow-2.18.0
3.9-3.12
CLANG 17.0.6
Bazel 6.5.0
+
tensorflow-2.17.0
3.9-3.12
CLANG 17.0.6
Bazel 6.5.0
+
tensorflow-2.16.1
3.9-3.12
CLANG 17.0.6
Bazel 6.5.0
+
tensorflow-2.15.0
3.9-3.11
MSVC 2019
Bazel 6.1.0
+
tensorflow-2.14.0
3.9-3.11
MSVC 2019
Bazel 6.1.0
+
tensorflow-2.12.0
3.8-3.11
MSVC 2019
Bazel 5.3.0
+
tensorflow-2.11.0
3.7-3.10
MSVC 2019
Bazel 5.3.0
+
tensorflow-2.10.0
3.7-3.10
MSVC 2019
Bazel 5.1.1
tensorflow-2.9.0
3.7-3.10
MSVC 2019
Bazel 5.0.0
tensorflow-2.8.0
3.7-3.10
MSVC 2019
Bazel 4.2.1
tensorflow-2.7.0
3.7-3.9
MSVC 2019
Bazel 3.7.2
@@ -313,9 +380,11 @@ For GPU support, add the CUDA and cuDNN bin directories to your `$PATH`:
### GPU
+Note: GPU support on native-Windows is only available for 2.10 or earlier versions, starting in TF 2.11, CUDA build is not supported for Windows. For using TensorFlow GPU on Windows, you will need to build/install TensorFlow in WSL2 or use tensorflow-cpu with TensorFlow-DirectML-Plugin
Version
Python version
Compiler
Build tools
cuDNN
CUDA
+
tensorflow_gpu-2.10.0
3.7-3.10
MSVC 2019
Bazel 5.1.1
8.1
11.2
tensorflow_gpu-2.9.0
3.7-3.10
MSVC 2019
Bazel 5.0.0
8.1
11.2
tensorflow_gpu-2.8.0
3.7-3.10
MSVC 2019
Bazel 4.2.1
8.1
11.2
tensorflow_gpu-2.7.0
3.7-3.9
MSVC 2019
Bazel 3.7.2
8.1
11.2
diff --git a/site/en/r1/guide/autograph.ipynb b/site/en/r1/guide/autograph.ipynb
index f028b33ce9f..64d631a52b3 100644
--- a/site/en/r1/guide/autograph.ipynb
+++ b/site/en/r1/guide/autograph.ipynb
@@ -78,7 +78,7 @@
"id": "CydFK2CL7ZHA"
},
"source": [
- "[AutoGraph](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/autograph/) helps you write complicated graph code using normal Python. Behind the scenes, AutoGraph automatically transforms your code into the equivalent [TensorFlow graph code](https://www.tensorflow.org/r1/guide/graphs). AutoGraph already supports much of the Python language, and that coverage continues to grow. For a list of supported Python language features, see the [Autograph capabilities and limitations](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/autograph/g3doc/reference/limitations.md)."
+ "[AutoGraph](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/python/autograph/) helps you write complicated graph code using normal Python. Behind the scenes, AutoGraph automatically transforms your code into the equivalent [TensorFlow graph code](https://www.tensorflow.org/r1/guide/graphs). AutoGraph already supports much of the Python language, and that coverage continues to grow. For a list of supported Python language features, see the [Autograph capabilities and limitations](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/python/autograph/g3doc/reference/limitations.md)."
]
},
{
@@ -241,7 +241,7 @@
"id": "m-jWmsCmByyw"
},
"source": [
- "AutoGraph supports common Python statements like `while`, `for`, `if`, `break`, and `return`, with support for nesting. Compare this function with the complicated graph verson displayed in the following code blocks:"
+ "AutoGraph supports common Python statements like `while`, `for`, `if`, `break`, and `return`, with support for nesting. Compare this function with the complicated graph version displayed in the following code blocks:"
]
},
{
diff --git a/site/en/r1/guide/checkpoints.md b/site/en/r1/guide/checkpoints.md
index 682631449d5..41544f52b25 100644
--- a/site/en/r1/guide/checkpoints.md
+++ b/site/en/r1/guide/checkpoints.md
@@ -56,8 +56,8 @@ Suppose you call the Estimator's `train` method. For example:
```python
classifier.train(
- input_fn=lambda:train_input_fn(train_x, train_y, batch_size=100),
- steps=200)
+ input_fn=lambda: train_input_fn(train_x, train_y, batch_size=100),
+ steps=200)
```
As suggested by the following diagrams, the first call to `train`
diff --git a/site/en/r1/guide/custom_estimators.md b/site/en/r1/guide/custom_estimators.md
index 87dce26a0dc..7bbf3573909 100644
--- a/site/en/r1/guide/custom_estimators.md
+++ b/site/en/r1/guide/custom_estimators.md
@@ -592,10 +592,10 @@ function for custom Estimators; everything else is the same.
For more details, be sure to check out:
* The
- [official TensorFlow implementation of MNIST](https://github.com/tensorflow/models/tree/master/official/r1/mnist),
+ [official TensorFlow implementation of MNIST](https://github.com/tensorflow/models/tree/r1.15/official/r1/mnist),
which uses a custom estimator.
* The TensorFlow
- [official models repository](https://github.com/tensorflow/models/tree/master/official),
+ [official models repository](https://github.com/tensorflow/models/tree/r1.15/official),
which contains more curated examples using custom estimators.
* This [TensorBoard video](https://youtu.be/eBbEDRsCmv4), which introduces
TensorBoard.
diff --git a/site/en/r1/guide/datasets.md b/site/en/r1/guide/datasets.md
index b1ed1b6e113..d7c38bf2f92 100644
--- a/site/en/r1/guide/datasets.md
+++ b/site/en/r1/guide/datasets.md
@@ -437,7 +437,7 @@ dataset = dataset.batch(32)
iterator = dataset.make_initializable_iterator()
# You can feed the initializer with the appropriate filenames for the current
-# phase of execution, e.g. training vs. validation.
+# phase of execution, e.g., training vs. validation.
# Initialize `iterator` with training data.
training_filenames = ["/var/data/file1.tfrecord", "/var/data/file2.tfrecord"]
@@ -639,7 +639,7 @@ TODO(mrry): Add this section.
The simplest form of batching stacks `n` consecutive elements of a dataset into
a single element. The `Dataset.batch()` transformation does exactly this, with
the same constraints as the `tf.stack()` operator, applied to each component
-of the elements: i.e. for each component *i*, all elements must have a tensor
+of the elements: i.e., for each component *i*, all elements must have a tensor
of the exact same shape.
```python
diff --git a/site/en/r1/guide/debugger.md b/site/en/r1/guide/debugger.md
index 2b4b6497ec4..963765b97db 100644
--- a/site/en/r1/guide/debugger.md
+++ b/site/en/r1/guide/debugger.md
@@ -10,7 +10,7 @@ due to TensorFlow's computation-graph paradigm.
This guide focuses on the command-line interface (CLI) of `tfdbg`. For guide on
how to use the graphical user interface (GUI) of tfdbg, i.e., the
**TensorBoard Debugger Plugin**, please visit
-[its README](https://github.com/tensorflow/tensorboard/blob/master/tensorboard/plugins/debugger/README.md).
+[its README](https://github.com/tensorflow/tensorboard/blob/r1.15/tensorboard/plugins/debugger/README.md).
Note: The TensorFlow debugger uses a
[curses](https://en.wikipedia.org/wiki/Curses_\(programming_library\))-based text
@@ -35,7 +35,7 @@ TensorFlow. Later sections of this document describe how to use **tfdbg** with
higher-level APIs of TensorFlow, including `tf.estimator`, `tf.keras` / `keras`
and `tf.contrib.slim`. To *observe* such an issue, run the following command
without the debugger (the source code can be found
-[here](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/debug/examples/v1/debug_mnist.py)):
+[here](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/python/debug/examples/v1/debug_mnist.py)):
python -m tensorflow.python.debug.examples.v1.debug_mnist
@@ -64,7 +64,7 @@ numeric problem first surfaced.
To add support for tfdbg in our example, all that is needed is to add the
following lines of code and wrap the Session object with a debugger wrapper.
This code is already added in
-[debug_mnist.py](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/debug/examples/v1/debug_mnist.py),
+[debug_mnist.py](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/python/debug/examples/v1/debug_mnist.py),
so you can activate tfdbg CLI with the `--debug` flag at the command line.
```python
@@ -370,7 +370,7 @@ traceback of the node's construction.
From the traceback, you can see that the op is constructed at the following
line:
-[`debug_mnist.py`](https://www.tensorflow.org/code/tensorflow/python/debug/examples/v1/debug_mnist.py):
+[`debug_mnist.py`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/python/debug/examples/v1/debug_mnist.py):
```python
diff = y_ * tf.log(y)
@@ -457,7 +457,7 @@ accuracy_score = classifier.evaluate(eval_input_fn,
predict_results = classifier.predict(predict_input_fn, hooks=hooks)
```
-[debug_tflearn_iris.py](https://www.tensorflow.org/code/tensorflow/python/debug/examples/v1/debug_tflearn_iris.py),
+[debug_tflearn_iris.py](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/python/debug/examples/v1/debug_tflearn_iris.py),
contains a full example of how to use the tfdbg with `Estimator`s. To run this
example, do:
@@ -501,7 +501,7 @@ TensorFlow backend. You just need to replace `tf.keras.backend` with
## Debugging tf-slim with TFDBG
TFDBG supports debugging of training and evaluation with
-[tf-slim](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/slim).
+[tf-slim](https://github.com/tensorflow/tensorflow/tree/r1.15/tensorflow/contrib/slim).
As detailed below, training and evaluation require slightly different debugging
workflows.
@@ -605,7 +605,7 @@ The `watch_fn` argument accepts a `Callable` that allows you to configure what
If your model code is written in C++ or other languages, you can also
modify the `debug_options` field of `RunOptions` to generate debug dumps that
can be inspected offline. See
-[the proto definition](https://www.tensorflow.org/code/tensorflow/core/protobuf/debug.proto)
+[the proto definition](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/protobuf/debug.proto)
for more details.
### Debugging Remotely-Running Estimators
@@ -648,7 +648,7 @@ python -m tensorflow.python.debug.cli.offline_analyzer \
model, check out
1. The profiling mode of tfdbg: `tfdbg> run -p`.
- 2. [tfprof](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/core/profiler)
+ 2. [tfprof](https://github.com/tensorflow/tensorflow/tree/r1.15/tensorflow/core/profiler)
and other profiling tools for TensorFlow.
**Q**: _How do I link tfdbg against my `Session` in Bazel? Why do I see an
@@ -808,4 +808,4 @@ tensor dumps.
and conditional breakpoints, and tying tensors to their
graph-construction source code, all in the browser environment.
To get started, please visit
- [its README](https://github.com/tensorflow/tensorboard/blob/master/tensorboard/plugins/debugger/README.md).
+ [its README](https://github.com/tensorflow/tensorboard/blob/r1.15/tensorboard/plugins/debugger/README.md).
diff --git a/site/en/r1/guide/distribute_strategy.ipynb b/site/en/r1/guide/distribute_strategy.ipynb
index 79d6293eba7..4dd502d331b 100644
--- a/site/en/r1/guide/distribute_strategy.ipynb
+++ b/site/en/r1/guide/distribute_strategy.ipynb
@@ -118,7 +118,7 @@
"## Types of strategies\n",
"`tf.distribute.Strategy` intends to cover a number of use cases along different axes. Some of these combinations are currently supported and others will be added in the future. Some of these axes are:\n",
"\n",
- "* Syncronous vs asynchronous training: These are two common ways of distributing training with data parallelism. In sync training, all workers train over different slices of input data in sync, and aggregating gradients at each step. In async training, all workers are independently training over the input data and updating variables asynchronously. Typically sync training is supported via all-reduce and async through parameter server architecture.\n",
+ "* Synchronous vs asynchronous training: These are two common ways of distributing training with data parallelism. In sync training, all workers train over different slices of input data in sync, and aggregating gradients at each step. In async training, all workers are independently training over the input data and updating variables asynchronously. Typically sync training is supported via all-reduce and async through parameter server architecture.\n",
"* Hardware platform: Users may want to scale their training onto multiple GPUs on one machine, or multiple machines in a network (with 0 or more GPUs each), or on Cloud TPUs.\n",
"\n",
"In order to support these use cases, we have 4 strategies available. In the next section we will talk about which of these are supported in which scenarios in TF."
@@ -245,7 +245,7 @@
"\n",
"`tf.distribute.experimental.MultiWorkerMirroredStrategy` is very similar to `MirroredStrategy`. It implements synchronous distributed training across multiple workers, each with potentially multiple GPUs. Similar to `MirroredStrategy`, it creates copies of all variables in the model on each device across all workers.\n",
"\n",
- "It uses [CollectiveOps](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/ops/collective_ops.py) as the multi-worker all-reduce communication method used to keep variables in sync. A collective op is a single op in the TensorFlow graph which can automatically choose an all-reduce algorithm in the TensorFlow runtime according to hardware, network topology and tensor sizes.\n",
+ "It uses [CollectiveOps](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/python/ops/collective_ops.py) as the multi-worker all-reduce communication method used to keep variables in sync. A collective op is a single op in the TensorFlow graph which can automatically choose an all-reduce algorithm in the TensorFlow runtime according to hardware, network topology and tensor sizes.\n",
"\n",
"It also implements additional performance optimizations. For example, it includes a static optimization that converts multiple all-reductions on small tensors into fewer all-reductions on larger tensors. In addition, we are designing it to have a plugin architecture - so that in the future, users will be able to plugin algorithms that are better tuned for their hardware. Note that collective ops also implement other collective operations such as broadcast and all-gather.\n",
"\n",
@@ -371,7 +371,7 @@
"id": "hQv1lm9UPDFy"
},
"source": [
- "So far we've talked about what are the different stategies available and how you can instantiate them. In the next few sections, we will talk about the different ways in which you can use them to distribute your training. We will show short code snippets in this guide and link off to full tutorials which you can run end to end."
+ "So far we've talked about what are the different strategies available and how you can instantiate them. In the next few sections, we will talk about the different ways in which you can use them to distribute your training. We will show short code snippets in this guide and link off to full tutorials which you can run end to end."
]
},
{
@@ -490,8 +490,8 @@
"Here is a list of tutorials and examples that illustrate the above integration end to end with Keras:\n",
"\n",
"1. [Tutorial](../tutorials/distribute/keras.ipynb) to train MNIST with `MirroredStrategy`.\n",
- "2. Official [ResNet50](https://github.com/tensorflow/models/blob/master/official/vision/image_classification/resnet_imagenet_main.py) training with ImageNet data using `MirroredStrategy`.\n",
- "3. [ResNet50](https://github.com/tensorflow/tpu/blob/master/models/experimental/resnet50_keras/resnet50.py) trained with Imagenet data on Cloud TPus with `TPUStrategy`."
+ "2. Official [ResNet50](https://github.com/tensorflow/models/blob/r1.15/official/vision/image_classification/resnet_imagenet_main.py) training with ImageNet data using `MirroredStrategy`.\n",
+ "3. [ResNet50](https://github.com/tensorflow/tpu/blob/1.15/models/experimental/resnet50_keras/resnet50.py) trained with Imagenet data on Cloud TPus with `TPUStrategy`."
]
},
{
@@ -595,9 +595,9 @@
"### Examples and Tutorials\n",
"Here are some examples that show end to end usage of various strategies with Estimator:\n",
"\n",
- "1. [End to end example](https://github.com/tensorflow/ecosystem/tree/master/distribution_strategy) for multi worker training in tensorflow/ecosystem using Kuberentes templates. This example starts with a Keras model and converts it to an Estimator using the `tf.keras.estimator.model_to_estimator` API.\n",
- "2. Official [ResNet50](https://github.com/tensorflow/models/blob/master/official/r1/resnet/imagenet_main.py) model, which can be trained using either `MirroredStrategy` or `MultiWorkerMirroredStrategy`.\n",
- "3. [ResNet50](https://github.com/tensorflow/tpu/blob/master/models/experimental/distribution_strategy/resnet_estimator.py) example with TPUStrategy."
+ "1. [End to end example](https://github.com/tensorflow/ecosystem/tree/r1.15/distribution_strategy) for multi worker training in tensorflow/ecosystem using Kuberentes templates. This example starts with a Keras model and converts it to an Estimator using the `tf.keras.estimator.model_to_estimator` API.\n",
+ "2. Official [ResNet50](https://github.com/tensorflow/models/blob/r1.15/official/r1/resnet/imagenet_main.py) model, which can be trained using either `MirroredStrategy` or `MultiWorkerMirroredStrategy`.\n",
+ "3. [ResNet50](https://github.com/tensorflow/tpu/blob/1.15/models/experimental/distribution_strategy/resnet_estimator.py) example with TPUStrategy."
]
},
{
@@ -607,7 +607,7 @@
},
"source": [
"## Using `tf.distribute.Strategy` with custom training loops\n",
- "As you've seen, using `tf.distrbute.Strategy` with high level APIs is only a couple lines of code change. With a little more effort, `tf.distrbute.Strategy` can also be used by other users who are not using these frameworks.\n",
+ "As you've seen, using `tf.distribute.Strategy` with high level APIs is only a couple lines of code change. With a little more effort, `tf.distribute.Strategy` can also be used by other users who are not using these frameworks.\n",
"\n",
"TensorFlow is used for a wide variety of use cases and some users (such as researchers) require more flexibility and control over their training loops. This makes it hard for them to use the high level frameworks such as Estimator or Keras. For instance, someone using a GAN may want to take a different number of generator or discriminator steps each round. Similarly, the high level frameworks are not very suitable for Reinforcement Learning training. So these users will usually write their own training loops.\n",
"\n",
diff --git a/site/en/r1/guide/eager.ipynb b/site/en/r1/guide/eager.ipynb
index 6a0a78c2443..f76acb4b702 100644
--- a/site/en/r1/guide/eager.ipynb
+++ b/site/en/r1/guide/eager.ipynb
@@ -95,7 +95,7 @@
"\n",
"Eager execution supports most TensorFlow operations and GPU acceleration. For a\n",
"collection of examples running in eager execution, see:\n",
- "[tensorflow/contrib/eager/python/examples](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/eager/python/examples).\n",
+ "[tensorflow/contrib/eager/python/examples](https://github.com/tensorflow/tensorflow/tree/r1.15/tensorflow/contrib/eager/python/examples).\n",
"\n",
"Note: Some models may experience increased overhead with eager execution\n",
"enabled. Performance improvements are ongoing, but please\n",
@@ -1160,7 +1160,7 @@
"### Benchmarks\n",
"\n",
"For compute-heavy models, such as\n",
- "[ResNet50](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/eager/python/examples/resnet50)\n",
+ "[ResNet50](https://github.com/tensorflow/tensorflow/tree/r1.15/tensorflow/contrib/eager/python/examples/resnet50)\n",
"training on a GPU, eager execution performance is comparable to graph execution.\n",
"But this gap grows larger for models with less computation and there is work to\n",
"be done for optimizing hot code paths for models with lots of small operations."
@@ -1225,7 +1225,7 @@
"production deployment. Use `tf.train.Checkpoint` to save and restore model\n",
"variables, this allows movement between eager and graph execution environments.\n",
"See the examples in:\n",
- "[tensorflow/contrib/eager/python/examples](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/eager/python/examples).\n"
+ "[tensorflow/contrib/eager/python/examples](https://github.com/tensorflow/tensorflow/tree/r1.15/tensorflow/contrib/eager/python/examples).\n"
]
},
{
diff --git a/site/en/r1/guide/extend/architecture.md b/site/en/r1/guide/extend/architecture.md
index 1f2ac53066f..0753824e15e 100644
--- a/site/en/r1/guide/extend/architecture.md
+++ b/site/en/r1/guide/extend/architecture.md
@@ -34,7 +34,7 @@ This document focuses on the following layers:
* **Client**:
* Defines the computation as a dataflow graph.
* Initiates graph execution using a [**session**](
- https://www.tensorflow.org/code/tensorflow/python/client/session.py).
+ https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/python/client/session.py).
* **Distributed Master**
* Prunes a specific subgraph from the graph, as defined by the arguments
to Session.run().
@@ -144,8 +144,8 @@ The distributed master then ships the graph pieces to the distributed tasks.
### Code
-* [MasterService API definition](https://www.tensorflow.org/code/tensorflow/core/protobuf/master_service.proto)
-* [Master interface](https://www.tensorflow.org/code/tensorflow/core/distributed_runtime/master_interface.h)
+* [MasterService API definition](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/protobuf/master_service.proto)
+* [Master interface](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/distributed_runtime/master_interface.h)
## Worker Service
@@ -178,7 +178,7 @@ For transfers between tasks, TensorFlow uses multiple protocols, including:
We also have preliminary support for NVIDIA's NCCL library for multi-GPU
communication, see:
-[`tf.contrib.nccl`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/ops/nccl_ops.py).
+[`tf.contrib.nccl`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/python/ops/nccl_ops.py).
@@ -186,9 +186,9 @@ communication, see:
### Code
-* [WorkerService API definition](https://www.tensorflow.org/code/tensorflow/core/protobuf/worker_service.proto)
-* [Worker interface](https://www.tensorflow.org/code/tensorflow/core/distributed_runtime/worker_interface.h)
-* [Remote rendezvous (for Send and Recv implementations)](https://www.tensorflow.org/code/tensorflow/core/distributed_runtime/rpc/rpc_rendezvous_mgr.h)
+* [WorkerService API definition](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/protobuf/worker_service.proto)
+* [Worker interface](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/distributed_runtime/worker_interface.h)
+* [Remote rendezvous (for Send and Recv implementations)](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/distributed_runtime/rpc/rpc_rendezvous_mgr.h)
## Kernel Implementations
@@ -199,7 +199,7 @@ Many of the operation kernels are implemented using Eigen::Tensor, which uses
C++ templates to generate efficient parallel code for multicore CPUs and GPUs;
however, we liberally use libraries like cuDNN where a more efficient kernel
implementation is possible. We have also implemented
-[quantization](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/g3doc/performance/post_training_quantization.md), which enables
+[quantization](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/lite/g3doc/performance/post_training_quantization.md), which enables
faster inference in environments such as mobile devices and high-throughput
datacenter applications, and use the
[gemmlowp](https://github.com/google/gemmlowp) low-precision matrix library to
@@ -215,4 +215,4 @@ experimental implementation of automatic kernel fusion.
### Code
-* [`OpKernel` interface](https://www.tensorflow.org/code/tensorflow/core/framework/op_kernel.h)
+* [`OpKernel` interface](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/framework/op_kernel.h)
diff --git a/site/en/r1/guide/extend/bindings.md b/site/en/r1/guide/extend/bindings.md
index 9c10e90840f..7daa2212106 100644
--- a/site/en/r1/guide/extend/bindings.md
+++ b/site/en/r1/guide/extend/bindings.md
@@ -112,11 +112,11 @@ There are a few ways to get a list of the `OpDef`s for the registered ops:
to interpret the `OpDef` messages.
- The C++ function `OpRegistry::Global()->GetRegisteredOps()` returns the same
list of all registered `OpDef`s (defined in
- [`tensorflow/core/framework/op.h`](https://www.tensorflow.org/code/tensorflow/core/framework/op.h)). This can be used to write the generator
+ [`tensorflow/core/framework/op.h`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/framework/op.h)). This can be used to write the generator
in C++ (particularly useful for languages that do not have protocol buffer
support).
- The ASCII-serialized version of that list is periodically checked in to
- [`tensorflow/core/ops/ops.pbtxt`](https://www.tensorflow.org/code/tensorflow/core/ops/ops.pbtxt) by an automated process.
+ [`tensorflow/core/ops/ops.pbtxt`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/ops/ops.pbtxt) by an automated process.
The `OpDef` specifies the following:
@@ -159,7 +159,7 @@ between the generated code and the `OpDef`s checked into the repository, but is
useful for languages where code is expected to be generated ahead of time like
`go get` for Go and `cargo ops` for Rust. At the other end of the spectrum, for
some languages the code could be generated dynamically from
-[`tensorflow/core/ops/ops.pbtxt`](https://www.tensorflow.org/code/tensorflow/core/ops/ops.pbtxt).
+[`tensorflow/core/ops/ops.pbtxt`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/ops/ops.pbtxt).
#### Handling Constants
@@ -228,4 +228,4 @@ At this time, support for gradients, functions and control flow operations ("if"
and "while") is not available in languages other than Python. This will be
updated when the [C API] provides necessary support.
-[C API]: https://www.tensorflow.org/code/tensorflow/c/c_api.h
+[C API]: https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/c/c_api.h
diff --git a/site/en/r1/guide/extend/filesystem.md b/site/en/r1/guide/extend/filesystem.md
index 4d34c07102e..2d6ea0c4645 100644
--- a/site/en/r1/guide/extend/filesystem.md
+++ b/site/en/r1/guide/extend/filesystem.md
@@ -54,7 +54,7 @@ To implement a custom filesystem plugin, you must do the following:
### The FileSystem interface
The `FileSystem` interface is an abstract C++ interface defined in
-[file_system.h](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/platform/file_system.h).
+[file_system.h](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/platform/file_system.h).
An implementation of the `FileSystem` interface should implement all relevant
the methods defined by the interface. Implementing the interface requires
defining operations such as creating `RandomAccessFile`, `WritableFile`, and
@@ -70,26 +70,26 @@ involves calling `stat()` on the file and then returns the filesize as reported
by the return of the stat object. Similarly, for the `HDFSFileSystem`
implementation, these calls simply delegate to the `libHDFS` implementation of
similar functionality, such as `hdfsDelete` for
-[DeleteFile](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/platform/hadoop/hadoop_file_system.cc#L386).
+[DeleteFile](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/platform/hadoop/hadoop_file_system.cc#L386).
We suggest looking through these code examples to get an idea of how different
filesystem implementations call their existing libraries. Examples include:
* [POSIX
- plugin](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/platform/posix/posix_file_system.h)
+ plugin](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/platform/posix/posix_file_system.h)
* [HDFS
- plugin](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/platform/hadoop/hadoop_file_system.h)
+ plugin](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/platform/hadoop/hadoop_file_system.h)
* [GCS
- plugin](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/platform/cloud/gcs_file_system.h)
+ plugin](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/platform/cloud/gcs_file_system.h)
* [S3
- plugin](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/platform/s3/s3_file_system.h)
+ plugin](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/platform/s3/s3_file_system.h)
#### The File interfaces
Beyond operations that allow you to query and manipulate files and directories
in a filesystem, the `FileSystem` interface requires you to implement factories
that return implementations of abstract objects such as the
-[RandomAccessFile](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/platform/file_system.h#L223),
+[RandomAccessFile](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/platform/file_system.h#L223),
the `WritableFile`, so that TensorFlow code and read and write to files in that
`FileSystem` implementation.
@@ -224,7 +224,7 @@ it will use the `FooBarFileSystem` implementation.
Next, you must build a shared object containing this implementation. An example
of doing so using bazel's `cc_binary` rule can be found
-[here](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/BUILD#L244),
+[here](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/python/BUILD#L244),
but you may use any build system to do so. See the section on [building the op library](../extend/op.md#build_the_op_library) for similar
instructions.
@@ -236,7 +236,7 @@ passing the path to the shared object. Calling this in your client program loads
the shared object in the process, thus registering your implementation as
available for any file operations going through the `FileSystem` interface. You
can see
-[test_file_system.py](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/framework/file_system_test.py)
+[test_file_system.py](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/python/framework/file_system_test.py)
for an example.
## What goes through this interface?
diff --git a/site/en/r1/guide/extend/formats.md b/site/en/r1/guide/extend/formats.md
index 3b7b4aafbd6..bdebee5487d 100644
--- a/site/en/r1/guide/extend/formats.md
+++ b/site/en/r1/guide/extend/formats.md
@@ -28,11 +28,11 @@ individual records in a file. There are several examples of "reader" datasets
that are already built into TensorFlow:
* `tf.data.TFRecordDataset`
- ([source in `kernels/data/reader_dataset_ops.cc`](https://www.tensorflow.org/code/tensorflow/core/kernels/data/reader_dataset_ops.cc))
+ ([source in `kernels/data/reader_dataset_ops.cc`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/kernels/data/reader_dataset_ops.cc))
* `tf.data.FixedLengthRecordDataset`
- ([source in `kernels/data/reader_dataset_ops.cc`](https://www.tensorflow.org/code/tensorflow/core/kernels/data/reader_dataset_ops.cc))
+ ([source in `kernels/data/reader_dataset_ops.cc`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/kernels/data/reader_dataset_ops.cc))
* `tf.data.TextLineDataset`
- ([source in `kernels/data/reader_dataset_ops.cc`](https://www.tensorflow.org/code/tensorflow/core/kernels/data/reader_dataset_ops.cc))
+ ([source in `kernels/data/reader_dataset_ops.cc`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/kernels/data/reader_dataset_ops.cc))
Each of these implementations comprises three related classes:
@@ -279,7 +279,7 @@ if __name__ == "__main__":
```
You can see some examples of `Dataset` wrapper classes in
-[`tensorflow/python/data/ops/dataset_ops.py`](https://www.tensorflow.org/code/tensorflow/python/data/ops/dataset_ops.py).
+[`tensorflow/python/data/ops/dataset_ops.py`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/python/data/ops/dataset_ops.py).
## Writing an Op for a record format
@@ -297,7 +297,7 @@ Examples of Ops useful for decoding records:
Note that it can be useful to use multiple Ops to decode a particular record
format. For example, you may have an image saved as a string in
-[a `tf.train.Example` protocol buffer](https://www.tensorflow.org/code/tensorflow/core/example/example.proto).
+[a `tf.train.Example` protocol buffer](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/example/example.proto).
Depending on the format of that image, you might take the corresponding output
from a `tf.parse_single_example` op and call `tf.image.decode_jpeg`,
`tf.image.decode_png`, or `tf.decode_raw`. It is common to take the output
diff --git a/site/en/r1/guide/extend/model_files.md b/site/en/r1/guide/extend/model_files.md
index 30e73a5169e..e590fcf1f27 100644
--- a/site/en/r1/guide/extend/model_files.md
+++ b/site/en/r1/guide/extend/model_files.md
@@ -28,7 +28,7 @@ by calling `as_graph_def()`, which returns a `GraphDef` object.
The GraphDef class is an object created by the ProtoBuf library from the
definition in
-[tensorflow/core/framework/graph.proto](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/graph.proto). The protobuf tools parse
+[tensorflow/core/framework/graph.proto](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/framework/graph.proto). The protobuf tools parse
this text file, and generate the code to load, store, and manipulate graph
definitions. If you see a standalone TensorFlow file representing a model, it's
likely to contain a serialized version of one of these `GraphDef` objects
@@ -87,7 +87,7 @@ for node in graph_def.node
```
Each node is a `NodeDef` object, defined in
-[tensorflow/core/framework/node_def.proto](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/node_def.proto). These
+[tensorflow/core/framework/node_def.proto](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/framework/node_def.proto). These
are the fundamental building blocks of TensorFlow graphs, with each one defining
a single operation along with its input connections. Here are the members of a
`NodeDef`, and what they mean.
@@ -107,7 +107,7 @@ This defines what operation to run, for example `"Add"`, `"MatMul"`, or
`"Conv2D"`. When a graph is run, this op name is looked up in a registry to
find an implementation. The registry is populated by calls to the
`REGISTER_OP()` macro, like those in
-[tensorflow/core/ops/nn_ops.cc](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/ops/nn_ops.cc).
+[tensorflow/core/ops/nn_ops.cc](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/ops/nn_ops.cc).
### `input`
@@ -133,7 +133,7 @@ size of filters for convolutions, or the values of constant ops. Because there
can be so many different types of attribute values, from strings, to ints, to
arrays of tensor values, there's a separate protobuf file defining the data
structure that holds them, in
-[tensorflow/core/framework/attr_value.proto](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/attr_value.proto).
+[tensorflow/core/framework/attr_value.proto](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/framework/attr_value.proto).
Each attribute has a unique name string, and the expected attributes are listed
when the operation is defined. If an attribute isn't present in a node, but it
@@ -151,7 +151,7 @@ the file format during training. Instead, they're held in separate checkpoint
files, and there are `Variable` ops in the graph that load the latest values
when they're initialized. It's often not very convenient to have separate files
when you're deploying to production, so there's the
-[freeze_graph.py](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/tools/freeze_graph.py) script that takes a graph definition and a set
+[freeze_graph.py](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/python/tools/freeze_graph.py) script that takes a graph definition and a set
of checkpoints and freezes them together into a single file.
What this does is load the `GraphDef`, pull in the values for all the variables
@@ -167,7 +167,7 @@ the most common problems is extracting and interpreting the weight values. A
common way to store them, for example in graphs created by the freeze_graph
script, is as `Const` ops containing the weights as `Tensors`. These are
defined in
-[tensorflow/core/framework/tensor.proto](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/tensor.proto), and contain information
+[tensorflow/core/framework/tensor.proto](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/framework/tensor.proto), and contain information
about the size and type of the data, as well as the values themselves. In
Python, you get a `TensorProto` object from a `NodeDef` representing a `Const`
op by calling something like `some_node_def.attr['value'].tensor`.
diff --git a/site/en/r1/guide/extend/op.md b/site/en/r1/guide/extend/op.md
index d006a6251d0..186d9c28c04 100644
--- a/site/en/r1/guide/extend/op.md
+++ b/site/en/r1/guide/extend/op.md
@@ -47,7 +47,7 @@ To incorporate your custom op you'll need to:
test the op in C++. If you define gradients, you can verify them with the
Python `tf.test.compute_gradient_error`.
See
- [`relu_op_test.py`](https://www.tensorflow.org/code/tensorflow/python/kernel_tests/relu_op_test.py) as
+ [`relu_op_test.py`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/python/kernel_tests/relu_op_test.py) as
an example that tests the forward functions of Relu-like operators and
their gradients.
@@ -155,17 +155,17 @@ REGISTER_KERNEL_BUILDER(Name("ZeroOut").Device(DEVICE_CPU), ZeroOutOp);
> Important: Instances of your OpKernel may be accessed concurrently.
> Your `Compute` method must be thread-safe. Guard any access to class
> members with a mutex. Or better yet, don't share state via class members!
-> Consider using a [`ResourceMgr`](https://www.tensorflow.org/code/tensorflow/core/framework/resource_mgr.h)
+> Consider using a [`ResourceMgr`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/framework/resource_mgr.h)
> to keep track of op state.
### Multi-threaded CPU kernels
To write a multi-threaded CPU kernel, the Shard function in
-[`work_sharder.h`](https://www.tensorflow.org/code/tensorflow/core/util/work_sharder.h)
+[`work_sharder.h`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/util/work_sharder.h)
can be used. This function shards a computation function across the
threads configured to be used for intra-op threading (see
intra_op_parallelism_threads in
-[`config.proto`](https://www.tensorflow.org/code/tensorflow/core/protobuf/config.proto)).
+[`config.proto`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/protobuf/config.proto)).
### GPU kernels
@@ -348,12 +348,13 @@ g++ -std=c++11 -shared zero_out.cc -o zero_out.so -fPIC ${TF_CFLAGS[@]} ${TF_LFL
On macOS, the additional flag "-undefined dynamic_lookup" is required when
building the `.so` file.
-> Note on `gcc` version `>=5`: gcc uses the new C++
-> [ABI](https://gcc.gnu.org/gcc-5/changes.html#libstdcxx) since version `5`. The binary pip
-> packages available on the TensorFlow website are built with `gcc4` that uses
-> the older ABI. If you compile your op library with `gcc>=5`, add
-> `-D_GLIBCXX_USE_CXX11_ABI=0` to the command line to make the library
-> compatible with the older abi.
+> Note on `gcc` version `>=5`: gcc uses the new C++
+> [ABI](https://gcc.gnu.org/gcc-5/changes.html#libstdcxx) since version `5`.
+> TensorFlow 2.8 and earlier were built with `gcc4` that uses the older ABI. If
+> you are using these versions of TensorFlow and are trying to compile your op
+> library with `gcc>=5`, add `-D_GLIBCXX_USE_CXX11_ABI=0` to the command line to
+> make the library compatible with the older ABI. TensorFlow 2.9+ packages are
+> compatible with the newer ABI by default.
### Compile the op using bazel (TensorFlow source installation)
@@ -485,13 +486,13 @@ This asserts that the input is a vector, and returns having set the
* The `context`, which can either be an `OpKernelContext` or
`OpKernelConstruction` pointer (see
- [`tensorflow/core/framework/op_kernel.h`](https://www.tensorflow.org/code/tensorflow/core/framework/op_kernel.h)),
+ [`tensorflow/core/framework/op_kernel.h`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/framework/op_kernel.h)),
for its `SetStatus()` method.
* The condition. For example, there are functions for validating the shape
of a tensor in
- [`tensorflow/core/framework/tensor_shape.h`](https://www.tensorflow.org/code/tensorflow/core/framework/tensor_shape.h)
+ [`tensorflow/core/framework/tensor_shape.h`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/framework/tensor_shape.h)
* The error itself, which is represented by a `Status` object, see
- [`tensorflow/core/lib/core/status.h`](https://www.tensorflow.org/code/tensorflow/core/lib/core/status.h). A
+ [`tensorflow/core/lib/core/status.h`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/lib/core/status.h). A
`Status` has both a type (frequently `InvalidArgument`, but see the list of
types) and a message. Functions for constructing an error may be found in
[`tensorflow/core/lib/core/errors.h`][validation-macros].
@@ -632,7 +633,7 @@ define an attr with constraints, you can use the following ``s:
The specific lists of types allowed by these are defined by the functions
(like `NumberTypes()`) in
- [`tensorflow/core/framework/types.h`](https://www.tensorflow.org/code/tensorflow/core/framework/types.h).
+ [`tensorflow/core/framework/types.h`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/framework/types.h).
In this example the attr `t` must be one of the numeric types:
```c++
@@ -1179,7 +1180,7 @@ There are several ways to preserve backwards-compatibility.
type into a list of varying types).
The full list of safe and unsafe changes can be found in
-[`tensorflow/core/framework/op_compatibility_test.cc`](https://www.tensorflow.org/code/tensorflow/core/framework/op_compatibility_test.cc).
+[`tensorflow/core/framework/op_compatibility_test.cc`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/framework/op_compatibility_test.cc).
If you cannot make your change to an operation backwards compatible, then create
a new operation with a new name with the new semantics.
@@ -1189,23 +1190,23 @@ callers. The Python API may be kept compatible by careful changes in a
hand-written Python wrapper, by keeping the old signature except possibly adding
new optional arguments to the end. Generally incompatible changes may only be
made when TensorFlow changes major versions, and must conform to the
-[`GraphDef` version semantics](../guide/version_compat.md#compatibility_of_graphs_and_checkpoints).
+[`GraphDef` version semantics](../version_compat.md).
### GPU Support
You can implement different OpKernels and register one for CPU and another for
GPU, just like you can [register kernels for different types](#polymorphism).
There are several examples of kernels with GPU support in
-[`tensorflow/core/kernels/`](https://www.tensorflow.org/code/tensorflow/core/kernels/).
+[`tensorflow/core/kernels/`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/kernels/).
Notice some kernels have a CPU version in a `.cc` file, a GPU version in a file
ending in `_gpu.cu.cc`, and some code shared in common in a `.h` file.
For example, the `tf.pad` has
everything but the GPU kernel in [`tensorflow/core/kernels/pad_op.cc`][pad_op].
The GPU kernel is in
-[`tensorflow/core/kernels/pad_op_gpu.cu.cc`](https://www.tensorflow.org/code/tensorflow/core/kernels/pad_op_gpu.cu.cc),
+[`tensorflow/core/kernels/pad_op_gpu.cu.cc`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/kernels/pad_op_gpu.cu.cc),
and the shared code is a templated class defined in
-[`tensorflow/core/kernels/pad_op.h`](https://www.tensorflow.org/code/tensorflow/core/kernels/pad_op.h).
+[`tensorflow/core/kernels/pad_op.h`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/kernels/pad_op.h).
We organize the code this way for two reasons: it allows you to share common
code among the CPU and GPU implementations, and it puts the GPU implementation
into a separate file so that it can be compiled only by the GPU compiler.
@@ -1226,16 +1227,16 @@ kept on the CPU, add a `HostMemory()` call to the kernel registration, e.g.:
#### Compiling the kernel for the GPU device
Look at
-[cuda_op_kernel.cu.cc](https://www.tensorflow.org/code/tensorflow/examples/adding_an_op/cuda_op_kernel.cu.cc)
+[cuda_op_kernel.cu.cc](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/examples/adding_an_op/cuda_op_kernel.cu.cc)
for an example that uses a CUDA kernel to implement an op. The
`tf_custom_op_library` accepts a `gpu_srcs` argument in which the list of source
files containing the CUDA kernels (`*.cu.cc` files) can be specified. For use
with a binary installation of TensorFlow, the CUDA kernels have to be compiled
with NVIDIA's `nvcc` compiler. Here is the sequence of commands you can use to
compile the
-[cuda_op_kernel.cu.cc](https://www.tensorflow.org/code/tensorflow/examples/adding_an_op/cuda_op_kernel.cu.cc)
+[cuda_op_kernel.cu.cc](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/examples/adding_an_op/cuda_op_kernel.cu.cc)
and
-[cuda_op_kernel.cc](https://www.tensorflow.org/code/tensorflow/examples/adding_an_op/cuda_op_kernel.cc)
+[cuda_op_kernel.cc](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/examples/adding_an_op/cuda_op_kernel.cc)
into a single dynamically loadable library:
```bash
@@ -1360,7 +1361,7 @@ be set to the first input's shape. If the output is selected by its index as in
There are a number of common shape functions
that apply to many ops, such as `shape_inference::UnchangedShape` which can be
-found in [common_shape_fns.h](https://www.tensorflow.org/code/tensorflow/core/framework/common_shape_fns.h) and used as follows:
+found in [common_shape_fns.h](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/framework/common_shape_fns.h) and used as follows:
```c++
REGISTER_OP("ZeroOut")
@@ -1407,7 +1408,7 @@ provides access to the attributes of the op).
Since shape inference is an optional feature, and the shapes of tensors may vary
dynamically, shape functions must be robust to incomplete shape information for
-any of the inputs. The `Merge` method in [`InferenceContext`](https://www.tensorflow.org/code/tensorflow/core/framework/shape_inference.h)
+any of the inputs. The `Merge` method in [`InferenceContext`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/framework/shape_inference.h)
allows the caller to assert that two shapes are the same, even if either
or both of them do not have complete information. Shape functions are defined
for all of the core TensorFlow ops and provide many different usage examples.
@@ -1432,7 +1433,7 @@ If you have a complicated shape function, you should consider adding a test for
validating that various input shape combinations produce the expected output
shape combinations. You can see examples of how to write these tests in some
our
-[core ops tests](https://www.tensorflow.org/code/tensorflow/core/ops/array_ops_test.cc).
+[core ops tests](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/ops/array_ops_test.cc).
(The syntax of `INFER_OK` and `INFER_ERROR` are a little cryptic, but try to be
compact in representing input and output shape specifications in tests. For
now, see the surrounding comments in those tests to get a sense of the shape
@@ -1445,20 +1446,20 @@ To build a `pip` package for your op, see the
guide shows how to build custom ops from the TensorFlow pip package instead
of building TensorFlow from source.
-[core-array_ops]:https://www.tensorflow.org/code/tensorflow/core/ops/array_ops.cc
-[python-user_ops]:https://www.tensorflow.org/code/tensorflow/python/user_ops/user_ops.py
-[tf-kernels]:https://www.tensorflow.org/code/tensorflow/core/kernels/
-[user_ops]:https://www.tensorflow.org/code/tensorflow/core/user_ops/
-[pad_op]:https://www.tensorflow.org/code/tensorflow/core/kernels/pad_op.cc
-[standard_ops-py]:https://www.tensorflow.org/code/tensorflow/python/ops/standard_ops.py
-[standard_ops-cc]:https://www.tensorflow.org/code/tensorflow/cc/ops/standard_ops.h
-[python-BUILD]:https://www.tensorflow.org/code/tensorflow/python/BUILD
-[validation-macros]:https://www.tensorflow.org/code/tensorflow/core/lib/core/errors.h
-[op_def_builder]:https://www.tensorflow.org/code/tensorflow/core/framework/op_def_builder.h
-[register_types]:https://www.tensorflow.org/code/tensorflow/core/framework/register_types.h
-[FinalizeAttr]:https://www.tensorflow.org/code/tensorflow/core/framework/op_def_builder.cc
-[DataTypeString]:https://www.tensorflow.org/code/tensorflow/core/framework/types.cc
-[python-BUILD]:https://www.tensorflow.org/code/tensorflow/python/BUILD
-[types-proto]:https://www.tensorflow.org/code/tensorflow/core/framework/types.proto
-[TensorShapeProto]:https://www.tensorflow.org/code/tensorflow/core/framework/tensor_shape.proto
-[TensorProto]:https://www.tensorflow.org/code/tensorflow/core/framework/tensor.proto
+[core-array_ops]:https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/ops/array_ops.cc
+[python-user_ops]:https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/python/user_ops/user_ops.py
+[tf-kernels]:https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/kernels/
+[user_ops]:https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/user_ops/
+[pad_op]:https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/kernels/pad_op.cc
+[standard_ops-py]:https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/python/ops/standard_ops.py
+[standard_ops-cc]:https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/cc/ops/standard_ops.h
+[python-BUILD]:https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/python/BUILD
+[validation-macros]:https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/lib/core/errors.h
+[op_def_builder]:https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/framework/op_def_builder.h
+[register_types]:https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/framework/register_types.h
+[FinalizeAttr]:https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/framework/op_def_builder.cc
+[DataTypeString]:https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/framework/types.cc
+[python-BUILD]:https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/python/BUILD
+[types-proto]:https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/framework/types.proto
+[TensorShapeProto]:https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/framework/tensor_shape.proto
+[TensorProto]:https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/framework/tensor.proto
diff --git a/site/en/r1/guide/feature_columns.md b/site/en/r1/guide/feature_columns.md
index 5a4dfbbf46d..e4259f85e9f 100644
--- a/site/en/r1/guide/feature_columns.md
+++ b/site/en/r1/guide/feature_columns.md
@@ -562,7 +562,7 @@ For more examples on feature columns, view the following:
* The [Low Level Introduction](../guide/low_level_intro.md#feature_columns) demonstrates how
experiment directly with `feature_columns` using TensorFlow's low level APIs.
-* The [Estimator wide and deep learning tutorial](https://github.com/tensorflow/models/tree/master/official/r1/wide_deep)
+* The [Estimator wide and deep learning tutorial](https://github.com/tensorflow/models/tree/r1.15/official/r1/wide_deep)
solves a binary classification problem using `feature_columns` on a variety of
input data types.
diff --git a/site/en/r1/guide/graph_viz.md b/site/en/r1/guide/graph_viz.md
index 1965378e03e..1e3780e7928 100644
--- a/site/en/r1/guide/graph_viz.md
+++ b/site/en/r1/guide/graph_viz.md
@@ -251,7 +251,7 @@ is a snippet from the train and test section of a modification of the
[Estimators MNIST tutorial](../tutorials/estimators/cnn.md), in which we have
recorded summaries and
runtime statistics. See the
-[Tensorboard](https://tensorflow.org/tensorboard)
+[TensorBoard](https://tensorflow.org/tensorboard)
for details on how to record summaries.
Full source is [here](https://github.com/tensorflow/tensorflow/tree/r1.15/tensorflow/examples/tutorials/mnist/mnist_with_summaries.py).
diff --git a/site/en/r1/guide/keras.ipynb b/site/en/r1/guide/keras.ipynb
index d722660a59c..e9e577a1228 100644
--- a/site/en/r1/guide/keras.ipynb
+++ b/site/en/r1/guide/keras.ipynb
@@ -1211,7 +1211,6 @@
"colab": {
"collapsed_sections": [],
"name": "keras.ipynb",
- "provenance": [],
"toc_visible": true
},
"kernelspec": {
diff --git a/site/en/r1/guide/performance/benchmarks.md b/site/en/r1/guide/performance/benchmarks.md
index 8998c0723db..a56959ea416 100644
--- a/site/en/r1/guide/performance/benchmarks.md
+++ b/site/en/r1/guide/performance/benchmarks.md
@@ -401,7 +401,7 @@ GPUs | InceptionV3 (batch size 32) | ResNet-50 (batch size 32)
## Methodology
This
-[script](https://github.com/tensorflow/benchmarks/tree/master/scripts/tf_cnn_benchmarks)
+[script](https://github.com/tensorflow/benchmarks/tree/r1.15/scripts/tf_cnn_benchmarks)
was run on the various platforms to generate the above results.
In order to create results that are as repeatable as possible, each test was run
diff --git a/site/en/r1/guide/performance/overview.md b/site/en/r1/guide/performance/overview.md
index af74f0f28c6..be7217f4b99 100644
--- a/site/en/r1/guide/performance/overview.md
+++ b/site/en/r1/guide/performance/overview.md
@@ -19,9 +19,9 @@ Reading large numbers of small files significantly impacts I/O performance.
One approach to get maximum I/O throughput is to preprocess input data into
larger (~100MB) `TFRecord` files. For smaller data sets (200MB-1GB), the best
approach is often to load the entire data set into memory. The document
-[Downloading and converting to TFRecord format](https://github.com/tensorflow/models/tree/master/research/slim#downloading-and-converting-to-tfrecord-format)
+[Downloading and converting to TFRecord format](https://github.com/tensorflow/models/tree/r1.15/research/slim#downloading-and-converting-to-tfrecord-format)
includes information and scripts for creating `TFRecord`s, and this
-[script](https://github.com/tensorflow/models/tree/master/research/tutorials/image/cifar10_estimator/generate_cifar10_tfrecords.py)
+[script](https://github.com/tensorflow/models/tree/r1.15/research/tutorials/image/cifar10_estimator/generate_cifar10_tfrecords.py)
converts the CIFAR-10 dataset into `TFRecord`s.
While feeding data using a `feed_dict` offers a high level of flexibility, in
@@ -122,7 +122,7 @@ tf.Session(config=config)
Intel® has added optimizations to TensorFlow for Intel® Xeon® and Intel® Xeon
Phi™ through the use of the Intel® Math Kernel Library for Deep Neural Networks
(Intel® MKL-DNN) optimized primitives. The optimizations also provide speedups
-for the consumer line of processors, e.g. i5 and i7 Intel processors. The Intel
+for the consumer line of processors, e.g., i5 and i7 Intel processors. The Intel
published paper
[TensorFlow* Optimizations on Modern Intel® Architecture](https://software.intel.com/en-us/articles/tensorflow-optimizations-on-modern-intel-architecture)
contains additional details on the implementation.
@@ -255,7 +255,7 @@ bazel build -c opt --copt=-march="broadwell" --config=cuda //tensorflow/tools/pi
a docker container, the data is not cached and the penalty is paid each time
TensorFlow starts. The best practice is to include the
[compute capabilities](http://developer.nvidia.com/cuda-gpus)
- of the GPUs that will be used, e.g. P100: 6.0, Titan X (Pascal): 6.1,
+ of the GPUs that will be used, e.g., P100: 6.0, Titan X (Pascal): 6.1,
Titan X (Maxwell): 5.2, and K80: 3.7.
* Use a version of `gcc` that supports all of the optimizations of the target
CPU. The recommended minimum gcc version is 4.8.3. On macOS, upgrade to the
diff --git a/site/en/r1/guide/saved_model.md b/site/en/r1/guide/saved_model.md
index 623863a9df9..34447ffe861 100644
--- a/site/en/r1/guide/saved_model.md
+++ b/site/en/r1/guide/saved_model.md
@@ -23,7 +23,7 @@ TensorFlow saves variables in binary *checkpoint files* that map variable
names to tensor values.
Caution: TensorFlow model files are code. Be careful with untrusted code.
-See [Using TensorFlow Securely](https://github.com/tensorflow/tensorflow/blob/master/SECURITY.md)
+See [Using TensorFlow Securely](https://github.com/tensorflow/tensorflow/blob/r1.15/SECURITY.md)
for details.
### Save variables
@@ -148,7 +148,7 @@ Notes:
`tf.variables_initializer` for more information.
* To inspect the variables in a checkpoint, you can use the
- [`inspect_checkpoint`](https://www.tensorflow.org/code/tensorflow/python/tools/inspect_checkpoint.py)
+ [`inspect_checkpoint`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/python/tools/inspect_checkpoint.py)
library, particularly the `print_tensors_in_checkpoint_file` function.
* By default, `Saver` uses the value of the `tf.Variable.name` property
@@ -159,7 +159,7 @@ Notes:
### Inspect variables in a checkpoint
We can quickly inspect variables in a checkpoint with the
-[`inspect_checkpoint`](https://www.tensorflow.org/code/tensorflow/python/tools/inspect_checkpoint.py) library.
+[`inspect_checkpoint`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/python/tools/inspect_checkpoint.py) library.
Continuing from the save/restore examples shown earlier:
@@ -216,7 +216,7 @@ simple_save(session,
This configures the `SavedModel` so it can be loaded by
[TensorFlow serving](https://www.tensorflow.org/tfx/tutorials/serving/rest_simple) and supports the
-[Predict API](https://github.com/tensorflow/serving/blob/master/tensorflow_serving/apis/predict.proto).
+[Predict API](https://github.com/tensorflow/serving/blob/r1.15/tensorflow_serving/apis/predict.proto).
To access the classify, regress, or multi-inference APIs, use the manual
`SavedModel` builder APIs or an `tf.estimator.Estimator`.
@@ -328,7 +328,7 @@ with tf.Session(graph=tf.Graph()) as sess:
### Load a SavedModel in C++
The C++ version of the SavedModel
-[loader](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/cc/saved_model/loader.h)
+[loader](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/cc/saved_model/loader.h)
provides an API to load a SavedModel from a path, while allowing
`SessionOptions` and `RunOptions`.
You have to specify the tags associated with the graph to be loaded.
@@ -383,20 +383,20 @@ reuse and share across tools consistently.
You may use sets of tags to uniquely identify a `MetaGraphDef` saved in a
SavedModel. A subset of commonly used tags is specified in:
-* [Python](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/saved_model/tag_constants.py)
-* [C++](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/cc/saved_model/tag_constants.h)
+* [Python](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/python/saved_model/tag_constants.py)
+* [C++](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/cc/saved_model/tag_constants.h)
#### Standard SignatureDef constants
-A [**SignatureDef**](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/protobuf/meta_graph.proto)
+A [**SignatureDef**](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/protobuf/meta_graph.proto)
is a protocol buffer that defines the signature of a computation
supported by a graph.
Commonly used input keys, output keys, and method names are
defined in:
-* [Python](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/saved_model/signature_constants.py)
-* [C++](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/cc/saved_model/signature_constants.h)
+* [Python](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/python/saved_model/signature_constants.py)
+* [C++](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/cc/saved_model/signature_constants.h)
## Using SavedModel with Estimators
@@ -408,7 +408,7 @@ To prepare a trained Estimator for serving, you must export it in the standard
SavedModel format. This section explains how to:
* Specify the output nodes and the corresponding
- [APIs](https://github.com/tensorflow/serving/blob/master/tensorflow_serving/apis/prediction_service.proto)
+ [APIs](https://github.com/tensorflow/serving/blob/r1.15/tensorflow_serving/apis/prediction_service.proto)
that can be served (Classify, Regress, or Predict).
* Export your model to the SavedModel format.
* Serve the model from a local server and request predictions.
@@ -506,7 +506,7 @@ Each `output` value must be an `ExportOutput` object such as
`tf.estimator.export.PredictOutput`.
These output types map straightforwardly to the
-[TensorFlow Serving APIs](https://github.com/tensorflow/serving/blob/master/tensorflow_serving/apis/prediction_service.proto),
+[TensorFlow Serving APIs](https://github.com/tensorflow/serving/blob/r1.15/tensorflow_serving/apis/prediction_service.proto),
and so determine which request types will be honored.
Note: In the multi-headed case, a `SignatureDef` will be generated for each
@@ -515,7 +515,7 @@ the same keys. These `SignatureDef`s differ only in their outputs, as
provided by the corresponding `ExportOutput` entry. The inputs are always
those provided by the `serving_input_receiver_fn`.
An inference request may specify the head by name. One head must be named
-using [`signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY`](https://www.tensorflow.org/code/tensorflow/python/saved_model/signature_constants.py)
+using [`signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/python/saved_model/signature_constants.py)
indicating which `SignatureDef` will be served when an inference request
does not specify one.
@@ -566,9 +566,9 @@ Now you have a server listening for inference requests via gRPC on port 9000!
### Request predictions from a local server
The server responds to gRPC requests according to the
-[PredictionService](https://github.com/tensorflow/serving/blob/master/tensorflow_serving/apis/prediction_service.proto#L15)
+[PredictionService](https://github.com/tensorflow/serving/blob/r1.15/tensorflow_serving/apis/prediction_service.proto#L15)
gRPC API service definition. (The nested protocol buffers are defined in
-various [neighboring files](https://github.com/tensorflow/serving/blob/master/tensorflow_serving/apis)).
+various [neighboring files](https://github.com/tensorflow/serving/blob/r1.15/tensorflow_serving/apis)).
From the API service definition, the gRPC framework generates client libraries
in various languages providing remote access to the API. In a project using the
@@ -620,7 +620,7 @@ The returned result in this example is a `ClassificationResponse` protocol
buffer.
This is a skeletal example; please see the [Tensorflow Serving](../deploy/index.md)
-documentation and [examples](https://github.com/tensorflow/serving/tree/master/tensorflow_serving/example)
+documentation and [examples](https://github.com/tensorflow/serving/tree/r1.15/tensorflow_serving/example)
for more details.
> Note: `ClassificationRequest` and `RegressionRequest` contain a
diff --git a/site/en/r1/guide/using_tpu.md b/site/en/r1/guide/using_tpu.md
index 74169092189..e3e338adf49 100644
--- a/site/en/r1/guide/using_tpu.md
+++ b/site/en/r1/guide/using_tpu.md
@@ -7,8 +7,8 @@ changing the *hardware accelerator* in your notebook settings:
TPU-enabled Colab notebooks are available to test:
1. [A quick test, just to measure FLOPS](https://colab.research.google.com/notebooks/tpu.ipynb).
- 2. [A CNN image classifier with `tf.keras`](https://colab.research.google.com/github/tensorflow/tpu/blob/master/tools/colab/fashion_mnist.ipynb).
- 3. [An LSTM markov chain text generator with `tf.keras`](https://colab.research.google.com/github/tensorflow/tpu/blob/master/tools/colab/shakespeare_with_tpu_and_keras.ipynb)
+ 2. [A CNN image classifier with `tf.keras`](https://colab.research.google.com/github/tensorflow/tpu/blob/r1.15/tools/colab/fashion_mnist.ipynb).
+ 3. [An LSTM markov chain text generator with `tf.keras`](https://colab.research.google.com/github/tensorflow/tpu/blob/r1.15/tools/colab/shakespeare_with_tpu_and_keras.ipynb)
## TPUEstimator
@@ -25,7 +25,7 @@ Cloud TPU is to define the model's inference phase (from inputs to predictions)
outside of the `model_fn`. Then maintain separate implementations of the
`Estimator` setup and `model_fn`, both wrapping this inference step. For an
example of this pattern compare the `mnist.py` and `mnist_tpu.py` implementation in
-[tensorflow/models](https://github.com/tensorflow/models/tree/master/official/r1/mnist).
+[tensorflow/models](https://github.com/tensorflow/models/tree/r1.15/official/r1/mnist).
### Run a TPUEstimator locally
@@ -350,10 +350,10 @@ in bytes. A minimum of a few MB (`buffer_size=8*1024*1024`) is recommended so
that data is available when needed.
The TPU-demos repo includes
-[a script](https://github.com/tensorflow/tpu/blob/master/tools/datasets/imagenet_to_gcs.py)
+[a script](https://github.com/tensorflow/tpu/blob/1.15/tools/datasets/imagenet_to_gcs.py)
for downloading the imagenet dataset and converting it to an appropriate format.
This together with the imagenet
-[models](https://github.com/tensorflow/tpu/tree/master/models)
+[models](https://github.com/tensorflow/tpu/tree/r1.15/models)
included in the repo demonstrate all of these best-practices.
## Next steps
diff --git a/site/en/r1/guide/version_compat.md b/site/en/r1/guide/version_compat.md
index 6702f6e0819..a765620518d 100644
--- a/site/en/r1/guide/version_compat.md
+++ b/site/en/r1/guide/version_compat.md
@@ -49,19 +49,19 @@ patch versions. The public APIs consist of
submodules, but is not documented, then it is **not** considered part of the
public API.
-* The [C API](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/c/c_api.h).
+* The [C API](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/c/c_api.h).
* The following protocol buffer files:
- * [`attr_value`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/attr_value.proto)
- * [`config`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/protobuf/config.proto)
- * [`event`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/util/event.proto)
- * [`graph`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/graph.proto)
- * [`op_def`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/op_def.proto)
- * [`reader_base`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/reader_base.proto)
- * [`summary`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/summary.proto)
- * [`tensor`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/tensor.proto)
- * [`tensor_shape`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/tensor_shape.proto)
- * [`types`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/types.proto)
+ * [`attr_value`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/framework/attr_value.proto)
+ * [`config`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/protobuf/config.proto)
+ * [`event`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/util/event.proto)
+ * [`graph`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/framework/graph.proto)
+ * [`op_def`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/framework/op_def.proto)
+ * [`reader_base`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/framework/reader_base.proto)
+ * [`summary`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/framework/summary.proto)
+ * [`tensor`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/framework/tensor.proto)
+ * [`tensor_shape`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/framework/tensor_shape.proto)
+ * [`types`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/framework/types.proto)
## What is *not* covered
@@ -79,7 +79,7 @@ backward incompatible ways between minor releases. These include:
such as:
- [C++](./extend/cc.md) (exposed through header files in
- [`tensorflow/cc`](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/cc)).
+ [`tensorflow/cc`](https://github.com/tensorflow/tensorflow/tree/r1.15/tensorflow/cc)).
- [Java](../api_docs/java/reference/org/tensorflow/package-summary),
- [Go](https://pkg.go.dev/github.com/tensorflow/tensorflow/tensorflow/go)
- [JavaScript](https://js.tensorflow.org)
@@ -209,7 +209,7 @@ guidelines for evolving `GraphDef` versions.
There are different data versions for graphs and checkpoints. The two data
formats evolve at different rates from each other and also at different rates
from TensorFlow. Both versioning systems are defined in
-[`core/public/version.h`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/public/version.h).
+[`core/public/version.h`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/public/version.h).
Whenever a new version is added, a note is added to the header detailing what
changed and the date.
@@ -224,7 +224,7 @@ We distinguish between the following kinds of data version information:
(`min_producer`).
Each piece of versioned data has a [`VersionDef
-versions`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/versions.proto)
+versions`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/framework/versions.proto)
field which records the `producer` that made the data, the `min_consumer`
that it is compatible with, and a list of `bad_consumers` versions that are
disallowed.
@@ -239,7 +239,7 @@ accept a piece of data if the following are all true:
* `consumer` not in data's `bad_consumers`
Since both producers and consumers come from the same TensorFlow code base,
-[`core/public/version.h`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/public/version.h)
+[`core/public/version.h`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/public/version.h)
contains a main data version which is treated as either `producer` or
`consumer` depending on context and both `min_consumer` and `min_producer`
(needed by producers and consumers, respectively). Specifically,
@@ -309,7 +309,7 @@ existing producer scripts will not suddenly use the new functionality.
1. Add a new similar op named `SomethingV2` or similar and go through the
process of adding it and switching existing Python wrappers to use it.
To ensure forward compatibility use the checks suggested in
- [compat.py](https://www.tensorflow.org/code/tensorflow/python/compat/compat.py)
+ [compat.py](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/python/compat/compat.py)
when changing the Python wrappers.
2. Remove the old op (Can only take place with a major version change due to
backward compatibility).
diff --git a/site/en/r1/tutorials/README.md b/site/en/r1/tutorials/README.md
index 5094e645e6e..9ff164ad77c 100644
--- a/site/en/r1/tutorials/README.md
+++ b/site/en/r1/tutorials/README.md
@@ -68,4 +68,4 @@ implement common ML algorithms. See the
* [Boosted trees](./estimators/boosted_trees.ipynb)
* [Gradient Boosted Trees: Model understanding](./estimators/boosted_trees_model_understanding.ipynb)
* [Build a Convolutional Neural Network using Estimators](./estimators/cnn.ipynb)
-* [Wide and deep learning with Estimators](https://github.com/tensorflow/models/tree/master/official/r1/wide_deep)
+* [Wide and deep learning with Estimators](https://github.com/tensorflow/models/tree/r1.15/official/r1/wide_deep)
diff --git a/site/en/r1/tutorials/distribute/keras.ipynb b/site/en/r1/tutorials/distribute/keras.ipynb
index 6b5258ee5a0..14e8bf739a9 100644
--- a/site/en/r1/tutorials/distribute/keras.ipynb
+++ b/site/en/r1/tutorials/distribute/keras.ipynb
@@ -86,7 +86,7 @@
"Essentially, it copies all of the model's variables to each processor.\n",
"Then, it uses [all-reduce](http://mpitutorial.com/tutorials/mpi-reduce-and-allreduce/) to combine the gradients from all processors and applies the combined value to all copies of the model.\n",
"\n",
- "`MirroredStategy` is one of several distribution strategy available in TensorFlow core. You can read about more strategies at [distribution strategy guide](../../guide/distribute_strategy.ipynb).\n"
+ "`MirroredStrategy` is one of several distribution strategy available in TensorFlow core. You can read about more strategies at [distribution strategy guide](../../guide/distribute_strategy.ipynb).\n"
]
},
{
@@ -554,7 +554,7 @@
},
"outputs": [],
"source": [
- "tf.keras.experimental.export_saved_model(model, path)"
+ "model.save(path)"
]
},
{
@@ -574,7 +574,7 @@
},
"outputs": [],
"source": [
- "unreplicated_model = tf.keras.experimental.load_from_saved_model(path)\n",
+ "unreplicated_model = tf.keras.models.load_model(path)\n",
"\n",
"unreplicated_model.compile(\n",
" loss='sparse_categorical_crossentropy',\n",
diff --git a/site/en/r1/tutorials/eager/custom_layers.ipynb b/site/en/r1/tutorials/eager/custom_layers.ipynb
index 954601e371b..48b55ed943e 100644
--- a/site/en/r1/tutorials/eager/custom_layers.ipynb
+++ b/site/en/r1/tutorials/eager/custom_layers.ipynb
@@ -230,7 +230,7 @@
"source": [
"## Models: composing layers\n",
"\n",
- "Many interesting layer-like things in machine learning models are implemented by composing existing layers. For example, each residual block in a resnet is a composition of convolutions, batch normalizations, and a shortcut.\n",
+ "Many interesting layer-like things in machine learning models are implemented by composing existing layers. For example, each residual block in a ResNet is a composition of convolutions, batch normalizations, and a shortcut.\n",
"\n",
"The main class used when creating a layer-like thing which contains other layers is tf.keras.Model. Implementing one is done by inheriting from tf.keras.Model."
]
diff --git a/site/en/r1/tutorials/images/deep_cnn.md b/site/en/r1/tutorials/images/deep_cnn.md
index 00a914d8976..885f3907aa7 100644
--- a/site/en/r1/tutorials/images/deep_cnn.md
+++ b/site/en/r1/tutorials/images/deep_cnn.md
@@ -80,15 +80,15 @@ for details. It consists of 1,068,298 learnable parameters and requires about
## Code Organization
The code for this tutorial resides in
-[`models/tutorials/image/cifar10/`](https://github.com/tensorflow/models/tree/master/research/tutorials/image/cifar10/).
+[`models/tutorials/image/cifar10/`](https://github.com/tensorflow/models/tree/r1.15/research/tutorials/image/cifar10/).
File | Purpose
--- | ---
-[`cifar10_input.py`](https://github.com/tensorflow/models/tree/master/research/tutorials/image/cifar10/cifar10_input.py) | Loads CIFAR-10 dataset using [tensorflow-datasets library](https://github.com/tensorflow/datasets).
-[`cifar10.py`](https://github.com/tensorflow/models/tree/master/research/tutorials/image/cifar10/cifar10.py) | Builds the CIFAR-10 model.
-[`cifar10_train.py`](https://github.com/tensorflow/models/tree/master/research/tutorials/image/cifar10/cifar10_train.py) | Trains a CIFAR-10 model on a CPU or GPU.
-[`cifar10_multi_gpu_train.py`](https://github.com/tensorflow/models/tree/master/research/tutorials/image/cifar10/cifar10_multi_gpu_train.py) | Trains a CIFAR-10 model on multiple GPUs.
-[`cifar10_eval.py`](https://github.com/tensorflow/models/tree/master/research/tutorials/image/cifar10/cifar10_eval.py) | Evaluates the predictive performance of a CIFAR-10 model.
+[`cifar10_input.py`](https://github.com/tensorflow/models/tree/r1.15/research/tutorials/image/cifar10/cifar10_input.py) | Loads CIFAR-10 dataset using [tensorflow-datasets library](https://github.com/tensorflow/datasets).
+[`cifar10.py`](https://github.com/tensorflow/models/tree/r1.15/research/tutorials/image/cifar10/cifar10.py) | Builds the CIFAR-10 model.
+[`cifar10_train.py`](https://github.com/tensorflow/models/tree/r1.15/research/tutorials/image/cifar10/cifar10_train.py) | Trains a CIFAR-10 model on a CPU or GPU.
+[`cifar10_multi_gpu_train.py`](https://github.com/tensorflow/models/tree/r1.15/research/tutorials/image/cifar10/cifar10_multi_gpu_train.py) | Trains a CIFAR-10 model on multiple GPUs.
+[`cifar10_eval.py`](https://github.com/tensorflow/models/tree/r1.15/research/tutorials/image/cifar10/cifar10_eval.py) | Evaluates the predictive performance of a CIFAR-10 model.
To run this tutorial, you will need to:
@@ -99,7 +99,7 @@ pip install tensorflow-datasets
## CIFAR-10 Model
The CIFAR-10 network is largely contained in
-[`cifar10.py`](https://github.com/tensorflow/models/tree/master/research/tutorials/image/cifar10/cifar10.py).
+[`cifar10.py`](https://github.com/tensorflow/models/tree/r1.15/research/tutorials/image/cifar10/cifar10.py).
The complete training
graph contains roughly 765 operations. We find that we can make the code most
reusable by constructing the graph with the following modules:
@@ -108,7 +108,7 @@ reusable by constructing the graph with the following modules:
operations that read and preprocess CIFAR images for evaluation and training,
respectively.
1. [**Model prediction:**](#model-prediction) `inference()`
-adds operations that perform inference, i.e. classification, on supplied images.
+adds operations that perform inference, i.e., classification, on supplied images.
1. [**Model training:**](#model-training) `loss()` and `train()`
add operations that compute the loss,
gradients, variable updates and visualization summaries.
@@ -405,7 +405,7 @@ a "tower". We must set two attributes for each tower:
* A unique name for all operations within a tower.
`tf.name_scope` provides
this unique name by prepending a scope. For instance, all operations in
-the first tower are prepended with `tower_0`, e.g. `tower_0/conv1/Conv2D`.
+the first tower are prepended with `tower_0`, e.g., `tower_0/conv1/Conv2D`.
* A preferred hardware device to run the operation within a tower.
`tf.device` specifies this. For
diff --git a/site/en/r1/tutorials/images/hub_with_keras.ipynb b/site/en/r1/tutorials/images/hub_with_keras.ipynb
index 5572fd237d9..f4e683e8936 100644
--- a/site/en/r1/tutorials/images/hub_with_keras.ipynb
+++ b/site/en/r1/tutorials/images/hub_with_keras.ipynb
@@ -841,7 +841,7 @@
"t = time.time()\n",
"\n",
"export_path = \"/tmp/saved_models/{}\".format(int(t))\n",
- "tf.keras.experimental.export_saved_model(model, export_path)\n",
+ "model.save(export_path)\n",
"\n",
"export_path"
]
@@ -863,7 +863,7 @@
},
"outputs": [],
"source": [
- "reloaded = tf.keras.experimental.load_from_saved_model(export_path, custom_objects={'KerasLayer':hub.KerasLayer})"
+ "reloaded = tf.keras.models.load_model(export_path, custom_objects={'KerasLayer':hub.KerasLayer})"
]
},
{
diff --git a/site/en/r1/tutorials/images/image_recognition.md b/site/en/r1/tutorials/images/image_recognition.md
index 0be884de403..cb66e594629 100644
--- a/site/en/r1/tutorials/images/image_recognition.md
+++ b/site/en/r1/tutorials/images/image_recognition.md
@@ -140,13 +140,13 @@ score of 0.8.
-Next, try it out on your own images by supplying the --image= argument, e.g.
+Next, try it out on your own images by supplying the --image= argument, e.g.,
```bash
bazel-bin/tensorflow/examples/label_image/label_image --image=my_image.png
```
-If you look inside the [`tensorflow/examples/label_image/main.cc`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/examples/label_image/main.cc)
+If you look inside the [`tensorflow/examples/label_image/main.cc`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/examples/label_image/main.cc)
file, you can find out
how it works. We hope this code will help you integrate TensorFlow into
your own applications, so we will walk step by step through the main functions:
@@ -164,7 +164,7 @@ training. If you have a graph that you've trained yourself, you'll just need
to adjust the values to match whatever you used during your training process.
You can see how they're applied to an image in the
-[`ReadTensorFromImageFile()`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/examples/label_image/main.cc#L88)
+[`ReadTensorFromImageFile()`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/examples/label_image/main.cc#L88)
function.
```C++
@@ -334,7 +334,7 @@ The `PrintTopLabels()` function takes those sorted results, and prints them out
friendly way. The `CheckTopLabel()` function is very similar, but just makes sure that
the top label is the one we expect, for debugging purposes.
-At the end, [`main()`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/examples/label_image/main.cc#L252)
+At the end, [`main()`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/examples/label_image/main.cc#L252)
ties together all of these calls.
```C++
diff --git a/site/en/r1/tutorials/images/transfer_learning.ipynb b/site/en/r1/tutorials/images/transfer_learning.ipynb
index 25779babd17..232ffa40bdd 100644
--- a/site/en/r1/tutorials/images/transfer_learning.ipynb
+++ b/site/en/r1/tutorials/images/transfer_learning.ipynb
@@ -148,7 +148,7 @@
},
"outputs": [],
"source": [
- "zip_file = tf.keras.utils.get_file(origin=\"https://storage.googleapis.com/mledu-datasets/cats_and_dogs_filtered.zip\",\n",
+ "zip_file = tf.keras.utils.get_file(origin=\"https://download.mlcc.google.com/mledu-datasets/cats_and_dogs_filtered.zip\",\n",
" fname=\"cats_and_dogs_filtered.zip\", extract=True)\n",
"base_dir, _ = os.path.splitext(zip_file)"
]
diff --git a/site/en/r1/tutorials/keras/save_and_restore_models.ipynb b/site/en/r1/tutorials/keras/save_and_restore_models.ipynb
index e9d112bd3f3..04cc94417a9 100644
--- a/site/en/r1/tutorials/keras/save_and_restore_models.ipynb
+++ b/site/en/r1/tutorials/keras/save_and_restore_models.ipynb
@@ -115,7 +115,7 @@
"\n",
"Sharing this data helps others understand how the model works and try it themselves with new data.\n",
"\n",
- "Caution: Be careful with untrusted code—TensorFlow models are code. See [Using TensorFlow Securely](https://github.com/tensorflow/tensorflow/blob/master/SECURITY.md) for details.\n",
+ "Caution: Be careful with untrusted code—TensorFlow models are code. See [Using TensorFlow Securely](https://github.com/tensorflow/tensorflow/blob/r1.15/SECURITY.md) for details.\n",
"\n",
"### Options\n",
"\n",
diff --git a/site/en/r1/tutorials/load_data/tf_records.ipynb b/site/en/r1/tutorials/load_data/tf_records.ipynb
index 64631cf0eab..45635034c69 100644
--- a/site/en/r1/tutorials/load_data/tf_records.ipynb
+++ b/site/en/r1/tutorials/load_data/tf_records.ipynb
@@ -141,7 +141,7 @@
"source": [
"Fundamentally a `tf.Example` is a `{\"string\": tf.train.Feature}` mapping.\n",
"\n",
- "The `tf.train.Feature` message type can accept one of the following three types (See the [`.proto` file](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/example/feature.proto) for reference). Most other generic types can be coerced into one of these.\n",
+ "The `tf.train.Feature` message type can accept one of the following three types (See the [`.proto` file](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/example/feature.proto) for reference). Most other generic types can be coerced into one of these.\n",
"\n",
"1. `tf.train.BytesList` (the following types can be coerced)\n",
"\n",
@@ -276,7 +276,7 @@
"\n",
"1. We create a map (dictionary) from the feature name string to the encoded feature value produced in #1.\n",
"\n",
- "1. The map produced in #2 is converted to a [`Features` message](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/example/feature.proto#L85)."
+ "1. The map produced in #2 is converted to a [`Features` message](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/example/feature.proto#L85)."
]
},
{
@@ -365,7 +365,7 @@
"id": "XftzX9CN_uGT"
},
"source": [
- "For example, suppose we have a single observation from the dataset, `[False, 4, bytes('goat'), 0.9876]`. We can create and print the `tf.Example` message for this observation using `create_message()`. Each single observation will be written as a `Features` message as per the above. Note that the `tf.Example` [message](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/example/example.proto#L88) is just a wrapper around the `Features` message."
+ "For example, suppose we have a single observation from the dataset, `[False, 4, bytes('goat'), 0.9876]`. We can create and print the `tf.Example` message for this observation using `create_message()`. Each single observation will be written as a `Features` message as per the above. Note that the `tf.Example` [message](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/example/example.proto#L88) is just a wrapper around the `Features` message."
]
},
{
@@ -632,7 +632,7 @@
"source": [
"We can also read the TFRecord file using the `tf.data.TFRecordDataset` class.\n",
"\n",
- "More information on consuming TFRecord files using `tf.data` can be found [here](https://www.tensorflow.org/r1/guide/datasets#consuming_tfrecord_data).\n",
+ "More information on consuming TFRecord files using `tf.data` can be found [here](https://www.tensorflow.org/guide/data#consuming_tfrecord_data).\n",
"\n",
"Using `TFRecordDataset`s can be useful for standardizing input data and optimizing performance."
]
diff --git a/site/en/r1/tutorials/representation/kernel_methods.md b/site/en/r1/tutorials/representation/kernel_methods.md
index 67adc4951c6..227fe81d515 100644
--- a/site/en/r1/tutorials/representation/kernel_methods.md
+++ b/site/en/r1/tutorials/representation/kernel_methods.md
@@ -24,7 +24,7 @@ following sources for an introduction:
Currently, TensorFlow supports explicit kernel mappings for dense features only;
TensorFlow will provide support for sparse features at a later release.
-This tutorial uses [tf.contrib.learn](https://www.tensorflow.org/code/tensorflow/contrib/learn/python/learn)
+This tutorial uses [tf.contrib.learn](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/contrib/learn/python/learn)
(TensorFlow's high-level Machine Learning API) Estimators for our ML models.
If you are not familiar with this API, The [Estimator guide](../../guide/estimators.md)
is a good place to start. We will use the MNIST dataset. The tutorial consists
@@ -131,7 +131,7 @@ In addition to experimenting with the (training) batch size and the number of
training steps, there are a couple other parameters that can be tuned as well.
For instance, you can change the optimization method used to minimize the loss
by explicitly selecting another optimizer from the collection of
-[available optimizers](https://www.tensorflow.org/code/tensorflow/python/training).
+[available optimizers](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/python/training).
As an example, the following code constructs a LinearClassifier estimator that
uses the Follow-The-Regularized-Leader (FTRL) optimization strategy with a
specific learning rate and L2-regularization.
diff --git a/site/en/r1/tutorials/representation/linear.md b/site/en/r1/tutorials/representation/linear.md
index 5516672b34a..d996a13bc1f 100644
--- a/site/en/r1/tutorials/representation/linear.md
+++ b/site/en/r1/tutorials/representation/linear.md
@@ -12,7 +12,7 @@ those tools. It explains:
Read this overview to decide whether the Estimator's linear model tools might
be useful to you. Then work through the
-[Estimator wide and deep learning tutorial](https://github.com/tensorflow/models/tree/master/official/r1/wide_deep)
+[Estimator wide and deep learning tutorial](https://github.com/tensorflow/models/tree/r1.15/official/r1/wide_deep)
to give it a try. This overview uses code samples from the tutorial, but the
tutorial walks through the code in greater detail.
@@ -177,7 +177,7 @@ the name of a `FeatureColumn`. Each key's value is a tensor containing the
values of that feature for all data instances. See
[Premade Estimators](../../guide/premade_estimators.md#input_fn) for a
more comprehensive look at input functions, and `input_fn` in the
-[wide and deep learning tutorial](https://github.com/tensorflow/models/tree/master/official/r1/wide_deep)
+[wide and deep learning tutorial](https://github.com/tensorflow/models/tree/r1.15/official/r1/wide_deep)
for an example implementation of an input function.
The input function is passed to the `train()` and `evaluate()` calls that
@@ -236,4 +236,4 @@ e = tf.estimator.DNNLinearCombinedClassifier(
dnn_hidden_units=[100, 50])
```
For more information, see the
-[wide and deep learning tutorial](https://github.com/tensorflow/models/tree/master/official/r1/wide_deep).
+[wide and deep learning tutorial](https://github.com/tensorflow/models/tree/r1.15/official/r1/wide_deep).
diff --git a/site/en/r1/tutorials/representation/unicode.ipynb b/site/en/r1/tutorials/representation/unicode.ipynb
index 98aaacff5b9..f76977c3c92 100644
--- a/site/en/r1/tutorials/representation/unicode.ipynb
+++ b/site/en/r1/tutorials/representation/unicode.ipynb
@@ -136,7 +136,7 @@
"id": "jsMPnjb6UDJ1"
},
"source": [
- "Note: When using python to construct strings, the handling of unicode differs betweeen v2 and v3. In v2, unicode strings are indicated by the \"u\" prefix, as above. In v3, strings are unicode-encoded by default."
+ "Note: When using python to construct strings, the handling of unicode differs between v2 and v3. In v2, unicode strings are indicated by the \"u\" prefix, as above. In v3, strings are unicode-encoded by default."
]
},
{
@@ -425,7 +425,7 @@
"source": [
"### Character substrings\n",
"\n",
- "Similarly, the `tf.strings.substr` operation accepts the \"`unit`\" parameter, and uses it to determine what kind of offsets the \"`pos`\" and \"`len`\" paremeters contain."
+ "Similarly, the `tf.strings.substr` operation accepts the \"`unit`\" parameter, and uses it to determine what kind of offsets the \"`pos`\" and \"`len`\" parameters contain."
]
},
{
@@ -587,7 +587,7 @@
"id": "CapnbShuGU8i"
},
"source": [
- "First, we decode the sentences into character codepoints, and find the script identifeir for each character."
+ "First, we decode the sentences into character codepoints, and find the script identifier for each character."
]
},
{
diff --git a/site/en/r1/tutorials/representation/word2vec.md b/site/en/r1/tutorials/representation/word2vec.md
index f6a27c68f3c..517a5dbc5c5 100644
--- a/site/en/r1/tutorials/representation/word2vec.md
+++ b/site/en/r1/tutorials/representation/word2vec.md
@@ -36,7 +36,7 @@ like to get your hands dirty with the details.
Image and audio processing systems work with rich, high-dimensional datasets
encoded as vectors of the individual raw pixel-intensities for image data, or
-e.g. power spectral density coefficients for audio data. For tasks like object
+e.g., power spectral density coefficients for audio data. For tasks like object
or speech recognition we know that all the information required to successfully
perform the task is encoded in the data (because humans can perform these tasks
from the raw data). However, natural language processing systems traditionally
@@ -109,7 +109,7 @@ $$
where \\(\text{score}(w_t, h)\\) computes the compatibility of word \\(w_t\\)
with the context \\(h\\) (a dot product is commonly used). We train this model
by maximizing its [log-likelihood](https://en.wikipedia.org/wiki/Likelihood_function)
-on the training set, i.e. by maximizing
+on the training set, i.e., by maximizing
$$
\begin{align}
@@ -176,7 +176,7 @@ As an example, let's consider the dataset
We first form a dataset of words and the contexts in which they appear. We
could define 'context' in any way that makes sense, and in fact people have
looked at syntactic contexts (i.e. the syntactic dependents of the current
-target word, see e.g.
+target word, see e.g.,
[Levy et al.](https://levyomer.files.wordpress.com/2014/04/dependency-based-word-embeddings-acl-2014.pdf)),
words-to-the-left of the target, words-to-the-right of the target, etc. For now,
let's stick to the vanilla definition and define 'context' as the window
@@ -204,7 +204,7 @@ where the goal is to predict `the` from `quick`. We select `num_noise` number
of noisy (contrastive) examples by drawing from some noise distribution,
typically the unigram distribution, \\(P(w)\\). For simplicity let's say
`num_noise=1` and we select `sheep` as a noisy example. Next we compute the
-loss for this pair of observed and noisy examples, i.e. the objective at time
+loss for this pair of observed and noisy examples, i.e., the objective at time
step \\(t\\) becomes
$$J^{(t)}_\text{NEG} = \log Q_\theta(D=1 | \text{the, quick}) +
@@ -212,7 +212,7 @@ $$J^{(t)}_\text{NEG} = \log Q_\theta(D=1 | \text{the, quick}) +
The goal is to make an update to the embedding parameters \\(\theta\\) to improve
(in this case, maximize) this objective function. We do this by deriving the
-gradient of the loss with respect to the embedding parameters \\(\theta\\), i.e.
+gradient of the loss with respect to the embedding parameters \\(\theta\\), i.e.,
\\(\frac{\partial}{\partial \theta} J_\text{NEG}\\) (luckily TensorFlow provides
easy helper functions for doing this!). We then perform an update to the
embeddings by taking a small step in the direction of the gradient. When this
@@ -227,7 +227,7 @@ When we inspect these visualizations it becomes apparent that the vectors
capture some general, and in fact quite useful, semantic information about
words and their relationships to one another. It was very interesting when we
first discovered that certain directions in the induced vector space specialize
-towards certain semantic relationships, e.g. *male-female*, *verb tense* and
+towards certain semantic relationships, e.g., *male-female*, *verb tense* and
even *country-capital* relationships between words, as illustrated in the figure
below (see also for example
[Mikolov et al., 2013](https://www.aclweb.org/anthology/N13-1090)).
@@ -327,7 +327,7 @@ for inputs, labels in generate_batch(...):
```
See the full example code in
-[tensorflow/examples/tutorials/word2vec/word2vec_basic.py](https://www.tensorflow.org/code/tensorflow/examples/tutorials/word2vec/word2vec_basic.py).
+[tensorflow/examples/tutorials/word2vec/word2vec_basic.py](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/examples/tutorials/word2vec/word2vec_basic.py).
## Visualizing the learned embeddings
@@ -341,7 +341,7 @@ t-SNE.
Et voila! As expected, words that are similar end up clustering nearby each
other. For a more heavyweight implementation of word2vec that showcases more of
the advanced features of TensorFlow, see the implementation in
-[models/tutorials/embedding/word2vec.py](https://github.com/tensorflow/models/tree/master/research/tutorials/embedding/word2vec.py).
+[models/tutorials/embedding/word2vec.py](https://github.com/tensorflow/models/tree/r1.15/research/tutorials/embedding/word2vec.py).
## Evaluating embeddings: analogical reasoning
@@ -357,7 +357,7 @@ Download the dataset for this task from
To see how we do this evaluation, have a look at the `build_eval_graph()` and
`eval()` functions in
-[models/tutorials/embedding/word2vec.py](https://github.com/tensorflow/models/tree/master/research/tutorials/embedding/word2vec.py).
+[models/tutorials/embedding/word2vec.py](https://github.com/tensorflow/models/tree/r1.15/research/tutorials/embedding/word2vec.py).
The choice of hyperparameters can strongly influence the accuracy on this task.
To achieve state-of-the-art performance on this task requires training over a
diff --git a/site/en/r1/tutorials/sequences/audio_recognition.md b/site/en/r1/tutorials/sequences/audio_recognition.md
index 8ad71b88a3c..0388514ec92 100644
--- a/site/en/r1/tutorials/sequences/audio_recognition.md
+++ b/site/en/r1/tutorials/sequences/audio_recognition.md
@@ -159,9 +159,9 @@ accuracy. If the training accuracy increases but the validation doesn't, that's
a sign that overfitting is occurring, and your model is only learning things
about the training clips, not broader patterns that generalize.
-## Tensorboard
+## TensorBoard
-A good way to visualize how the training is progressing is using Tensorboard. By
+A good way to visualize how the training is progressing is using TensorBoard. By
default, the script saves out events to /tmp/retrain_logs, and you can load
these by running:
diff --git a/site/en/r1/tutorials/sequences/recurrent.md b/site/en/r1/tutorials/sequences/recurrent.md
index 6654795d944..e7c1f8c0b16 100644
--- a/site/en/r1/tutorials/sequences/recurrent.md
+++ b/site/en/r1/tutorials/sequences/recurrent.md
@@ -2,7 +2,7 @@
## Introduction
-See [Understanding LSTM Networks](https://colah.github.io/posts/2015-08-Understanding-LSTMs/){:.external}
+See [Understanding LSTM Networks](https://colah.github.io/posts/2015-08-Understanding-LSTMs/)
for an introduction to recurrent neural networks and LSTMs.
## Language Modeling
diff --git a/site/en/r1/tutorials/sequences/recurrent_quickdraw.md b/site/en/r1/tutorials/sequences/recurrent_quickdraw.md
index 435076f629c..d6a85377d17 100644
--- a/site/en/r1/tutorials/sequences/recurrent_quickdraw.md
+++ b/site/en/r1/tutorials/sequences/recurrent_quickdraw.md
@@ -109,7 +109,7 @@ This download will take a while and download a bit more than 23GB of data.
To convert the `ndjson` files to
[TFRecord](../../api_guides/python/python_io.md#TFRecords_Format_Details) files containing
-[`tf.train.Example`](https://www.tensorflow.org/code/tensorflow/core/example/example.proto)
+[`tf.train.Example`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/example/example.proto)
protos run the following command.
```shell
@@ -213,7 +213,7 @@ screen coordinates and normalize the size such that the drawing has unit height.
Finally, we compute the differences between consecutive points and store these
as a `VarLenFeature` in a
-[tensorflow.Example](https://www.tensorflow.org/code/tensorflow/core/example/example.proto)
+[tensorflow.Example](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/example/example.proto)
under the key `ink`. In addition we store the `class_index` as a single entry
`FixedLengthFeature` and the `shape` of the `ink` as a `FixedLengthFeature` of
length 2.
diff --git a/site/en/r1/tutorials/sequences/text_generation.ipynb b/site/en/r1/tutorials/sequences/text_generation.ipynb
index 1350df2a486..84d942c8bd0 100644
--- a/site/en/r1/tutorials/sequences/text_generation.ipynb
+++ b/site/en/r1/tutorials/sequences/text_generation.ipynb
@@ -77,9 +77,9 @@
"id": "BwpJ5IffzRG6"
},
"source": [
- "This tutorial demonstrates how to generate text using a character-based RNN. We will work with a dataset of Shakespeare's writing from Andrej Karpathy's [The Unreasonable Effectiveness of Recurrent Neural Networks](http://karpathy.github.io/2015/05/21/rnn-effectiveness/). Given a sequence of characters from this data (\"Shakespear\"), train a model to predict the next character in the sequence (\"e\"). Longer sequences of text can be generated by calling the model repeatedly.\n",
+ "This tutorial demonstrates how to generate text using a character-based RNN. You will work with a dataset of Shakespeare's writing from Andrej Karpathy's [The Unreasonable Effectiveness of Recurrent Neural Networks](http://karpathy.github.io/2015/05/21/rnn-effectiveness/). Given a sequence of characters from this data (\"Shakespear\"), train a model to predict the next character in the sequence (\"e\"). Longer sequences of text can be generated by calling the model repeatedly.\n",
"\n",
- "Note: Enable GPU acceleration to execute this notebook faster. In Colab: *Runtime > Change runtime type > Hardware acclerator > GPU*. If running locally make sure TensorFlow version >= 1.11.\n",
+ "Note: Enable GPU acceleration to execute this notebook faster. In Colab: *Runtime > Change runtime type > Hardware accelerator > GPU*. If running locally make sure TensorFlow version >= 1.11.\n",
"\n",
"This tutorial includes runnable code implemented using [tf.keras](https://www.tensorflow.org/programmers_guide/keras) and [eager execution](https://www.tensorflow.org/programmers_guide/eager). The following is sample output when the model in this tutorial trained for 30 epochs, and started with the string \"Q\":\n",
"\n",
@@ -98,7 +98,7 @@
"To watch the next way with his father with his face?\n",
"\n",
"ESCALUS:\n",
- "The cause why then we are all resolved more sons.\n",
+ "The cause why then us all resolved more sons.\n",
"\n",
"VOLUMNIA:\n",
"O, no, no, no, no, no, no, no, no, no, no, no, no, no, no, no, no, no, no, no, no, it is no sin it should be dead,\n",
@@ -248,7 +248,7 @@
"source": [
"### Vectorize the text\n",
"\n",
- "Before training, we need to map strings to a numerical representation. Create two lookup tables: one mapping characters to numbers, and another for numbers to characters."
+ "Before training, you need to map strings to a numerical representation. Create two lookup tables: one mapping characters to numbers, and another for numbers to characters."
]
},
{
@@ -272,7 +272,7 @@
"id": "tZfqhkYCymwX"
},
"source": [
- "Now we have an integer representation for each character. Notice that we mapped the character as indexes from 0 to `len(unique)`."
+ "Now you have an integer representation for each character. Notice that you mapped the character as indexes from 0 to `len(unique)`."
]
},
{
@@ -316,7 +316,7 @@
"id": "wssHQ1oGymwe"
},
"source": [
- "Given a character, or a sequence of characters, what is the most probable next character? This is the task we're training the model to perform. The input to the model will be a sequence of characters, and we train the model to predict the output—the following character at each time step.\n",
+ "Given a character, or a sequence of characters, what is the most probable next character? This is the task you are training the model to perform. The input to the model will be a sequence of characters, and you train the model to predict the output—the following character at each time step.\n",
"\n",
"Since RNNs maintain an internal state that depends on the previously seen elements, given all the characters computed until this moment, what is the next character?\n"
]
@@ -346,7 +346,7 @@
},
"outputs": [],
"source": [
- "# The maximum length sentence we want for a single input in characters\n",
+ "# The maximum length sentence you want for a single input in characters\n",
"seq_length = 100\n",
"examples_per_epoch = len(text)//seq_length\n",
"\n",
@@ -458,7 +458,7 @@
"source": [
"### Create training batches\n",
"\n",
- "We used `tf.data` to split the text into manageable sequences. But before feeding this data into the model, we need to shuffle the data and pack it into batches."
+ "You used `tf.data` to split the text into manageable sequences. But before feeding this data into the model, you need to shuffle the data and pack it into batches."
]
},
{
@@ -650,7 +650,7 @@
"id": "uwv0gEkURfx1"
},
"source": [
- "To get actual predictions from the model we need to sample from the output distribution, to get actual character indices. This distribution is defined by the logits over the character vocabulary.\n",
+ "To get actual predictions from the model you need to sample from the output distribution, to get actual character indices. This distribution is defined by the logits over the character vocabulary.\n",
"\n",
"Note: It is important to _sample_ from this distribution as taking the _argmax_ of the distribution can easily get the model stuck in a loop.\n",
"\n",
@@ -746,7 +746,7 @@
"source": [
"The standard `tf.keras.losses.sparse_categorical_crossentropy` loss function works in this case because it is applied across the last dimension of the predictions.\n",
"\n",
- "Because our model returns logits, we need to set the `from_logits` flag.\n"
+ "Because our model returns logits, you need to set the `from_logits` flag.\n"
]
},
{
@@ -771,7 +771,7 @@
"id": "jeOXriLcymww"
},
"source": [
- "Configure the training procedure using the `tf.keras.Model.compile` method. We'll use `tf.train.AdamOptimizer` with default arguments and the loss function."
+ "Configure the training procedure using the `tf.keras.Model.compile` method. You'll use `tf.train.AdamOptimizer` with default arguments and the loss function."
]
},
{
@@ -891,7 +891,7 @@
"\n",
"Because of the way the RNN state is passed from timestep to timestep, the model only accepts a fixed batch size once built.\n",
"\n",
- "To run the model with a different `batch_size`, we need to rebuild the model and restore the weights from the checkpoint.\n"
+ "To run the model with a different `batch_size`, you need to rebuild the model and restore the weights from the checkpoint.\n"
]
},
{
@@ -992,7 +992,7 @@
" predictions = predictions / temperature\n",
" predicted_id = tf.multinomial(predictions, num_samples=1)[-1,0].numpy()\n",
"\n",
- " # We pass the predicted word as the next input to the model\n",
+ " # You pass the predicted word as the next input to the model\n",
" # along with the previous hidden state\n",
" input_eval = tf.expand_dims([predicted_id], 0)\n",
"\n",
@@ -1035,11 +1035,11 @@
"\n",
"So now that you've seen how to run the model manually let's unpack the training loop, and implement it ourselves. This gives a starting point, for example, to implement _curriculum learning_ to help stabilize the model's open-loop output.\n",
"\n",
- "We will use `tf.GradientTape` to track the gradients. You can learn more about this approach by reading the [eager execution guide](https://www.tensorflow.org/r1/guide/eager).\n",
+ "You will use `tf.GradientTape` to track the gradients. You can learn more about this approach by reading the [eager execution guide](https://www.tensorflow.org/r1/guide/eager).\n",
"\n",
"The procedure works as follows:\n",
"\n",
- "* First, initialize the RNN state. We do this by calling the `tf.keras.Model.reset_states` method.\n",
+ "* First, initialize the RNN state. You do this by calling the `tf.keras.Model.reset_states` method.\n",
"\n",
"* Next, iterate over the dataset (batch by batch) and calculate the *predictions* associated with each.\n",
"\n",
diff --git a/site/en/tutorials/_index.yaml b/site/en/tutorials/_index.yaml
index c38a23ddb35..0d09f04c5c7 100644
--- a/site/en/tutorials/_index.yaml
+++ b/site/en/tutorials/_index.yaml
@@ -16,8 +16,9 @@ landing_page:
- description: >
The TensorFlow tutorials are written as Jupyter notebooks and run
- directly in Google Colab—a hosted notebook environment that requires
- no setup. Click the Run in Google Colab button.
+ directly in Google Colab—a hosted notebook environment that requires
+ no setup. At the top of each tutorial, you'll see a Run in Google Colab button. Click
+ the button to open the notebook and run the code yourself.
path: /xla
icon:
diff --git a/site/en/tutorials/_toc.yaml b/site/en/tutorials/_toc.yaml
index 7c629d69dea..a3907ffe9a4 100644
--- a/site/en/tutorials/_toc.yaml
+++ b/site/en/tutorials/_toc.yaml
@@ -35,6 +35,9 @@ toc:
section:
- title: "Images"
path: /tutorials/load_data/images
+ - title: "Video"
+ path: /tutorials/load_data/video
+ status: new
- title: "CSV"
path: /tutorials/load_data/csv
- title: "NumPy"
@@ -94,9 +97,14 @@ toc:
- title: "Distributed input"
path: /tutorials/distribute/input
-- title: "Images"
+- title: "Vision"
style: accordion
section:
+ - title: "Computer vision"
+ path: /tutorials/images
+ - title: "KerasCV"
+ path: https://keras.io/keras_cv/
+ status: external
- title: "Convolutional Neural Network"
path: /tutorials/images/cnn
- title: "Image classification"
@@ -112,29 +120,25 @@ toc:
- title: "Object detection with TF Hub"
path: /hub/tutorials/tf2_object_detection
status: external
+ - title: "Video classification"
+ status: new
+ path: /tutorials/video/video_classification
+ - title: "Transfer learning with MoViNet"
+ status: new
+ path: /tutorials/video/transfer_learning_with_movinet
- title: "Text"
style: accordion
section:
- - title: "Word embeddings"
- path: /text/guide/word_embeddings
- status: external
- - title: "Word2Vec"
- path: /tutorials/text/word2vec
- - title: "Text classification with an RNN"
- path: /text/tutorials/text_classification_rnn
+ - title: "Text and natural language processing"
+ path: /tutorials/text/index
+ - title: "Get started with KerasNLP"
+ path: https://keras.io/guides/keras_nlp/getting_started/
status: external
- - title: "Classify Text with BERT"
- path: /text/tutorials/classify_text_with_bert
+ - title: "Text and NLP guide"
+ path: /text
status: external
- - title: "Solve GLUE tasks using BERT on TPU"
- path: /text/tutorials/bert_glue
- status: external
- - title: "Neural machine translation with attention"
- path: /text/tutorials/nmt_with_attention
- status: external
- - title: "Image captioning"
- path: /tutorials/text/image_captioning
+
- title: "Audio"
style: accordion
section:
@@ -142,10 +146,8 @@ toc:
path: /tutorials/audio/simple_audio
- title: "Transfer learning for audio recognition"
path: /tutorials/audio/transfer_learning_audio
- status: new
- title: "Generate music with an RNN"
path: /tutorials/audio/music_generation
- status: new
- title: "Structured data"
style: accordion
@@ -166,6 +168,9 @@ toc:
- title: "Generative"
style: accordion
section:
+ - title: "Stable Diffusion"
+ status: new
+ path: /tutorials/generative/generate_images_with_stable_diffusion
- title: "Neural style transfer"
path: /tutorials/generative/style_transfer
- title: "DeepDream"
@@ -184,7 +189,15 @@ toc:
path: /tutorials/generative/cvae
- title: "Lossy data compression"
path: /tutorials/generative/data_compression
- status: new
+
+- title: "Model optimization"
+ style: accordion
+ section:
+ - title: "Scalable model compression with EPR"
+ path: /tutorials/optimization/compression
+ - title: "TensorFlow model optimization"
+ status: external
+ path: /model_optimization
- title: "Model Understanding"
style: accordion
@@ -196,6 +209,7 @@ toc:
- title: "Probabilistic regression"
path: /probability/examples/Probabilistic_Layers_Regression
status: external
+
- title: "Reinforcement learning"
style: accordion
section:
diff --git a/site/en/tutorials/audio/music_generation.ipynb b/site/en/tutorials/audio/music_generation.ipynb
index 1eee7809ca7..e1423ef7cf2 100644
--- a/site/en/tutorials/audio/music_generation.ipynb
+++ b/site/en/tutorials/audio/music_generation.ipynb
@@ -145,7 +145,7 @@
"\n",
"from IPython import display\n",
"from matplotlib import pyplot as plt\n",
- "from typing import Dict, List, Optional, Sequence, Tuple"
+ "from typing import Optional"
]
},
{
@@ -1072,9 +1072,9 @@
"source": [
"def predict_next_note(\n",
" notes: np.ndarray, \n",
- " keras_model: tf.keras.Model, \n",
- " temperature: float = 1.0) -> int:\n",
- " \"\"\"Generates a note IDs using a trained sequence model.\"\"\"\n",
+ " model: tf.keras.Model, \n",
+ " temperature: float = 1.0) -> tuple[int, float, float]:\n",
+ " \"\"\"Generates a note as a tuple of (pitch, step, duration), using a trained sequence model.\"\"\"\n",
"\n",
" assert temperature > 0\n",
"\n",
diff --git a/site/en/tutorials/audio/simple_audio.ipynb b/site/en/tutorials/audio/simple_audio.ipynb
index a4d4c2fa634..9d79742fbb7 100644
--- a/site/en/tutorials/audio/simple_audio.ipynb
+++ b/site/en/tutorials/audio/simple_audio.ipynb
@@ -74,9 +74,9 @@
"id": "SPfDNFlb66XF"
},
"source": [
- "This tutorial demonstrates how to preprocess audio files in the WAV format and build and train a basic automatic speech recognition (ASR) model for recognizing ten different words. You will use a portion of the [Speech Commands dataset](https://www.tensorflow.org/datasets/catalog/speech_commands) (Warden, 2018), which contains short (one-second or less) audio clips of commands, such as \"down\", \"go\", \"left\", \"no\", \"right\", \"stop\", \"up\" and \"yes\".\n",
+ "This tutorial demonstrates how to preprocess audio files in the WAV format and build and train a basic [automatic speech recognition](https://en.wikipedia.org/wiki/Speech_recognition) (ASR) model for recognizing ten different words. You will use a portion of the [Speech Commands dataset](https://www.tensorflow.org/datasets/catalog/speech_commands) ([Warden, 2018](https://arxiv.org/abs/1804.03209)), which contains short (one-second or less) audio clips of commands, such as \"down\", \"go\", \"left\", \"no\", \"right\", \"stop\", \"up\" and \"yes\".\n",
"\n",
- "Real-world speech and audio recognition systems are complex. But, like [image classification with the MNIST dataset](../quickstart/beginner.ipynb), this tutorial should give you a basic understanding of the techniques involved."
+ "Real-world speech and audio recognition [systems](https://ai.googleblog.com/search/label/Speech%20Recognition) are complex. But, like [image classification with the MNIST dataset](../quickstart/beginner.ipynb), this tutorial should give you a basic understanding of the techniques involved."
]
},
{
@@ -87,7 +87,18 @@
"source": [
"## Setup\n",
"\n",
- "Import necessary modules and dependencies. Note that you'll be using seaborn for visualization in this tutorial."
+ "Import necessary modules and dependencies. You'll be using `tf.keras.utils.audio_dataset_from_directory` (introduced in TensorFlow 2.10), which helps generate audio classification datasets from directories of `.wav` files. You'll also need [seaborn](https://seaborn.pydata.org) for visualization in this tutorial."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "hhNW45sjDEDe"
+ },
+ "outputs": [],
+ "source": [
+ "!pip install -U -q tensorflow tensorflow_datasets"
]
},
{
@@ -124,7 +135,7 @@
"source": [
"## Import the mini Speech Commands dataset\n",
"\n",
- "To save time with data loading, you will be working with a smaller version of the Speech Commands dataset. The [original dataset](https://www.tensorflow.org/datasets/catalog/speech_commands) consists of over 105,000 audio files in the WAV (Waveform) audio file format of people saying 35 different words. This data was collected by Google and released under a CC BY license.\n",
+ "To save time with data loading, you will be working with a smaller version of the Speech Commands dataset. The [original dataset](https://www.tensorflow.org/datasets/catalog/speech_commands) consists of over 105,000 audio files in the [WAV (Waveform) audio file format](https://www.aelius.com/njh/wavemetatools/doc/riffmci.pdf) of people saying 35 different words. This data was collected by Google and released under a CC BY license.\n",
"\n",
"Download and extract the `mini_speech_commands.zip` file containing the smaller Speech Commands datasets with `tf.keras.utils.get_file`:"
]
@@ -166,218 +177,140 @@
"outputs": [],
"source": [
"commands = np.array(tf.io.gfile.listdir(str(data_dir)))\n",
- "commands = commands[commands != 'README.md']\n",
+ "commands = commands[(commands != 'README.md') & (commands != '.DS_Store')]\n",
"print('Commands:', commands)"
]
},
{
"cell_type": "markdown",
"metadata": {
- "id": "aMvdU9SY8WXN"
- },
- "source": [
- "Extract the audio clips into a list called `filenames`, and shuffle it:"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "id": "hlX685l1wD9k"
- },
- "outputs": [],
- "source": [
- "filenames = tf.io.gfile.glob(str(data_dir) + '/*/*')\n",
- "filenames = tf.random.shuffle(filenames)\n",
- "num_samples = len(filenames)\n",
- "print('Number of total examples:', num_samples)\n",
- "print('Number of examples per label:',\n",
- " len(tf.io.gfile.listdir(str(data_dir/commands[0]))))\n",
- "print('Example file tensor:', filenames[0])"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {
- "id": "9vK3ymy23MCP"
+ "id": "TZ7GJjDvHqtt"
},
"source": [
- "Split `filenames` into training, validation and test sets using a 80:10:10 ratio, respectively:"
+ "Divided into directories this way, you can easily load the data using `keras.utils.audio_dataset_from_directory`. \n",
+ "\n",
+ "The audio clips are 1 second or less at 16kHz. The `output_sequence_length=16000` pads the short ones to exactly 1 second (and would trim longer ones) so that they can be easily batched."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
- "id": "Cv_wts-l3KgD"
+ "id": "mFM4c3aMC8Qv"
},
"outputs": [],
"source": [
- "train_files = filenames[:6400]\n",
- "val_files = filenames[6400: 6400 + 800]\n",
- "test_files = filenames[-800:]\n",
+ "train_ds, val_ds = tf.keras.utils.audio_dataset_from_directory(\n",
+ " directory=data_dir,\n",
+ " batch_size=64,\n",
+ " validation_split=0.2,\n",
+ " seed=0,\n",
+ " output_sequence_length=16000,\n",
+ " subset='both')\n",
"\n",
- "print('Training set size', len(train_files))\n",
- "print('Validation set size', len(val_files))\n",
- "print('Test set size', len(test_files))"
+ "label_names = np.array(train_ds.class_names)\n",
+ "print()\n",
+ "print(\"label names:\", label_names)"
]
},
{
"cell_type": "markdown",
"metadata": {
- "id": "g2Cj9FyvfweD"
+ "id": "cestp83qFnU5"
},
"source": [
- "## Read the audio files and their labels"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {
- "id": "j1zjcWteOcBy"
- },
- "source": [
- "In this section you will preprocess the dataset, creating decoded tensors for the waveforms and the corresponding labels. Note that:\n",
- "\n",
- "- Each WAV file contains time-series data with a set number of samples per second.\n",
- "- Each sample represents the amplitude of the audio signal at that specific time.\n",
- "- In a 16-bit system, like the WAV files in the mini Speech Commands dataset, the amplitude values range from -32,768 to 32,767.\n",
- "- The sample rate for this dataset is 16kHz.\n",
- "\n",
- "The shape of the tensor returned by `tf.audio.decode_wav` is `[samples, channels]`, where `channels` is `1` for mono or `2` for stereo. The mini Speech Commands dataset only contains mono recordings. "
+ "The dataset now contains batches of audio clips and integer labels. The audio clips have a shape of `(batch, samples, channels)`. "
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
- "id": "d16bb8416f90"
+ "id": "3yU6SQGIFb3H"
},
"outputs": [],
"source": [
- "test_file = tf.io.read_file(DATASET_PATH+'/down/0a9f9af7_nohash_0.wav')\n",
- "test_audio, _ = tf.audio.decode_wav(contents=test_file)\n",
- "test_audio.shape"
+ "train_ds.element_spec"
]
},
{
"cell_type": "markdown",
"metadata": {
- "id": "e6bb8defd2ef"
+ "id": "ppG9Dgq2Ex8R"
},
"source": [
- "Now, let's define a function that preprocesses the dataset's raw WAV audio files into audio tensors:"
+ "This dataset only contains single channel audio, so use the `tf.squeeze` function to drop the extra axis:"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
- "id": "9PjJ2iXYwftD"
+ "id": "Xl-tnniUIBlM"
},
"outputs": [],
"source": [
- "def decode_audio(audio_binary):\n",
- " # Decode WAV-encoded audio files to `float32` tensors, normalized\n",
- " # to the [-1.0, 1.0] range. Return `float32` audio and a sample rate.\n",
- " audio, _ = tf.audio.decode_wav(contents=audio_binary)\n",
- " # Since all the data is single channel (mono), drop the `channels`\n",
- " # axis from the array.\n",
- " return tf.squeeze(audio, axis=-1)"
+ "def squeeze(audio, labels):\n",
+ " audio = tf.squeeze(audio, axis=-1)\n",
+ " return audio, labels\n",
+ "\n",
+ "train_ds = train_ds.map(squeeze, tf.data.AUTOTUNE)\n",
+ "val_ds = val_ds.map(squeeze, tf.data.AUTOTUNE)"
]
},
{
"cell_type": "markdown",
"metadata": {
- "id": "GPQseZElOjVN"
+ "id": "DtsCSWZN5ILv"
},
"source": [
- "Define a function that creates labels using the parent directories for each file:\n",
- "\n",
- "- Split the file paths into `tf.RaggedTensor`s (tensors with ragged dimensions—with slices that may have different lengths)."
+ "The `utils.audio_dataset_from_directory` function only returns up to two splits. It's a good idea to keep a test set separate from your validation set.\n",
+ "Ideally you'd keep it in a separate directory, but in this case you can use `Dataset.shard` to split the validation set into two halves. Note that iterating over **any** shard will load **all** the data, and only keep its fraction. "
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
- "id": "8VTtX1nr3YT-"
+ "id": "u5UEGsqM5Gss"
},
"outputs": [],
"source": [
- "def get_label(file_path):\n",
- " parts = tf.strings.split(\n",
- " input=file_path,\n",
- " sep=os.path.sep)\n",
- " # Note: You'll use indexing here instead of tuple unpacking to enable this\n",
- " # to work in a TensorFlow graph.\n",
- " return parts[-2]"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {
- "id": "E8Y9w_5MOsr-"
- },
- "source": [
- "Define another helper function—`get_waveform_and_label`—that puts it all together:\n",
- "\n",
- "- The input is the WAV audio filename.\n",
- "- The output is a tuple containing the audio and label tensors ready for supervised learning."
+ "test_ds = val_ds.shard(num_shards=2, index=0)\n",
+ "val_ds = val_ds.shard(num_shards=2, index=1)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
- "id": "WdgUD5T93NyT"
+ "id": "xIeoJcwJH5h9"
},
"outputs": [],
"source": [
- "def get_waveform_and_label(file_path):\n",
- " label = get_label(file_path)\n",
- " audio_binary = tf.io.read_file(file_path)\n",
- " waveform = decode_audio(audio_binary)\n",
- " return waveform, label"
+ "for example_audio, example_labels in train_ds.take(1): \n",
+ " print(example_audio.shape)\n",
+ " print(example_labels.shape)"
]
},
{
"cell_type": "markdown",
"metadata": {
- "id": "nvN8W_dDjYjc"
+ "id": "voxGEwvuh2L7"
},
"source": [
- "Build the training set to extract the audio-label pairs:\n",
- "\n",
- "- Create a `tf.data.Dataset` with `Dataset.from_tensor_slices` and `Dataset.map`, using `get_waveform_and_label` defined earlier.\n",
- "\n",
- "You'll build the validation and test sets using a similar procedure later on."
+ "Let's plot a few audio waveforms:"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
- "id": "0SQl8yXl3kNP"
+ "id": "dYtGq2zYNHuT"
},
"outputs": [],
"source": [
- "AUTOTUNE = tf.data.AUTOTUNE\n",
- "\n",
- "files_ds = tf.data.Dataset.from_tensor_slices(train_files)\n",
- "\n",
- "waveform_ds = files_ds.map(\n",
- " map_func=get_waveform_and_label,\n",
- " num_parallel_calls=AUTOTUNE)"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {
- "id": "voxGEwvuh2L7"
- },
- "source": [
- "Let's plot a few audio waveforms:"
+ "label_names[[1,1,3,0]]"
]
},
{
@@ -388,21 +321,17 @@
},
"outputs": [],
"source": [
+ "plt.figure(figsize=(16, 10))\n",
"rows = 3\n",
"cols = 3\n",
"n = rows * cols\n",
- "fig, axes = plt.subplots(rows, cols, figsize=(10, 12))\n",
- "\n",
- "for i, (audio, label) in enumerate(waveform_ds.take(n)):\n",
- " r = i // cols\n",
- " c = i % cols\n",
- " ax = axes[r][c]\n",
- " ax.plot(audio.numpy())\n",
- " ax.set_yticks(np.arange(-1.2, 1.2, 0.2))\n",
- " label = label.numpy().decode('utf-8')\n",
- " ax.set_title(label)\n",
- "\n",
- "plt.show()"
+ "for i in range(n):\n",
+ " plt.subplot(rows, cols, i+1)\n",
+ " audio_signal = example_audio[i]\n",
+ " plt.plot(audio_signal)\n",
+ " plt.title(label_names[example_labels[i]])\n",
+ " plt.yticks(np.arange(-1.2, 1.2, 0.2))\n",
+ " plt.ylim([-1.1, 1.1])"
]
},
{
@@ -413,14 +342,14 @@
"source": [
"## Convert waveforms to spectrograms\n",
"\n",
- "The waveforms in the dataset are represented in the time domain. Next, you'll transform the waveforms from the time-domain signals into the time-frequency-domain signals by computing the short-time Fourier transform (STFT) to convert the waveforms to as spectrograms, which show frequency changes over time and can be represented as 2D images. You will feed the spectrogram images into your neural network to train the model.\n",
+ "The waveforms in the dataset are represented in the time domain. Next, you'll transform the waveforms from the time-domain signals into the time-frequency-domain signals by computing the [short-time Fourier transform (STFT)](https://en.wikipedia.org/wiki/Short-time_Fourier_transform) to convert the waveforms to as [spectrograms](https://en.wikipedia.org/wiki/Spectrogram), which show frequency changes over time and can be represented as 2D images. You will feed the spectrogram images into your neural network to train the model.\n",
"\n",
"A Fourier transform (`tf.signal.fft`) converts a signal to its component frequencies, but loses all time information. In comparison, STFT (`tf.signal.stft`) splits the signal into windows of time and runs a Fourier transform on each window, preserving some time information, and returning a 2D tensor that you can run standard convolutions on.\n",
"\n",
"Create a utility function for converting waveforms to spectrograms:\n",
"\n",
"- The waveforms need to be of the same length, so that when you convert them to spectrograms, the results have similar dimensions. This can be done by simply zero-padding the audio clips that are shorter than one second (using `tf.zeros`).\n",
- "- When calling `tf.signal.stft`, choose the `frame_length` and `frame_step` parameters such that the generated spectrogram \"image\" is almost square. For more information on the STFT parameters choice, refer to this Coursera video on audio signal processing and STFT.\n",
+ "- When calling `tf.signal.stft`, choose the `frame_length` and `frame_step` parameters such that the generated spectrogram \"image\" is almost square. For more information on the STFT parameters choice, refer to [this Coursera video](https://www.coursera.org/lecture/audio-signal-processing/stft-2-tjEQe) on audio signal processing and STFT.\n",
"- The STFT produces an array of complex numbers representing magnitude and phase. However, in this tutorial you'll only use the magnitude, which you can derive by applying `tf.abs` on the output of `tf.signal.stft`."
]
},
@@ -433,20 +362,9 @@
"outputs": [],
"source": [
"def get_spectrogram(waveform):\n",
- " # Zero-padding for an audio waveform with less than 16,000 samples.\n",
- " input_len = 16000\n",
- " waveform = waveform[:input_len]\n",
- " zero_padding = tf.zeros(\n",
- " [16000] - tf.shape(waveform),\n",
- " dtype=tf.float32)\n",
- " # Cast the waveform tensors' dtype to float32.\n",
- " waveform = tf.cast(waveform, dtype=tf.float32)\n",
- " # Concatenate the waveform with `zero_padding`, which ensures all audio\n",
- " # clips are of the same length.\n",
- " equal_length = tf.concat([waveform, zero_padding], 0)\n",
" # Convert the waveform to a spectrogram via a STFT.\n",
" spectrogram = tf.signal.stft(\n",
- " equal_length, frame_length=255, frame_step=128)\n",
+ " waveform, frame_length=255, frame_step=128)\n",
" # Obtain the magnitude of the STFT.\n",
" spectrogram = tf.abs(spectrogram)\n",
" # Add a `channels` dimension, so that the spectrogram can be used\n",
@@ -473,15 +391,16 @@
},
"outputs": [],
"source": [
- "for waveform, label in waveform_ds.take(1):\n",
- " label = label.numpy().decode('utf-8')\n",
+ "for i in range(3):\n",
+ " label = label_names[example_labels[i]]\n",
+ " waveform = example_audio[i]\n",
" spectrogram = get_spectrogram(waveform)\n",
"\n",
- "print('Label:', label)\n",
- "print('Waveform shape:', waveform.shape)\n",
- "print('Spectrogram shape:', spectrogram.shape)\n",
- "print('Audio playback')\n",
- "display.display(display.Audio(waveform, rate=16000))"
+ " print('Label:', label)\n",
+ " print('Waveform shape:', waveform.shape)\n",
+ " print('Spectrogram shape:', spectrogram.shape)\n",
+ " print('Audio playback')\n",
+ " display.display(display.Audio(waveform, rate=16000))"
]
},
{
@@ -541,6 +460,7 @@
"\n",
"plot_spectrogram(spectrogram.numpy(), axes[1])\n",
"axes[1].set_title('Spectrogram')\n",
+ "plt.suptitle(label.title())\n",
"plt.show()"
]
},
@@ -550,30 +470,21 @@
"id": "GyYXjW07jCHA"
},
"source": [
- "Now, define a function that transforms the waveform dataset into spectrograms and their corresponding labels as integer IDs:"
+ "Now, create spectrogram datasets from the audio datasets:"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
- "id": "43IS2IouEV40"
+ "id": "mAD0LpkgqtQo"
},
"outputs": [],
"source": [
- "def get_spectrogram_and_label_id(audio, label):\n",
- " spectrogram = get_spectrogram(audio)\n",
- " label_id = tf.math.argmax(label == commands)\n",
- " return spectrogram, label_id"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {
- "id": "cf5d5b033a45"
- },
- "source": [
- "Map `get_spectrogram_and_label_id` across the dataset's elements with `Dataset.map`:"
+ "def make_spec_ds(ds):\n",
+ " return ds.map(\n",
+ " map_func=lambda audio,label: (get_spectrogram(audio), label),\n",
+ " num_parallel_calls=tf.data.AUTOTUNE)"
]
},
{
@@ -584,9 +495,9 @@
},
"outputs": [],
"source": [
- "spectrogram_ds = waveform_ds.map(\n",
- " map_func=get_spectrogram_and_label_id,\n",
- " num_parallel_calls=AUTOTUNE)"
+ "train_spectrogram_ds = make_spec_ds(train_ds)\n",
+ "val_spectrogram_ds = make_spec_ds(val_ds)\n",
+ "test_spectrogram_ds = make_spec_ds(test_ds)"
]
},
{
@@ -602,89 +513,44 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
- "id": "QUbHfTuon4iF"
- },
- "outputs": [],
- "source": [
- "rows = 3\n",
- "cols = 3\n",
- "n = rows*cols\n",
- "fig, axes = plt.subplots(rows, cols, figsize=(10, 10))\n",
- "\n",
- "for i, (spectrogram, label_id) in enumerate(spectrogram_ds.take(n)):\n",
- " r = i // cols\n",
- " c = i % cols\n",
- " ax = axes[r][c]\n",
- " plot_spectrogram(spectrogram.numpy(), ax)\n",
- " ax.set_title(commands[label_id.numpy()])\n",
- " ax.axis('off')\n",
- "\n",
- "plt.show()"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {
- "id": "z5KdY8IF8rkt"
- },
- "source": [
- "## Build and train the model\n",
- "\n",
- "Repeat the training set preprocessing on the validation and test sets:"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "id": "10UI32QH_45b"
+ "id": "EaM2q5aGis-d"
},
"outputs": [],
"source": [
- "def preprocess_dataset(files):\n",
- " files_ds = tf.data.Dataset.from_tensor_slices(files)\n",
- " output_ds = files_ds.map(\n",
- " map_func=get_waveform_and_label,\n",
- " num_parallel_calls=AUTOTUNE)\n",
- " output_ds = output_ds.map(\n",
- " map_func=get_spectrogram_and_label_id,\n",
- " num_parallel_calls=AUTOTUNE)\n",
- " return output_ds"
+ "for example_spectrograms, example_spect_labels in train_spectrogram_ds.take(1):\n",
+ " break"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
- "id": "HNv4xwYkB2P6"
+ "id": "QUbHfTuon4iF"
},
"outputs": [],
"source": [
- "train_ds = spectrogram_ds\n",
- "val_ds = preprocess_dataset(val_files)\n",
- "test_ds = preprocess_dataset(test_files)"
+ "rows = 3\n",
+ "cols = 3\n",
+ "n = rows*cols\n",
+ "fig, axes = plt.subplots(rows, cols, figsize=(16, 9))\n",
+ "\n",
+ "for i in range(n):\n",
+ " r = i // cols\n",
+ " c = i % cols\n",
+ " ax = axes[r][c]\n",
+ " plot_spectrogram(example_spectrograms[i].numpy(), ax)\n",
+ " ax.set_title(label_names[example_spect_labels[i].numpy()])\n",
+ "\n",
+ "plt.show()"
]
},
{
"cell_type": "markdown",
"metadata": {
- "id": "assnWo6SB3lR"
- },
- "source": [
- "Batch the training and validation sets for model training:"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "id": "UgY9WYzn61EX"
+ "id": "z5KdY8IF8rkt"
},
- "outputs": [],
"source": [
- "batch_size = 64\n",
- "train_ds = train_ds.batch(batch_size)\n",
- "val_ds = val_ds.batch(batch_size)"
+ "## Build and train the model"
]
},
{
@@ -704,8 +570,9 @@
},
"outputs": [],
"source": [
- "train_ds = train_ds.cache().prefetch(AUTOTUNE)\n",
- "val_ds = val_ds.cache().prefetch(AUTOTUNE)"
+ "train_spectrogram_ds = train_spectrogram_ds.cache().shuffle(10000).prefetch(tf.data.AUTOTUNE)\n",
+ "val_spectrogram_ds = val_spectrogram_ds.cache().prefetch(tf.data.AUTOTUNE)\n",
+ "test_spectrogram_ds = test_spectrogram_ds.cache().prefetch(tf.data.AUTOTUNE)"
]
},
{
@@ -732,16 +599,15 @@
},
"outputs": [],
"source": [
- "for spectrogram, _ in spectrogram_ds.take(1):\n",
- " input_shape = spectrogram.shape\n",
+ "input_shape = example_spectrograms.shape[1:]\n",
"print('Input shape:', input_shape)\n",
- "num_labels = len(commands)\n",
+ "num_labels = len(label_names)\n",
"\n",
"# Instantiate the `tf.keras.layers.Normalization` layer.\n",
"norm_layer = layers.Normalization()\n",
"# Fit the state of the layer to the spectrograms\n",
"# with `Normalization.adapt`.\n",
- "norm_layer.adapt(data=spectrogram_ds.map(map_func=lambda spec, label: spec))\n",
+ "norm_layer.adapt(data=train_spectrogram_ds.map(map_func=lambda spec, label: spec))\n",
"\n",
"model = models.Sequential([\n",
" layers.Input(shape=input_shape),\n",
@@ -805,8 +671,8 @@
"source": [
"EPOCHS = 10\n",
"history = model.fit(\n",
- " train_ds,\n",
- " validation_data=val_ds,\n",
+ " train_spectrogram_ds,\n",
+ " validation_data=val_spectrogram_ds,\n",
" epochs=EPOCHS,\n",
" callbacks=tf.keras.callbacks.EarlyStopping(verbose=1, patience=2),\n",
")"
@@ -830,9 +696,20 @@
"outputs": [],
"source": [
"metrics = history.history\n",
+ "plt.figure(figsize=(16,6))\n",
+ "plt.subplot(1,2,1)\n",
"plt.plot(history.epoch, metrics['loss'], metrics['val_loss'])\n",
"plt.legend(['loss', 'val_loss'])\n",
- "plt.show()"
+ "plt.ylim([0, max(plt.ylim())])\n",
+ "plt.xlabel('Epoch')\n",
+ "plt.ylabel('Loss [CrossEntropy]')\n",
+ "\n",
+ "plt.subplot(1,2,2)\n",
+ "plt.plot(history.epoch, 100*np.array(metrics['accuracy']), 100*np.array(metrics['val_accuracy']))\n",
+ "plt.legend(['accuracy', 'val_accuracy'])\n",
+ "plt.ylim([0, 100])\n",
+ "plt.xlabel('Epoch')\n",
+ "plt.ylabel('Accuracy [%]')"
]
},
{
@@ -850,45 +727,55 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
- "id": "biU2MwzyAo8o"
+ "id": "FapuRT_SsWGQ"
},
"outputs": [],
"source": [
- "test_audio = []\n",
- "test_labels = []\n",
- "\n",
- "for audio, label in test_ds:\n",
- " test_audio.append(audio.numpy())\n",
- " test_labels.append(label.numpy())\n",
+ "model.evaluate(test_spectrogram_ds, return_dict=True)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "en9Znt1NOabH"
+ },
+ "source": [
+ "### Display a confusion matrix\n",
"\n",
- "test_audio = np.array(test_audio)\n",
- "test_labels = np.array(test_labels)"
+ "Use a [confusion matrix](https://developers.google.com/machine-learning/glossary#confusion-matrix) to check how well the model did classifying each of the commands in the test set:\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
- "id": "ktUanr9mRZky"
+ "id": "5Y6vmWWQuuT1"
},
"outputs": [],
"source": [
- "y_pred = np.argmax(model.predict(test_audio), axis=1)\n",
- "y_true = test_labels\n",
- "\n",
- "test_acc = sum(y_pred == y_true) / len(y_true)\n",
- "print(f'Test set accuracy: {test_acc:.0%}')"
+ "y_pred = model.predict(test_spectrogram_ds)"
]
},
{
- "cell_type": "markdown",
+ "cell_type": "code",
+ "execution_count": null,
"metadata": {
- "id": "en9Znt1NOabH"
+ "id": "d6F0il82u7lW"
},
+ "outputs": [],
"source": [
- "### Display a confusion matrix\n",
- "\n",
- "Use a confusion matrix to check how well the model did classifying each of the commands in the test set:\n"
+ "y_pred = tf.argmax(y_pred, axis=1)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "vHSNoBYLvX81"
+ },
+ "outputs": [],
+ "source": [
+ "y_true = tf.concat(list(test_spectrogram_ds.map(lambda s,lab: lab)), axis=0)"
]
},
{
@@ -902,8 +789,8 @@
"confusion_mtx = tf.math.confusion_matrix(y_true, y_pred)\n",
"plt.figure(figsize=(10, 8))\n",
"sns.heatmap(confusion_mtx,\n",
- " xticklabels=commands,\n",
- " yticklabels=commands,\n",
+ " xticklabels=label_names,\n",
+ " yticklabels=label_names,\n",
" annot=True, fmt='g')\n",
"plt.xlabel('Prediction')\n",
"plt.ylabel('Label')\n",
@@ -929,15 +816,21 @@
},
"outputs": [],
"source": [
- "sample_file = data_dir/'no/01bb6a2a_nohash_0.wav'\n",
+ "x = data_dir/'no/01bb6a2a_nohash_0.wav'\n",
+ "x = tf.io.read_file(str(x))\n",
+ "x, sample_rate = tf.audio.decode_wav(x, desired_channels=1, desired_samples=16000,)\n",
+ "x = tf.squeeze(x, axis=-1)\n",
+ "waveform = x\n",
+ "x = get_spectrogram(x)\n",
+ "x = x[tf.newaxis,...]\n",
"\n",
- "sample_ds = preprocess_dataset([str(sample_file)])\n",
+ "prediction = model(x)\n",
+ "x_labels = ['no', 'yes', 'down', 'go', 'left', 'up', 'right', 'stop']\n",
+ "plt.bar(x_labels, tf.nn.softmax(prediction[0]))\n",
+ "plt.title('No')\n",
+ "plt.show()\n",
"\n",
- "for spectrogram, label in sample_ds.batch(1):\n",
- " prediction = model(spectrogram)\n",
- " plt.bar(commands, tf.nn.softmax(prediction[0]))\n",
- " plt.title(f'Predictions for \"{commands[label[0]]}\"')\n",
- " plt.show()"
+ "display.display(display.Audio(waveform, rate=16000))"
]
},
{
@@ -949,6 +842,106 @@
"As the output suggests, your model should have recognized the audio command as \"no\"."
]
},
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "h1icqlM3ISW0"
+ },
+ "source": [
+ "## Export the model with preprocessing"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "r7HX-MjgIbji"
+ },
+ "source": [
+ "The model's not very easy to use if you have to apply those preprocessing steps before passing data to the model for inference. So build an end-to-end version:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "2lIeXdWjIbDE"
+ },
+ "outputs": [],
+ "source": [
+ "class ExportModel(tf.Module):\n",
+ " def __init__(self, model):\n",
+ " self.model = model\n",
+ "\n",
+ " # Accept either a string-filename or a batch of waveforms.\n",
+ " # You could add additional signatures for a single wave, or a ragged-batch. \n",
+ " self.__call__.get_concrete_function(\n",
+ " x=tf.TensorSpec(shape=(), dtype=tf.string))\n",
+ " self.__call__.get_concrete_function(\n",
+ " x=tf.TensorSpec(shape=[None, 16000], dtype=tf.float32))\n",
+ "\n",
+ "\n",
+ " @tf.function\n",
+ " def __call__(self, x):\n",
+ " # If they pass a string, load the file and decode it. \n",
+ " if x.dtype == tf.string:\n",
+ " x = tf.io.read_file(x)\n",
+ " x, _ = tf.audio.decode_wav(x, desired_channels=1, desired_samples=16000,)\n",
+ " x = tf.squeeze(x, axis=-1)\n",
+ " x = x[tf.newaxis, :]\n",
+ " \n",
+ " x = get_spectrogram(x) \n",
+ " result = self.model(x, training=False)\n",
+ " \n",
+ " class_ids = tf.argmax(result, axis=-1)\n",
+ " class_names = tf.gather(label_names, class_ids)\n",
+ " return {'predictions':result,\n",
+ " 'class_ids': class_ids,\n",
+ " 'class_names': class_names}"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "gtZBmUiB9HGY"
+ },
+ "source": [
+ "Test run the \"export\" model:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "Z1_8TYaCIRue"
+ },
+ "outputs": [],
+ "source": [
+ "export = ExportModel(model)\n",
+ "export(tf.constant(str(data_dir/'no/01bb6a2a_nohash_0.wav')))"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "1J6Iuz829Cxo"
+ },
+ "source": [
+ "Save and reload the model, the reloaded model gives identical output:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "wTAg4vsn3oEb"
+ },
+ "outputs": [],
+ "source": [
+ "tf.saved_model.save(export, \"saved\")\n",
+ "imported = tf.saved_model.load(\"saved\")\n",
+ "imported(waveform[tf.newaxis, :])"
+ ]
+ },
{
"cell_type": "markdown",
"metadata": {
@@ -960,17 +953,17 @@
"This tutorial demonstrated how to carry out simple audio classification/automatic speech recognition using a convolutional neural network with TensorFlow and Python. To learn more, consider the following resources:\n",
"\n",
"- The [Sound classification with YAMNet](https://www.tensorflow.org/hub/tutorials/yamnet) tutorial shows how to use transfer learning for audio classification.\n",
- "- The notebooks from Kaggle's TensorFlow speech recognition challenge.\n",
+ "- The notebooks from [Kaggle's TensorFlow speech recognition challenge](https://www.kaggle.com/c/tensorflow-speech-recognition-challenge/overview).\n",
"- The \n",
- "TensorFlow.js - Audio recognition using transfer learning codelab teaches how to build your own interactive web app for audio classification.\n",
- "- A tutorial on deep learning for music information retrieval (Choi et al., 2017) on arXiv.\n",
+ "[TensorFlow.js - Audio recognition using transfer learning codelab](https://codelabs.developers.google.com/codelabs/tensorflowjs-audio-codelab/index.html#0) teaches how to build your own interactive web app for audio classification.\n",
+ "- [A tutorial on deep learning for music information retrieval](https://arxiv.org/abs/1709.04396) (Choi et al., 2017) on arXiv.\n",
"- TensorFlow also has additional support for [audio data preparation and augmentation](https://www.tensorflow.org/io/tutorials/audio) to help with your own audio-based projects.\n",
- "- Consider using the librosa library—a Python package for music and audio analysis."
+ "- Consider using the [librosa](https://librosa.org/) library for music and audio analysis."
]
}
],
"metadata": {
- "accelerator": "GPU",
+ "accelerator": "CPU",
"colab": {
"collapsed_sections": [],
"name": "simple_audio.ipynb",
diff --git a/site/en/tutorials/audio/transfer_learning_audio.ipynb b/site/en/tutorials/audio/transfer_learning_audio.ipynb
index 44c9c33d46b..160aeeb7103 100644
--- a/site/en/tutorials/audio/transfer_learning_audio.ipynb
+++ b/site/en/tutorials/audio/transfer_learning_audio.ipynb
@@ -99,7 +99,9 @@
},
"outputs": [],
"source": [
- "!pip install tensorflow_io"
+ "!pip install -q \"tensorflow==2.11.*\"\n",
+ "# tensorflow_io 0.28 is compatible with TensorFlow 2.11\n",
+ "!pip install -q \"tensorflow_io==0.28.*\""
]
},
{
diff --git a/site/en/tutorials/customization/custom_training_walkthrough.ipynb b/site/en/tutorials/customization/custom_training_walkthrough.ipynb
index eb68ab2dc9f..9a05d864815 100644
--- a/site/en/tutorials/customization/custom_training_walkthrough.ipynb
+++ b/site/en/tutorials/customization/custom_training_walkthrough.ipynb
@@ -191,7 +191,7 @@
"source": [
"### Preview the data\n",
"\n",
- "Download the simplified version of the penguins dataset (`penguins/simple`) using the TensorFlow Datasets [`tdfs.load`](https://www.tensorflow.org/datasets/api_docs/python/tfds/load) method. There are 344 data records in this dataset. Extract the first five records into a [`DataFrame`](https://www.tensorflow.org/datasets/api_docs/python/tfds/as_dataframe) object to inspect a sample of the values in this dataset:"
+ "Download the simplified version of the penguins dataset (`penguins/simple`) using the TensorFlow Datasets [`tfds.load`](https://www.tensorflow.org/datasets/api_docs/python/tfds/load) method. There are 344 data records in this dataset. Extract the first five records into a [`DataFrame`](https://www.tensorflow.org/datasets/api_docs/python/tfds/as_dataframe) object to inspect a sample of the values in this dataset:"
]
},
{
diff --git a/site/en/tutorials/distribute/custom_training.ipynb b/site/en/tutorials/distribute/custom_training.ipynb
index 4534f1a7044..d14b0ac003c 100644
--- a/site/en/tutorials/distribute/custom_training.ipynb
+++ b/site/en/tutorials/distribute/custom_training.ipynb
@@ -144,7 +144,7 @@
"* All the variables and the model graph are replicated across the replicas.\n",
"* Input is evenly distributed across the replicas.\n",
"* Each replica calculates the loss and gradients for the input it received.\n",
- "* The gradients are synced across all the replicas by summing them.\n",
+ "* The gradients are synced across all the replicas by **summing** them.\n",
"* After the sync, the same update is made to the copies of the variables on each replica.\n",
"\n",
"Note: You can put all the code below inside a single scope. This example divides it into several code cells for illustration purposes.\n"
@@ -216,8 +216,8 @@
},
"outputs": [],
"source": [
- "train_dataset = tf.data.Dataset.from_tensor_slices((train_images, train_labels)).shuffle(BUFFER_SIZE).batch(GLOBAL_BATCH_SIZE) \n",
- "test_dataset = tf.data.Dataset.from_tensor_slices((test_images, test_labels)).batch(GLOBAL_BATCH_SIZE) \n",
+ "train_dataset = tf.data.Dataset.from_tensor_slices((train_images, train_labels)).shuffle(BUFFER_SIZE).batch(GLOBAL_BATCH_SIZE)\n",
+ "test_dataset = tf.data.Dataset.from_tensor_slices((test_images, test_labels)).batch(GLOBAL_BATCH_SIZE)\n",
"\n",
"train_dist_dataset = strategy.experimental_distribute_dataset(train_dataset)\n",
"test_dist_dataset = strategy.experimental_distribute_dataset(test_dataset)"
@@ -243,14 +243,21 @@
"outputs": [],
"source": [
"def create_model():\n",
+ " regularizer = tf.keras.regularizers.L2(1e-5)\n",
" model = tf.keras.Sequential([\n",
- " tf.keras.layers.Conv2D(32, 3, activation='relu'),\n",
+ " tf.keras.layers.Conv2D(32, 3,\n",
+ " activation='relu',\n",
+ " kernel_regularizer=regularizer),\n",
" tf.keras.layers.MaxPooling2D(),\n",
- " tf.keras.layers.Conv2D(64, 3, activation='relu'),\n",
+ " tf.keras.layers.Conv2D(64, 3,\n",
+ " activation='relu',\n",
+ " kernel_regularizer=regularizer),\n",
" tf.keras.layers.MaxPooling2D(),\n",
" tf.keras.layers.Flatten(),\n",
- " tf.keras.layers.Dense(64, activation='relu'),\n",
- " tf.keras.layers.Dense(10)\n",
+ " tf.keras.layers.Dense(64,\n",
+ " activation='relu',\n",
+ " kernel_regularizer=regularizer),\n",
+ " tf.keras.layers.Dense(10, kernel_regularizer=regularizer)\n",
" ])\n",
"\n",
" return model"
@@ -277,25 +284,29 @@
"source": [
"## Define the loss function\n",
"\n",
- "Normally, on a single machine with single GPU/CPU, loss is divided by the number of examples in the batch of input.\n",
+ "Recall that the loss function consists of one or two parts:\n",
"\n",
- "*So, how should the loss be calculated when using a `tf.distribute.Strategy`?*\n",
+ " * The **prediction loss** measures how far off the model's predictions are from the training labels for a batch of training examples. It is computed for each labeled example and then reduced across the batch by computing the average value.\n",
+ " * Optionally, **regularization loss** terms can be added to the prediction loss, to steer the model away from overfitting the training data. A common choice is L2 regularization, which adds a small fixed multiple of the sum of squares of all model weights, independent of the number of examples. The model above uses L2 regularization to demonstrate its handling in the training loop below.\n",
"\n",
- "* For an example, let's say you have 4 GPU's and a batch size of 64. One batch of input is distributed\n",
- "across the replicas (4 GPUs), each replica getting an input of size 16.\n",
+ "For training on a single machine with a single GPU/CPU, this works as follows:\n",
"\n",
- "* The model on each replica does a forward pass with its respective input and calculates the loss. Now, instead of dividing the loss by the number of examples in its respective input (`BATCH_SIZE_PER_REPLICA` = 16), the loss should be divided by the `GLOBAL_BATCH_SIZE` (64)."
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {
- "id": "OCIcsaeoIHJX"
- },
- "source": [
- "*Why do this?*\n",
+ " * The prediction loss is computed for each example in the batch, summed across the batch, and then divided by the batch size.\n",
+ " * The regularization loss is added to the prediction loss.\n",
+ " * The gradient of the total loss is computed w.r.t. each model weight, and the optimizer updates each model weight from the corresponding gradient.\n",
+ "\n",
+ "With `tf.distribute.Strategy`, the input batch is split between replicas.\n",
+ "For example, let's say you have 4 GPUs, each with one replica of the model. One batch of 256 input examples is distributed evenly across the 4 replicas, so each replica gets a batch of size 64: We have `256 = 4*64`, or generally `GLOBAL_BATCH_SIZE = num_replicas_in_sync * BATCH_SIZE_PER_REPLICA`.\n",
+ "\n",
+ "Each replica computes the loss from the training examples it gets and computes the gradients of the loss w.r.t. each model weight. The optimizer takes care that these **gradients are summed up across replicas** before using them to update the copies of the model weights on each replica.\n",
+ "\n",
+ "*So, how should the loss be calculated when using a `tf.distribute.Strategy`?*\n",
+ "\n",
+ " * Each replica computes the prediction loss for all examples distributed to it, sums up the results and divides them by `num_replicas_in_sync * BATCH_SIZE_PER_REPLICA`, or equivently, `GLOBAL_BATCH_SIZE`.\n",
+ " * Each replica compues the regularization loss(es) and divides them by\n",
+ " `num_replicas_in_sync`.\n",
"\n",
- "* This needs to be done because after the gradients are calculated on each replica, they are synced across the replicas by **summing** them."
+ "Compared to non-distributed training, all per-replica loss terms are scaled down by a factor of `1/num_replicas_in_sync`. On the other hand, all loss terms -- or rather, their gradients -- are summed across that number of replicas before the optimizer applies them. In effect, the optimizer on each replica uses the same gradients as if a non-distributed computation with `GLOBAL_BATCH_SIZE` had happened. This is consistent with the distributed and undistributed behavior of Keras `Model.fit`. See the [Distributed training with Keras](./keras.ipynb) tutorial on how a larger gloabl batch size enables to scale up the learning rate."
]
},
{
@@ -306,31 +317,18 @@
"source": [
"*How to do this in TensorFlow?*\n",
"\n",
- "* If you're writing a custom training loop, as in this tutorial, you should sum the per example losses and divide the sum by the `GLOBAL_BATCH_SIZE`: \n",
- "`scale_loss = tf.reduce_sum(loss) * (1. / GLOBAL_BATCH_SIZE)`\n",
- "or you can use `tf.nn.compute_average_loss` which takes the per example loss,\n",
- "optional sample weights, and `GLOBAL_BATCH_SIZE` as arguments and returns the scaled loss.\n",
- "\n",
- "* If you are using regularization losses in your model then you need to scale\n",
- "the loss value by the number of replicas. You can do this by using the `tf.nn.scale_regularization_loss` function.\n",
+ " * Loss reduction and scaling is done automatically in Keras `Model.compile` and `Model.fit`\n",
"\n",
- "* Using `tf.reduce_mean` is not recommended. Doing so divides the loss by actual per replica batch size which may vary step to step.\n",
+ " * If you're writing a custom training loop, as in this tutorial, you should sum the per-example losses and divide the sum by the global batch size using `tf.nn.compute_average_loss`, which takes the per-example losses and\n",
+ "optional sample weights as arguments and returns the scaled loss.\n",
"\n",
- "* This reduction and scaling is done automatically in Keras `Model.compile` and `Model.fit`\n",
+ " * If using `tf.keras.losses` classes (as in the example below), the loss reduction needs to be explicitly specified to be one of `NONE` or `SUM`. The default `AUTO` and `SUM_OVER_BATCH_SIZE` are disallowed outside `Model.fit`.\n",
+ " * `AUTO` is disallowed because the user should explicitly think about what reduction they want to make sure it is correct in the distributed case.\n",
+ " * `SUM_OVER_BATCH_SIZE` is disallowed because currently it would only divide by per replica batch size, and leave the dividing by number of replicas to the user, which might be easy to miss. So, instead, you need to do the reduction yourself explicitly.\n",
"\n",
- "* If using `tf.keras.losses` classes (as in the example below), the loss reduction needs to be explicitly specified to be one of `NONE` or `SUM`. `AUTO` and `SUM_OVER_BATCH_SIZE` are disallowed when used with `tf.distribute.Strategy`. `AUTO` is disallowed because the user should explicitly think about what reduction they want to make sure it is correct in the distributed case. `SUM_OVER_BATCH_SIZE` is disallowed because currently it would only divide by per replica batch size, and leave the dividing by number of replicas to the user, which might be easy to miss. So, instead, you need to do the reduction yourself explicitly.\n",
- "* If `labels` is multi-dimensional, then average the `per_example_loss` across the number of elements in each sample. For example, if the shape of `predictions` is `(batch_size, H, W, n_classes)` and `labels` is `(batch_size, H, W)`, you will need to update `per_example_loss` like: `per_example_loss /= tf.cast(tf.reduce_prod(tf.shape(labels)[1:]), tf.float32)`\n",
+ " * If you're writing a custom training loop for a model with a non-empty list of `Model.losses` (e.g., weight regularizers), you should sum them up and divide the sum by the number of replicas. You can do this by using the `tf.nn.scale_regularization_loss` function. The model code itself remains unaware of the number of replicas.\n",
"\n",
- " Caution: **Verify the shape of your loss**. \n",
- " Loss functions in `tf.losses`/`tf.keras.losses` typically\n",
- " return the average over the last dimension of the input. The loss\n",
- " classes wrap these functions. Passing `reduction=Reduction.NONE` when\n",
- " creating an instance of a loss class means \"no **additional** reduction\".\n",
- " For categorical losses with an example input shape of `[batch, W, H, n_classes]` the `n_classes`\n",
- " dimension is reduced. For pointwise losses like\n",
- " `losses.mean_squared_error` or `losses.binary_crossentropy` include a\n",
- " dummy axis so that `[batch, W, H, 1]` is reduced to `[batch, W, H]`. Without\n",
- " the dummy axis `[batch, W, H]` will be incorrectly reduced to `[batch, W]`.\n"
+ " However, models can define input-dependent regularization losses with Keras APIs such as `Layer.add_loss(...)` and `Layer(activity_regularizer=...)`. For `Layer.add_loss(...)`, it falls on the modeling code to perform the division of the summed per-example terms by the per-replica(!) batch size, e.g., by using `tf.math.reduce_mean()`."
]
},
{
@@ -342,14 +340,51 @@
"outputs": [],
"source": [
"with strategy.scope():\n",
- " # Set reduction to `NONE` so you can do the reduction afterwards and divide by\n",
- " # global batch size.\n",
+ " # Set reduction to `NONE` so you can do the reduction yourself.\n",
" loss_object = tf.keras.losses.SparseCategoricalCrossentropy(\n",
" from_logits=True,\n",
" reduction=tf.keras.losses.Reduction.NONE)\n",
- " def compute_loss(labels, predictions):\n",
+ " def compute_loss(labels, predictions, model_losses):\n",
" per_example_loss = loss_object(labels, predictions)\n",
- " return tf.nn.compute_average_loss(per_example_loss, global_batch_size=GLOBAL_BATCH_SIZE)"
+ " loss = tf.nn.compute_average_loss(per_example_loss)\n",
+ " if model_losses:\n",
+ " loss += tf.nn.scale_regularization_loss(tf.add_n(model_losses))\n",
+ " return loss"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "6pM96bqQY52D"
+ },
+ "source": [
+ "### Special cases\n",
+ "\n",
+ "Advanced users should also consider the following special cases.\n",
+ "\n",
+ " * Input batches shorter than `GLOBAL_BATCH_SIZE` create unpleasant corner cases in several places. In practice, it often works best to avoid them by allowing batches to span epoch boundaries using `Dataset.repeat().batch()` and defining approximate epochs by step counts, not dataset ends. Alternatively, `Dataset.batch(drop_remainder=True)` maintains the notion of epoch but drops the last few examples.\n",
+ "\n",
+ " For illustration, this example goes the harder route and allows short batches, so that each training epoch contains each training example exactly once.\n",
+ " \n",
+ " Which denominator should be used by `tf.nn.compute_average_loss()`?\n",
+ "\n",
+ " * By default, in the example code above and equivalently in `Keras.fit()`, the sum of prediction losses is divided by `num_replicas_in_sync` times the actual batch size seen on the replica (with empty batches silently ignored). This preserves the balance between the prediction loss on the one hand and the regularization losses on the other hand. It is particularly appropriate for models that use input-dependent regularization losses. Plain L2 regularization just superimposes weight decay onto the gradients of the prediction loss and is less in need of such a balance.\n",
+ " * In practice, many custom training loops pass as a constant Python value into `tf.nn.compute_average_loss(..., global_batch_size=GLOBAL_BATCH_SIZE)` to use it as the denominator. This preserves the relative weighting of training examples between batches. Without it, the smaller denominator in short batches effectively upweights the examples in those. (Before TensorFlow 2.13, this was also needed to avoid NaNs in case some replica received an actual batch size of zero.)\n",
+ " \n",
+ " Both options are equivalent if short batches are avoided, as suggested above.\n",
+ "\n",
+ " * Multi-dimensional `labels` require you to average the `per_example_loss` across the number of predictions in each example. Consider a classification task for all pixels of an input image, with `predictions` of shape `(batch_size, H, W, n_classes)` and `labels` of shape `(batch_size, H, W)`. You will need to update `per_example_loss` like: `per_example_loss /= tf.cast(tf.reduce_prod(tf.shape(labels)[1:]), tf.float32)`\n",
+ "\n",
+ " Caution: **Verify the shape of your loss**.\n",
+ " Loss functions in `tf.losses`/`tf.keras.losses` typically\n",
+ " return the average over the last dimension of the input. The loss\n",
+ " classes wrap these functions. Passing `reduction=Reduction.NONE` when\n",
+ " creating an instance of a loss class means \"no **additional** reduction\".\n",
+ " For categorical losses with an example input shape of `[batch, W, H, n_classes]` the `n_classes`\n",
+ " dimension is reduced. For pointwise losses like\n",
+ " `losses.mean_squared_error` or `losses.binary_crossentropy` include a\n",
+ " dummy axis so that `[batch, W, H, 1]` is reduced to `[batch, W, H]`. Without\n",
+ " the dummy axis `[batch, W, H]` will be incorrectly reduced to `[batch, W]`."
]
},
{
@@ -401,7 +436,7 @@
"with strategy.scope():\n",
" model = create_model()\n",
"\n",
- " optimizer = tf.keras.optimizers.Adam()\n",
+ " optimizer = tf.keras.optimizers.Adam(learning_rate=0.001)\n",
"\n",
" checkpoint = tf.train.Checkpoint(optimizer=optimizer, model=model)"
]
@@ -419,13 +454,13 @@
"\n",
" with tf.GradientTape() as tape:\n",
" predictions = model(images, training=True)\n",
- " loss = compute_loss(labels, predictions)\n",
+ " loss = compute_loss(labels, predictions, model.losses)\n",
"\n",
" gradients = tape.gradient(loss, model.trainable_variables)\n",
" optimizer.apply_gradients(zip(gradients, model.trainable_variables))\n",
"\n",
" train_accuracy.update_state(labels, predictions)\n",
- " return loss \n",
+ " return loss\n",
"\n",
"def test_step(inputs):\n",
" images, labels = inputs\n",
@@ -490,12 +525,12 @@
"id": "Z1YvXqOpwy08"
},
"source": [
- "Things to note in the example above:\n",
+ "### Things to note in the example above\n",
"\n",
"* Iterate over the `train_dist_dataset` and `test_dist_dataset` using a `for x in ...` construct.\n",
"* The scaled loss is the return value of the `distributed_train_step`. This value is aggregated across replicas using the `tf.distribute.Strategy.reduce` call and then across batches by summing the return value of the `tf.distribute.Strategy.reduce` calls.\n",
"* `tf.keras.Metrics` should be updated inside `train_step` and `test_step` that gets executed by `tf.distribute.Strategy.run`.\n",
- "*`tf.distribute.Strategy.run` returns results from each local replica in the strategy, and there are multiple ways to consume this result. You can do `tf.distribute.Strategy.reduce` to get an aggregated value. You can also do `tf.distribute.Strategy.experimental_local_results` to get the list of values contained in the result, one per local replica.\n"
+ "* `tf.distribute.Strategy.run` returns results from each local replica in the strategy, and there are multiple ways to consume this result. You can do `tf.distribute.Strategy.reduce` to get an aggregated value. You can also do `tf.distribute.Strategy.experimental_local_results` to get the list of values contained in the result, one per local replica.\n"
]
},
{
@@ -659,7 +694,7 @@
"* Resulting loss values: [2, 3] and [4, 5] on each replica\n",
"* Global batch size = 4\n",
"\n",
- "With loss scaling, you calculate the per-sample value of loss on each replica by adding the loss values, and then dividing by the global batch size. In this case: `(2 + 3) / 4 = 1.25` and `(4 + 5) / 4 = 2.25`. \n",
+ "With loss scaling, you calculate the per-sample value of loss on each replica by adding the loss values, and then dividing by the global batch size. In this case: `(2 + 3) / 4 = 1.25` and `(4 + 5) / 4 = 2.25`.\n",
"\n",
"If you use `tf.keras.metrics.Mean` to track loss across the two replicas, the result is different. In this example, you end up with a `total` of 3.50 and `count` of 2, which results in `total`/`count` = 1.75 when `result()` is called on the metric. Loss calculated with `tf.keras.Metrics` is scaled by an additional factor that is equal to the number of replicas in sync."
]
diff --git a/site/en/tutorials/distribute/dtensor_keras_tutorial.ipynb b/site/en/tutorials/distribute/dtensor_keras_tutorial.ipynb
index b09348d9264..84f6478c2b5 100644
--- a/site/en/tutorials/distribute/dtensor_keras_tutorial.ipynb
+++ b/site/en/tutorials/distribute/dtensor_keras_tutorial.ipynb
@@ -69,15 +69,16 @@
},
"source": [
"## Overview\n",
- "In this tutoral, you will learn how to use DTensor with Keras.\n",
+ "\n",
+ "In this tutorial, you will learn how to use DTensors with Keras.\n",
"\n",
"Through DTensor integration with Keras, you can reuse your existing Keras layers and models to build and train distributed machine learning models.\n",
"\n",
"You will train a multi-layer classification model with the MNIST data. Setting the layout for subclassing model, Sequential model, and functional model will be demonstrated.\n",
"\n",
- "This tutoral assumes that you have already read the [DTensor programing guide](/guide/dtensor_overview), and are familiar with basic DTensor concepts like `Mesh` and `Layout`.\n",
+ "This tutorial assumes that you have already read the [DTensor programing guide](/guide/dtensor_overview), and are familiar with basic DTensor concepts like `Mesh` and `Layout`.\n",
"\n",
- "This tutoral is based on https://www.tensorflow.org/datasets/keras_example."
+ "This tutorial is based on [Training a neural network on MNIST with Keras](https://www.tensorflow.org/datasets/keras_example)."
]
},
{
@@ -88,7 +89,9 @@
"source": [
"## Setup\n",
"\n",
- "DTensor is part of TensorFlow 2.9.0 release."
+ "DTensor (`tf.experimental.dtensor`) has been part of TensorFlow since the 2.9.0 release.\n",
+ "\n",
+ "First, install or upgrade TensorFlow Datasets:"
]
},
{
@@ -99,7 +102,7 @@
},
"outputs": [],
"source": [
- "!pip install --quiet --upgrade --pre tensorflow tensorflow-datasets"
+ "!pip install --quiet --upgrade tensorflow-datasets"
]
},
{
@@ -108,9 +111,9 @@
"id": "VttBMZngDx8x"
},
"source": [
- "Next, import `tensorflow` and `tensorflow.experimental.dtensor`, and configure TensorFlow to use 8 virtual CPUs.\n",
+ "Next, import TensorFlow and `dtensor`, and configure TensorFlow to use 8 virtual CPUs.\n",
"\n",
- "Even though this example uses CPUs, DTensor works the same way on CPU, GPU or TPU devices."
+ "Even though this example uses virtual CPUs, DTensor works the same way on CPU, GPU or TPU devices."
]
},
{
@@ -176,11 +179,11 @@
"source": [
"## Creating a Data Parallel Mesh\n",
"\n",
- "This tutorial demonstrates Data Parallel training. Adapting to Model Parallel training and Spatial Parallel training can be as simple as switching to a different set of `Layout` objects. Refer to [DTensor in-depth ML Tutorial](https://www.tensorflow.org/tutorials/distribute/dtensor_ml_tutorial) for more information on distributed training beyond Data Parallel.\n",
+ "This tutorial demonstrates Data Parallel training. Adapting to Model Parallel training and Spatial Parallel training can be as simple as switching to a different set of `Layout` objects. Refer to the [Distributed training with DTensors](dtensor_ml_tutorial.ipynb) tutorial for more information on distributed training beyond Data Parallel.\n",
"\n",
- "Data Parallel training is a commonly used parallel training scheme, also used by for example `tf.distribute.MirroredStrategy`.\n",
+ "Data Parallel training is a commonly used parallel training scheme, also used by, for example, `tf.distribute.MirroredStrategy`.\n",
"\n",
- "With DTensor, a Data Parallel training loop uses a `Mesh` that consists of a single 'batch' dimension, where each device runs a replica of the model that receives a shard from the global batch.\n"
+ "With DTensor, a Data Parallel training loop uses a `Mesh` that consists of a single 'batch' dimension, where each device runs a replica of the model that receives a shard from the global batch."
]
},
{
@@ -248,7 +251,7 @@
"\n",
"In order to configure the layout information for your layers' weights, Keras has exposed an extra parameter in the layer constructor for most of the built-in layers.\n",
"\n",
- "The following example builds a small image classification model with fully replicated weight layout. You can specify layout information `kernel` and `bias` in `tf.keras.layers.Dense` via argument `kernel_layout` and `bias_layout`. Most of the built-in keras layers are ready for explicitly specifying the `Layout` for the layer weights."
+ "The following example builds a small image classification model with fully replicated weight layout. You can specify layout information `kernel` and `bias` in `tf.keras.layers.Dense` via arguments `kernel_layout` and `bias_layout`. Most of the built-in keras layers are ready for explicitly specifying the `Layout` for the layer weights."
]
},
{
@@ -315,7 +318,7 @@
"source": [
"## Load a dataset and build input pipeline\n",
"\n",
- "Load a MNIST dataset and configure some pre-processing input pipeline for it. The dataset itself is not associated with any DTensor layout information. There are plans to improve DTensor Keras integration with `tf.data` in future TensorFlow releases.\n"
+ "Load a MNIST dataset and configure some pre-processing input pipeline for it. The dataset itself is not associated with any DTensor layout information."
]
},
{
@@ -389,9 +392,9 @@
"source": [
"## Define the training logic for the model\n",
"\n",
- "Next define the training and evalution logic for the model. \n",
+ "Next, define the training and evaluation logic for the model. \n",
"\n",
- "As of TensorFlow 2.9, you have to write a custom-training-loop for a DTensor enabled Keras model. This is to pack the input data with proper layout information, which is not integrated with the standard `tf.keras.Model.fit()` or `tf.keras.Model.eval()` functions from Keras. you will get more `tf.data` support in the upcoming release. "
+ "As of TensorFlow 2.9, you have to write a custom-training-loop for a DTensor-enabled Keras model. This is to pack the input data with proper layout information, which is not integrated with the standard `tf.keras.Model.fit()` or `tf.keras.Model.eval()` functions from Keras. you will get more `tf.data` support in the upcoming release. "
]
},
{
@@ -467,7 +470,7 @@
"id": "9Eb-qIJGrxB9"
},
"source": [
- "## Metrics and Optimizers\n",
+ "## Metrics and optimizers\n",
"\n",
"When using DTensor API with Keras `Metric` and `Optimizer`, you will need to provide the extra mesh information, so that any internal state variables and tensors can work with variables in the model.\n",
"\n",
@@ -497,9 +500,9 @@
"source": [
"## Train the model\n",
"\n",
- "The following example shards the data from input pipeline on the batch dimension, and train with the model, which has fully replicated weights. \n",
+ "The following example demonstrates how to shard the data from input pipeline on the batch dimension, and train with the model, which has fully replicated weights. \n",
"\n",
- "With 3 epochs, the model should achieve about 97% of accuracy."
+ "After 3 epochs, the model should achieve about 97% of accuracy:"
]
},
{
@@ -561,13 +564,13 @@
"\n",
"Often you have models that work well for your use case. Specifying `Layout` information to each individual layer within the model will be a large amount of work requiring a lot of edits.\n",
"\n",
- "To help you easily convert your existing Keras model to work with DTensor API you can use the new `dtensor.LayoutMap` API that allow you to specify the `Layout` from a global point of view.\n",
+ "To help you easily convert your existing Keras model to work with DTensor API you can use the new `tf.keras.dtensor.experimental.LayoutMap` API that allow you to specify the `Layout` from a global point of view.\n",
"\n",
"First, you need to create a `LayoutMap` instance, which is a dictionary-like object that contains all the `Layout` you would like to specify for your model weights.\n",
"\n",
"`LayoutMap` needs a `Mesh` instance at init, which can be used to provide default replicated `Layout` for any weights that doesn't have Layout configured. In case you would like all your model weights to be just fully replicated, you can provide empty `LayoutMap`, and the default mesh will be used to create replicated `Layout`.\n",
"\n",
- "`LayoutMap` uses a string as key and a `Layout` as value. There is a behavior difference between a normal Python dict and this class. The string key will be treated as a regex when retrieving the value"
+ "`LayoutMap` uses a string as key and a `Layout` as value. There is a behavior difference between a normal Python dict and this class. The string key will be treated as a regex when retrieving the value."
]
},
{
@@ -616,9 +619,9 @@
"* `model.feature_2.kernel`\n",
"* `model.feature_2.bias`\n",
"\n",
- "Note: For Subclassed Models, the attribute name, rather than the `.name` attribute of layer are used as the key to retrieve the Layout from the mapping. This is consistent with the convention followed by `tf.Module` checkpointing. For complex models with more than a few layers, you can [manually inspect checkpoints](https://www.tensorflow.org/guide/checkpoint#manually_inspecting_checkpoints) to see the attribute mappings. \n",
+ "Note: For subclassed Models, the attribute name, rather than the `.name` attribute of the layer, is used as the key to retrieve the Layout from the mapping. This is consistent with the convention followed by `tf.Module` checkpointing. For complex models with more than a few layers, you can [manually inspect checkpoints](https://www.tensorflow.org/guide/checkpoint#manually_inspecting_checkpoints) to view the attribute mappings. \n",
"\n",
- "Now define the following `LayoutMap` and apply it to the model."
+ "Now define the following `LayoutMap` and apply it to the model:"
]
},
{
@@ -634,7 +637,7 @@
"layout_map['feature.*kernel'] = dtensor.Layout.batch_sharded(mesh, 'batch', rank=2)\n",
"layout_map['feature.*bias'] = dtensor.Layout.batch_sharded(mesh, 'batch', rank=1)\n",
"\n",
- "with tf.keras.dtensor.experimental.layout_map_scope(layout_map):\n",
+ "with layout_map.scope():\n",
" subclassed_model = SubclassedModel()"
]
},
@@ -644,7 +647,7 @@
"id": "M32HcSp_PyWs"
},
"source": [
- "The model weights are created on the first call, so call the model with a DTensor input and confirm the weights have the expected layouts."
+ "The model weights are created on the first call, so call the model with a DTensor input and confirm the weights have the expected layouts:"
]
},
{
@@ -686,9 +689,9 @@
"id": "6zzvTqAR2Teu"
},
"source": [
- "For keras functional and sequential models, you can use `LayoutMap` as well.\n",
+ "For Keras Functional and Sequential models, you can use `tf.keras.dtensor.experimental.LayoutMap` as well.\n",
"\n",
- "Note: For functional and sequential models, the mappings are slightly different. The layers in the model don't have a public attribute attached to the model (though you can access them via `model.layers` as a list). Use the string name as the key in this case. The string name is guaranteed to be unique within a model."
+ "Note: For Functional and Sequential models, the mappings are slightly different. The layers in the model don't have a public attribute attached to the model (though you can access them via `Model.layers` as a list). Use the string name as the key in this case. The string name is guaranteed to be unique within a model."
]
},
{
@@ -713,7 +716,7 @@
},
"outputs": [],
"source": [
- "with tf.keras.dtensor.experimental.layout_map_scope(layout_map):\n",
+ "with layout_map.scope():\n",
" inputs = tf.keras.Input((16,), batch_size=16)\n",
" x = tf.keras.layers.Dense(16, name='feature')(inputs)\n",
" x = tf.keras.layers.Dropout(0.1)(x)\n",
@@ -731,7 +734,7 @@
},
"outputs": [],
"source": [
- "with tf.keras.dtensor.experimental.layout_map_scope(layout_map):\n",
+ "with layout_map.scope():\n",
" model = tf.keras.Sequential([\n",
" tf.keras.layers.Dense(16, name='feature', input_shape=(16,)),\n",
" tf.keras.layers.Dropout(0.1),\n",
@@ -744,7 +747,6 @@
],
"metadata": {
"colab": {
- "collapsed_sections": [],
"name": "dtensor_keras_tutorial.ipynb",
"toc_visible": true
},
diff --git a/site/en/tutorials/distribute/dtensor_ml_tutorial.ipynb b/site/en/tutorials/distribute/dtensor_ml_tutorial.ipynb
index 4a0f7bd60f6..55557be6368 100644
--- a/site/en/tutorials/distribute/dtensor_ml_tutorial.ipynb
+++ b/site/en/tutorials/distribute/dtensor_ml_tutorial.ipynb
@@ -37,7 +37,7 @@
"id": "MfBg1C5NB3X0"
},
"source": [
- "# Distributed Training with DTensors\n"
+ "# Distributed training with DTensors"
]
},
{
@@ -70,25 +70,22 @@
"source": [
"## Overview\n",
"\n",
- "DTensor provides a way for you to distribute the training of your model across devices to improve efficiency, reliability and scalability. For more details on DTensor concepts, see [The DTensor Programming Guide](https://www.tensorflow.org/guide/dtensor_overview).\n",
+ "DTensor provides a way for you to distribute the training of your model across devices to improve efficiency, reliability and scalability. For more details, check out the [DTensor concepts](../../guide/dtensor_overview.ipynb) guide.\n",
"\n",
- "In this tutorial, you will train a Sentiment Analysis model with DTensor. Three distributed training schemes are demonstrated with this example:\n",
+ "In this tutorial, you will train a sentiment analysis model using DTensors. The example demonstrates three distributed training schemes:\n",
"\n",
" - Data Parallel training, where the training samples are sharded (partitioned) to devices.\n",
" - Model Parallel training, where the model variables are sharded to devices.\n",
- " - Spatial Parallel training, where the features of input data are sharded to devices. (Also known as [Spatial Partitioning](https://cloud.google.com/blog/products/ai-machine-learning/train-ml-models-on-large-images-and-3d-volumes-with-spatial-partitioning-on-cloud-tpus))\n",
+ " - Spatial Parallel training, where the features of input data are sharded to devices (also known as [Spatial Partitioning](https://cloud.google.com/blog/products/ai-machine-learning/train-ml-models-on-large-images-and-3d-volumes-with-spatial-partitioning-on-cloud-tpus)).\n",
"\n",
- "The training portion of this tutorial is inspired [A Kaggle guide on Sentiment Analysis](https://www.kaggle.com/code/anasofiauzsoy/yelp-review-sentiment-analysis-tensorflow-tfds/notebook) notebook. To learn about the complete training and evaluation workflow (without DTensor), refer to that notebook.\n",
+ "The training portion of this tutorial is inspired by a Kaggle notebook called [A Kaggle guide on sentiment analysis](https://www.kaggle.com/code/anasofiauzsoy/yelp-review-sentiment-analysis-tensorflow-tfds/notebook). To learn about the complete training and evaluation workflow (without DTensor), refer to that notebook.\n",
"\n",
"This tutorial will walk through the following steps:\n",
"\n",
- "- First start with some data cleaning to obtain a `tf.data.Dataset` of tokenized sentences and their polarity.\n",
- "\n",
- "- Next build an MLP model with custom Dense and BatchNorm layers. Use a `tf.Module` to track the inference variables. The model constructor takes additional `Layout` arguments to control the sharding of variables.\n",
- "\n",
- "- For training, you will first use data parallel training together with `tf.experimental.dtensor`'s checkpoint feature. Then continue with Model Parallel Training and Spatial Parallel Training.\n",
- "\n",
- "- The final section briefly describes the interaction between `tf.saved_model` and `tf.experimental.dtensor` as of TensorFlow 2.9.\n"
+ "- Some data cleaning to obtain a `tf.data.Dataset` of tokenized sentences and their polarity.\n",
+ "- Then, building an MLP model with custom Dense and BatchNorm layers using a `tf.Module` to track the inference variables. The model constructor will take additional `Layout` arguments to control the sharding of variables.\n",
+ "- For training, you will first use data parallel training together with `tf.experimental.dtensor`'s checkpoint feature. Then, you will continue with Model Parallel Training and Spatial Parallel Training.\n",
+ "- The final section briefly describes the interaction between `tf.saved_model` and `tf.experimental.dtensor` as of TensorFlow 2.9."
]
},
{
@@ -99,7 +96,9 @@
"source": [
"## Setup\n",
"\n",
- "DTensor is part of TensorFlow 2.9.0 release."
+ "DTensor (`tf.experimental.dtensor`) has been part of TensorFlow since the 2.9.0 release.\n",
+ "\n",
+ "First, install or upgrade TensorFlow Datasets:"
]
},
{
@@ -110,7 +109,7 @@
},
"outputs": [],
"source": [
- "!pip install --quiet --upgrade --pre tensorflow tensorflow-datasets"
+ "!pip install --quiet --upgrade tensorflow-datasets"
]
},
{
@@ -119,9 +118,9 @@
"id": "tcxP4_Zu7ciQ"
},
"source": [
- "Next, import `tensorflow` and `tensorflow.experimental.dtensor`. Then configure TensorFlow to use 8 virtual CPUs.\n",
+ "Next, import `tensorflow` and `dtensor`, and configure TensorFlow to use 8 virtual CPUs.\n",
"\n",
- "Even though this example uses CPUs, DTensor works the same way on CPU, GPU or TPU devices."
+ "Even though this example uses virtual CPUs, DTensor works the same way on CPU, GPU or TPU devices."
]
},
{
@@ -139,6 +138,7 @@
"import tensorflow as tf\n",
"\n",
"from tensorflow.experimental import dtensor\n",
+ "\n",
"print('TensorFlow version:', tf.__version__)"
]
},
@@ -170,7 +170,7 @@
"source": [
"## Download the dataset\n",
"\n",
- "Download the IMDB reviews data set to train the sentiment analysis model."
+ "Download the IMDB reviews data set to train the sentiment analysis model:"
]
},
{
@@ -697,9 +697,9 @@
"source": [
"### Checkpointing\n",
"\n",
- "You can checkpoint a DTensor model using `dtensor.DTensorCheckpoint`. The format of a DTensor checkpoint is fully compatible with a Standard TensorFlow Checkpoint. There is ongoing work to consolidate `dtensor.DTensorCheckpoint` into `tf.train.Checkpoint`.\n",
+ "You can checkpoint a DTensor model using `tf.train.Checkpoint` out of the box. Saving and restoring sharded DVariables will perform an efficient sharded save and restore. Currently, when using `tf.train.Checkpoint.save` and `tf.train.Checkpoint.restore`, all DVariables must be on the same host mesh, and DVariables and regular variables cannot be saved together. You can learn more about checkpointing in [this guide](../../guide/checkpoint.ipynb).\n",
"\n",
- "When a DTensor checkpoint is restored, `Layout`s of variables can be different from when the checkpoint is saved. This tutorial makes use of this feature to continue the training in the Model Parallel training and Spatial Parallel training sections.\n"
+ "When a DTensor checkpoint is restored, `Layout`s of variables can be different from when the checkpoint is saved. That is, saving DTensor models is layout- and mesh-agnostic, and only affects the efficiency of sharded saving. You can save a DTensor model with one mesh and layout and restore it on a different mesh and layout. This tutorial makes use of this feature to continue the training in the Model Parallel training and Spatial Parallel training sections.\n"
]
},
{
@@ -712,15 +712,15 @@
"source": [
"CHECKPOINT_DIR = tempfile.mkdtemp()\n",
"\n",
- "def start_checkpoint_manager(mesh, model):\n",
- " ckpt = dtensor.DTensorCheckpoint(mesh, root=model)\n",
+ "def start_checkpoint_manager(model):\n",
+ " ckpt = tf.train.Checkpoint(root=model)\n",
" manager = tf.train.CheckpointManager(ckpt, CHECKPOINT_DIR, max_to_keep=3)\n",
"\n",
" if manager.latest_checkpoint:\n",
" print(\"Restoring a checkpoint\")\n",
" ckpt.restore(manager.latest_checkpoint).assert_consumed()\n",
" else:\n",
- " print(\"new training\")\n",
+ " print(\"New training\")\n",
" return manager\n"
]
},
@@ -746,7 +746,7 @@
"outputs": [],
"source": [
"num_epochs = 2\n",
- "manager = start_checkpoint_manager(mesh, model)\n",
+ "manager = start_checkpoint_manager(model)\n",
"\n",
"for epoch in range(num_epochs):\n",
" step = 0\n",
@@ -839,7 +839,7 @@
"outputs": [],
"source": [
"num_epochs = 2\n",
- "manager = start_checkpoint_manager(mesh, model)\n",
+ "manager = start_checkpoint_manager(model)\n",
"\n",
"for epoch in range(num_epochs):\n",
" step = 0\n",
@@ -932,7 +932,7 @@
"source": [
"num_epochs = 2\n",
"\n",
- "manager = start_checkpoint_manager(mesh, model)\n",
+ "manager = start_checkpoint_manager(model)\n",
"for epoch in range(num_epochs):\n",
" step = 0\n",
" metrics = {'epoch': epoch}\n",
@@ -956,11 +956,9 @@
"source": [
"## SavedModel and DTensor\n",
"\n",
- "The integration of DTensor and SavedModel is still under development. This section only describes the current status quo for TensorFlow 2.9.0.\n",
- "\n",
- "As of TensorFlow 2.9.0, `tf.saved_model` only accepts DTensor models with fully replicated variables.\n",
+ "The integration of DTensor and SavedModel is still under development. \n",
"\n",
- "As a workaround, you can convert a DTensor model to a fully replicated one by reloading a checkpoint. However, after a model is saved, all DTensor annotations are lost and the saved signatures can only be used with regular Tensors, not DTensors."
+ "As of TensorFlow `2.11`, `tf.saved_model` can save sharded and replicated DTensor models, and saving will do an efficient sharded save on different devices of the mesh. However, after a model is saved, all DTensor annotations are lost and the saved signatures can only be used with regular Tensors, not DTensors."
]
},
{
@@ -975,7 +973,7 @@
"mlp = MLP([dtensor.Layout([dtensor.UNSHARDED, dtensor.UNSHARDED], mesh), \n",
" dtensor.Layout([dtensor.UNSHARDED, dtensor.UNSHARDED], mesh)])\n",
"\n",
- "manager = start_checkpoint_manager(mesh, mlp)\n",
+ "manager = start_checkpoint_manager(mlp)\n",
"\n",
"model_for_saving = tf.keras.Sequential([\n",
" text_vectorization,\n",
@@ -1060,7 +1058,6 @@
"colab": {
"collapsed_sections": [],
"name": "dtensor_ml_tutorial.ipynb",
- "provenance": [],
"toc_visible": true
},
"kernelspec": {
diff --git a/site/en/tutorials/distribute/input.ipynb b/site/en/tutorials/distribute/input.ipynb
index 073cdd66725..4500f95bf7e 100644
--- a/site/en/tutorials/distribute/input.ipynb
+++ b/site/en/tutorials/distribute/input.ipynb
@@ -73,7 +73,7 @@
"This guide will show you the different ways in which you can create distributed dataset and iterators using `tf.distribute` APIs. Additionally, the following topics will be covered:\n",
"- Usage, sharding and batching options when using `tf.distribute.Strategy.experimental_distribute_dataset` and `tf.distribute.Strategy.distribute_datasets_from_function`.\n",
"- Different ways in which you can iterate over the distributed dataset.\n",
- "- Differences between `tf.distribute.Strategy.experimental_distribute_dataset`/`tf.distribute.Strategy.distribute_datasets_from_function` APIs and `tf.data` APIs as well any limitations that users may come across in their usage.\n",
+ "- Differences between `tf.distribute.Strategy.experimental_distribute_dataset`/`tf.distribute.Strategy.distribute_datasets_from_function` APIs and `tf.data` APIs as well as any limitations that users may come across in their usage.\n",
"\n",
"This guide does not cover usage of distributed input with Keras APIs."
]
@@ -733,7 +733,7 @@
"id": "hNCYZ9L-BD2R"
},
"source": [
- "### Model.fit"
+ "### Preprocessing with `Model.fit`"
]
},
{
@@ -792,7 +792,7 @@
"id": "imZLQUOYBJyW"
},
"source": [
- "### Custom Training Loop"
+ "### Preprocessing with a custom training loop"
]
},
{
@@ -1096,7 +1096,6 @@
"colab": {
"collapsed_sections": [],
"name": "input.ipynb",
- "provenance": [],
"toc_visible": true
},
"kernelspec": {
diff --git a/site/en/tutorials/distribute/keras.ipynb b/site/en/tutorials/distribute/keras.ipynb
index 31a0dd7f386..b96656d4436 100644
--- a/site/en/tutorials/distribute/keras.ipynb
+++ b/site/en/tutorials/distribute/keras.ipynb
@@ -280,7 +280,7 @@
"id": "4xsComp8Kz5H"
},
"source": [
- "## Create the model"
+ "## Create the model and instantiate the optimizer"
]
},
{
@@ -310,10 +310,21 @@
" ])\n",
"\n",
" model.compile(loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n",
- " optimizer=tf.keras.optimizers.Adam(),\n",
+ " optimizer=tf.keras.optimizers.Adam(learning_rate=0.001),\n",
" metrics=['accuracy'])"
]
},
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "DCDKFcNJzdcd"
+ },
+ "source": [
+ "For this toy example with the MNIST dataset, you will be using the Adam optimizer's default learning rate of 0.001.\n",
+ "\n",
+ "For larger datasets, the key benefit of distributed training is to learn more in each training step, because each step processes more training data in parallel, which allows for a larger learning rate (within the limits of the model and dataset)."
+ ]
+ },
{
"cell_type": "markdown",
"metadata": {
@@ -352,7 +363,7 @@
"# Define the checkpoint directory to store the checkpoints.\n",
"checkpoint_dir = './training_checkpoints'\n",
"# Define the name of the checkpoint files.\n",
- "checkpoint_prefix = os.path.join(checkpoint_dir, \"ckpt_{epoch}\")"
+ "checkpoint_prefix = os.path.join(checkpoint_dir, \"ckpt_{epoch:04d}.weights.h5\")"
]
},
{
@@ -385,8 +396,7 @@
"# Define a callback for printing the learning rate at the end of each epoch.\n",
"class PrintLR(tf.keras.callbacks.Callback):\n",
" def on_epoch_end(self, epoch, logs=None):\n",
- " print('\\nLearning rate for epoch {} is {}'.format(",
- " epoch + 1, model.optimizer.lr.numpy()))"
+ " print('\\nLearning rate for epoch {} is {}'.format(epoch + 1, model.optimizer.learning_rate.numpy()))"
]
},
{
@@ -476,7 +486,10 @@
},
"outputs": [],
"source": [
- "model.load_weights(tf.train.latest_checkpoint(checkpoint_dir))\n",
+ "import pathlib\n",
+ "latest_checkpoint = sorted(pathlib.Path(checkpoint_dir).glob('*'))[-1]\n",
+ "\n",
+ "model.load_weights(latest_checkpoint)\n",
"\n",
"eval_loss, eval_acc = model.evaluate(eval_dataset)\n",
"\n",
@@ -529,7 +542,7 @@
"id": "kBLlogrDvMgg"
},
"source": [
- "## Export to SavedModel"
+ "## Save the model"
]
},
{
@@ -538,7 +551,7 @@
"id": "Xa87y_A0vRma"
},
"source": [
- "Export the graph and the variables to the platform-agnostic SavedModel format using Keras `Model.save`. After your model is saved, you can load it with or without the `Strategy.scope`."
+ "Save the model to a `.keras` zip archive using `Model.save`. After your model is saved, you can load it with or without the `Strategy.scope`."
]
},
{
@@ -549,7 +562,7 @@
},
"outputs": [],
"source": [
- "path = 'saved_model/'"
+ "path = 'my_model.keras'"
]
},
{
@@ -560,7 +573,7 @@
},
"outputs": [],
"source": [
- "model.save(path, save_format='tf')"
+ "model.save(path)"
]
},
{
diff --git a/site/en/tutorials/distribute/multi_worker_with_ctl.ipynb b/site/en/tutorials/distribute/multi_worker_with_ctl.ipynb
index aa53a89bb12..6932fd7910b 100644
--- a/site/en/tutorials/distribute/multi_worker_with_ctl.ipynb
+++ b/site/en/tutorials/distribute/multi_worker_with_ctl.ipynb
@@ -227,13 +227,18 @@
" return dataset\n",
"\n",
"def build_cnn_model():\n",
+ " regularizer = tf.keras.regularizers.L2(1e-5)\n",
" return tf.keras.Sequential([\n",
" tf.keras.Input(shape=(28, 28)),\n",
" tf.keras.layers.Reshape(target_shape=(28, 28, 1)),\n",
- " tf.keras.layers.Conv2D(32, 3, activation='relu'),\n",
+ " tf.keras.layers.Conv2D(32, 3,\n",
+ " activation='relu',\n",
+ " kernel_regularizer=regularizer),\n",
" tf.keras.layers.Flatten(),\n",
- " tf.keras.layers.Dense(128, activation='relu'),\n",
- " tf.keras.layers.Dense(10)\n",
+ " tf.keras.layers.Dense(128,\n",
+ " activation='relu',\n",
+ " kernel_regularizer=regularizer),\n",
+ " tf.keras.layers.Dense(10, kernel_regularizer=regularizer)\n",
" ])"
]
},
@@ -520,11 +525,13 @@
" x, y = inputs\n",
" with tf.GradientTape() as tape:\n",
" predictions = multi_worker_model(x, training=True)\n",
- " per_batch_loss = tf.keras.losses.SparseCategoricalCrossentropy(\n",
+ " per_example_loss = tf.keras.losses.SparseCategoricalCrossentropy(\n",
" from_logits=True,\n",
" reduction=tf.keras.losses.Reduction.NONE)(y, predictions)\n",
- " loss = tf.nn.compute_average_loss(\n",
- " per_batch_loss, global_batch_size=global_batch_size)\n",
+ " loss = tf.nn.compute_average_loss(per_example_loss)\n",
+ " model_losses = multi_worker_model.losses\n",
+ " if model_losses:\n",
+ " loss += tf.nn.scale_regularization_loss(tf.add_n(model_losses))\n",
"\n",
" grads = tape.gradient(loss, multi_worker_model.trainable_variables)\n",
" optimizer.apply_gradients(\n",
@@ -604,7 +611,7 @@
" name='step_in_epoch')\n",
"task_type, task_id = (strategy.cluster_resolver.task_type,\n",
" strategy.cluster_resolver.task_id)\n",
- "# Normally, you don't need to manually instantiate a `ClusterSpec`, but in this \n",
+ "# Normally, you don't need to manually instantiate a `ClusterSpec`, but in this\n",
"# illustrative example you did not set `'TF_CONFIG'` before initializing the\n",
"# strategy. Check out the next section for \"real-world\" usage.\n",
"cluster_spec = tf.train.ClusterSpec(tf_config['cluster'])\n",
@@ -739,7 +746,7 @@
" or (task_type == 'worker'\n",
" and task_id == 0\n",
" and 'chief' not in cluster_spec.as_dict()))\n",
- " \n",
+ "\n",
"def _get_temp_dir(dirpath, task_id):\n",
" base_dirpath = 'workertemp_' + str(task_id)\n",
" temp_dir = os.path.join(dirpath, base_dirpath)\n",
@@ -763,7 +770,7 @@
" multi_worker_model = mnist.build_cnn_model()\n",
"\n",
" multi_worker_dataset = strategy.distribute_datasets_from_function(\n",
- " lambda input_context: mnist.dataset_fn(global_batch_size, input_context)) \n",
+ " lambda input_context: mnist.dataset_fn(global_batch_size, input_context))\n",
" optimizer = tf.keras.optimizers.RMSprop(learning_rate=0.001)\n",
" train_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(\n",
" name='train_accuracy')\n",
@@ -777,11 +784,13 @@
" x, y = inputs\n",
" with tf.GradientTape() as tape:\n",
" predictions = multi_worker_model(x, training=True)\n",
- " per_batch_loss = tf.keras.losses.SparseCategoricalCrossentropy(\n",
+ " per_example_loss = tf.keras.losses.SparseCategoricalCrossentropy(\n",
" from_logits=True,\n",
" reduction=tf.keras.losses.Reduction.NONE)(y, predictions)\n",
- " loss = tf.nn.compute_average_loss(\n",
- " per_batch_loss, global_batch_size=global_batch_size)\n",
+ " loss = tf.nn.compute_average_loss(per_example_loss)\n",
+ " model_losses = multi_worker_model.losses\n",
+ " if model_losses:\n",
+ " loss += tf.nn.scale_regularization_loss(tf.add_n(model_losses))\n",
"\n",
" grads = tape.gradient(loss, multi_worker_model.trainable_variables)\n",
" optimizer.apply_gradients(\n",
@@ -831,7 +840,7 @@
" train_loss = total_loss / num_batches\n",
" print('Epoch: %d, accuracy: %f, train_loss: %f.'\n",
" %(epoch.numpy(), train_accuracy.result(), train_loss))\n",
- " \n",
+ "\n",
" train_accuracy.reset_states()\n",
"\n",
" checkpoint_manager.save()\n",
@@ -1079,7 +1088,6 @@
"colab": {
"collapsed_sections": [],
"name": "multi_worker_with_ctl.ipynb",
- "provenance": [],
"toc_visible": true
},
"kernelspec": {
diff --git a/site/en/tutorials/distribute/multi_worker_with_estimator.ipynb b/site/en/tutorials/distribute/multi_worker_with_estimator.ipynb
index b4fffa60fb4..3a19e8fbe2c 100644
--- a/site/en/tutorials/distribute/multi_worker_with_estimator.ipynb
+++ b/site/en/tutorials/distribute/multi_worker_with_estimator.ipynb
@@ -186,7 +186,7 @@
"\n",
"There are two components of `TF_CONFIG`: `cluster` and `task`. `cluster` provides information about the entire cluster, namely the workers and parameter servers in the cluster. `task` provides information about the current task. The first component `cluster` is the same for all workers and parameter servers in the cluster, and the second component `task` is different on each worker and parameter server and specifies its own `type` and `index`. In this example, the task `type` is `worker` and the task `index` is `0`.\n",
"\n",
- "For illustration purposes, this tutorial shows how to set a `TF_CONFIG` with 2 workers on `localhost`. In practice, you would create multiple workers on an external IP address and port, and set `TF_CONFIG` on each worker appropriately, i.e. modify the task `index`.\n",
+ "For illustration purposes, this tutorial shows how to set a `TF_CONFIG` with 2 workers on `localhost`. In practice, you would create multiple workers on an external IP address and port, and set `TF_CONFIG` on each worker appropriately, i.e., modify the task `index`.\n",
"\n",
"Warning: *Do not execute the following code in Colab.* TensorFlow's runtime will attempt to create a gRPC server at the specified IP address and port, which will likely fail. See the [keras version](multi_worker_with_keras.ipynb) of this tutorial for an example of how you can test run multiple workers on a single machine.\n",
"\n",
@@ -351,7 +351,6 @@
"Tce3stUlHN0L"
],
"name": "multi_worker_with_estimator.ipynb",
- "provenance": [],
"toc_visible": true
},
"kernelspec": {
diff --git a/site/en/tutorials/distribute/multi_worker_with_keras.ipynb b/site/en/tutorials/distribute/multi_worker_with_keras.ipynb
index 753bdb8afe3..a3710223f17 100644
--- a/site/en/tutorials/distribute/multi_worker_with_keras.ipynb
+++ b/site/en/tutorials/distribute/multi_worker_with_keras.ipynb
@@ -1011,6 +1011,15 @@
"With that, you're now ready to save:"
]
},
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "XnToxeIcg_6O"
+ },
+ "source": [
+ "Deprecated: For Keras objects, it's recommended to use the new high-level `.keras` format and `tf.keras.Model.export`, as demonstrated in the guide [here](https://www.tensorflow.org/guide/keras/save_and_serialize). The low-level SavedModel format continues to be supported for existing code."
+ ]
+ },
{
"cell_type": "code",
"execution_count": null,
@@ -1275,7 +1284,6 @@
],
"metadata": {
"colab": {
- "collapsed_sections": [],
"name": "multi_worker_with_keras.ipynb",
"toc_visible": true
},
diff --git a/site/en/tutorials/distribute/parameter_server_training.ipynb b/site/en/tutorials/distribute/parameter_server_training.ipynb
index f997cc4e159..820e98e9bd3 100644
--- a/site/en/tutorials/distribute/parameter_server_training.ipynb
+++ b/site/en/tutorials/distribute/parameter_server_training.ipynb
@@ -132,10 +132,9 @@
"\n",
"With custom training loops, the `tf.distribute.coordinator.ClusterCoordinator` class is the key component used for the coordinator.\n",
"\n",
- "- The `ClusterCoordinator` class needs to work in conjunction with a `tf.distribute.Strategy` object.\n",
+ "- The `ClusterCoordinator` class needs to work in conjunction with a `tf.distribute.ParameterServerStrategy` object.\n",
"- This `tf.distribute.Strategy` object is needed to provide the information of the cluster and is used to define a training step, as demonstrated in [Custom training with tf.distribute.Strategy](custom_training.ipynb).\n",
"- The `ClusterCoordinator` object then dispatches the execution of these training steps to remote workers.\n",
- "- For parameter server training, the `ClusterCoordinator` needs to work with a `tf.distribute.ParameterServerStrategy`.\n",
"\n",
"The most important API provided by the `ClusterCoordinator` object is `schedule`:\n",
"\n",
@@ -381,10 +380,10 @@
"\n",
"Keras `Model.fit` with `tf.distribute.ParameterServerStrategy` can take input data in the form of a `tf.data.Dataset`, `tf.distribute.DistributedDataset`, or a `tf.keras.utils.experimental.DatasetCreator`, with `Dataset` being the recommended option for ease of use. If you encounter memory issues using `Dataset`, however, you may need to use `DatasetCreator` with a callable `dataset_fn` argument (refer to the `tf.keras.utils.experimental.DatasetCreator` API documentation for details).\n",
"\n",
- "If you transform your dataset into a `tf.data.Dataset`, you should use `Dataset.shuffle` and `Dataset.repeat`, as demonstrated in the code example below. \n",
+ "If you transform your dataset into a `tf.data.Dataset`, you should use `Dataset.shuffle` and `Dataset.repeat`, as demonstrated in the code example below.\n",
"\n",
"- Keras `Model.fit` with parameter server training assumes that each worker receives the same dataset, except when it is shuffled differently. Therefore, by calling `Dataset.shuffle`, you ensure more even iterations over the data.\n",
- "- Because workers do not synchronize, they may finish processing their datasets at different times. Therefore, the easiest way to define epochs with parameter server training is to use `Dataset.repeat`—which repeats a dataset indefinitely when called without an argument—and specify the `steps_per_epoch` argument in the `Model.fit` call. \n",
+ "- Because workers do not synchronize, they may finish processing their datasets at different times. Therefore, the easiest way to define epochs with parameter server training is to use `Dataset.repeat`—which repeats a dataset indefinitely when called without an argument—and specify the `steps_per_epoch` argument in the `Model.fit` call.\n",
"\n",
"Refer to the \"Training workflows\" section of the [tf.data guide](../../guide/data.ipynb) for more details on `shuffle` and `repeat`."
]
@@ -438,7 +437,7 @@
"with strategy.scope():\n",
" model = tf.keras.models.Sequential([tf.keras.layers.Dense(10)])\n",
"\n",
- " model.compile(tf.keras.optimizers.SGD(), loss=\"mse\", steps_per_execution=10)"
+ " model.compile(tf.keras.optimizers.legacy.SGD(), loss=\"mse\", steps_per_execution=10)"
]
},
{
@@ -657,10 +656,13 @@
" emb_layer = tf.keras.layers.Embedding(\n",
" input_dim=len(feature_lookup_layer.get_vocabulary()), output_dim=16384)\n",
" emb_output = tf.reduce_mean(emb_layer(model_input), axis=1)\n",
- " dense_output = tf.keras.layers.Dense(units=1, activation=\"sigmoid\")(emb_output)\n",
+ " dense_output = tf.keras.layers.Dense(\n",
+ " units=1, activation=\"sigmoid\",\n",
+ " kernel_regularizer=tf.keras.regularizers.L2(1e-4),\n",
+ " )(emb_output)\n",
" model = tf.keras.Model({\"features\": model_input}, dense_output)\n",
"\n",
- " optimizer = tf.keras.optimizers.RMSprop(learning_rate=0.1)\n",
+ " optimizer = tf.keras.optimizers.legacy.RMSprop(learning_rate=0.1)\n",
" accuracy = tf.keras.metrics.Accuracy()"
]
},
@@ -716,7 +718,10 @@
" per_example_loss = tf.keras.losses.BinaryCrossentropy(\n",
" reduction=tf.keras.losses.Reduction.NONE)(labels, pred)\n",
" loss = tf.nn.compute_average_loss(per_example_loss)\n",
- " gradients = tape.gradient(loss, model.trainable_variables)\n",
+ " model_losses = model.losses\n",
+ " if model_losses:\n",
+ " loss += tf.nn.scale_regularization_loss(tf.add_n(model_losses))\n",
+ " gradients = tape.gradient(loss, model.trainable_variables)\n",
"\n",
" optimizer.apply_gradients(zip(gradients, model.trainable_variables))\n",
"\n",
@@ -735,7 +740,7 @@
"id": "rvrYQUeYiLNy"
},
"source": [
- "In the above training step function, calling `Strategy.run` and `Strategy.reduce` in the `step_fn` can support multiple GPUs per worker. If the workers have GPUs allocated, `Strategy.run` will distribute the datasets on multiple replicas.\n"
+ "In the above training step function, calling `Strategy.run` and `Strategy.reduce` in the `step_fn` can support multiple GPUs per worker. If the workers have GPUs allocated, `Strategy.run` will distribute the datasets on multiple replicas (GPUs). Their parallel calls to `tf.nn.compute_average_loss()` compute the average of the loss across the replicas (GPUs) of one worker, independent of the total number of workers."
]
},
{
@@ -883,7 +888,7 @@
"source": [
"## Evaluation\n",
"\n",
- "The two main approaches to performing evaluation with `tf.distribute.ParameterServerStrategy` training are inline evaluation and sidecar evaluation. Each has its own pros and cons as described below. The inline evaluation method is recommended if you don't have a preference."
+ "The two main approaches to performing evaluation with `tf.distribute.ParameterServerStrategy` training are inline evaluation and sidecar evaluation. Each has its own pros and cons as described below. The inline evaluation method is recommended if you don't have a preference. For users using `Model.fit`, `Model.evaluate` uses inline (distributed) evaluation under the hood."
]
},
{
@@ -985,7 +990,23 @@
"id": "cKrQktZX5z7a"
},
"source": [
- "Note: The `schedule` and `join` methods of `tf.distribute.coordinator.ClusterCoordinator` don’t support visitation guarantees or exactly-once semantics. In other words, there is no guarantee that all evaluation examples in a dataset will be evaluated exactly once; some may not be visited and some may be evaluated multiple times. The `tf.data` service API can be used to provide exactly-once visitation for evaluation when using `ParameterServerStrategy` (refer to the _Dynamic Sharding_ section of the `tf.data.experimental.service` API documentation)."
+ "#### Enabling exactly-once evaluation\n",
+ "\n",
+ "\n",
+ "The `schedule` and `join` methods of `tf.distribute.coordinator.ClusterCoordinator` don’t support visitation guarantees or exactly-once semantics by default. In other words, in the above example there is no guarantee that all evaluation examples in a dataset will be evaluated exactly once; some may not be visited and some may be evaluated multiple times.\n",
+ "\n",
+ "Exactly-once evaluation may be preferred to reduce the variance of evaluation across epochs, and improve model selection done via early stopping, hyperparameter tuning, or other methods. There are different ways to enable exactly-once evaluation:\n",
+ "\n",
+ "- With a `Model.fit/.evaluate` workflow, it can be enabled by adding an argument to `Model.compile`. Refer to docs for the `pss_evaluation_shards` argument.\n",
+ "- The `tf.data` service API can be used to provide exactly-once visitation for evaluation when using `ParameterServerStrategy` (refer to the _Dynamic Sharding_ section of the `tf.data.experimental.service` API documentation).\n",
+ "- [Sidecar evaluation](#sidecar_evaluation) provides exactly-once evaluation by default, since the evaluation happens on a single machine. However this can be much slower than performing evaluation distributed across many workers.\n",
+ "\n",
+ "The first option, using `Model.compile`, is the suggested solution for most users.\n",
+ "\n",
+ "Exactly-once evaluation has some limitations:\n",
+ "\n",
+ "- It is not supported to write a custom distributed evaluation loop with an exactly-once visitation guarantee. File a GitHub issue if you need support for this.\n",
+ "- It cannot automatically handle computation of metrics that use the `Layer.add_metric` API. These should be excluded from evaluation, or reworked into `Metric` objects."
]
},
{
@@ -997,7 +1018,7 @@
"### Sidecar evaluation\n",
"\n",
"\n",
- "Another method for defining and running an evaluation loop in `tf.distribute.ParameterServerStrategy` training is called _sidecar evaluation_, in which you create a dedicated evaluator task that repeatedly reads checkpoints and runs evaluation on the latest checkpoint (refer to [this guide](../../guide/checkpoint.ipynb) for more details on checkpointing). The chief and worker tasks do not spend any time on evaluation, so for a fixed number of iterations the overall training time should be shorter than using other evaluation methods. However, it requires an additional evaluator task and periodic checkpointing to trigger evaluation."
+ "Another method for defining and running an evaluation loop in `tf.distribute.ParameterServerStrategy` training is called _sidecar evaluation_, in which you create a dedicated evaluator task that repeatedly reads checkpoints and runs evaluation on the latest checkpoint (refer to [this guide](../../guide/checkpoint.ipynb) for more details on checkpointing). The coordinator and worker tasks do not spend any time on evaluation, so for a fixed number of iterations the overall training time should be shorter than using other evaluation methods. However, it requires an additional evaluator task and periodic checkpointing to trigger evaluation."
]
},
{
@@ -1007,10 +1028,10 @@
},
"source": [
"To write an evaluation loop for sidecar evaluation, you have two\n",
- "options: \n",
+ "options:\n",
"\n",
"1. Use the `tf.keras.utils.SidecarEvaluator` API.\n",
- "2. Create a custom evaluation loop. \n",
+ "2. Create a custom evaluation loop.\n",
"\n",
"Refer to the `tf.keras.utils.SidecarEvaluator` API documentation for more details on option 1."
]
@@ -1277,7 +1298,11 @@
"One common reason is that the parameter servers have unbalanced load and some heavily-loaded parameter servers have reached capacity. There can also be multiple root causes. Some simple methods to mitigate this issue are to:\n",
"\n",
"1. Shard your large model variables via specifying a `variable_partitioner` when constructing a `ParameterServerStrategy`.\n",
- "2. Avoid creating a hotspot variable that is required by all parameter servers in a single step if possible. For example, use a constant learning rate or subclass `tf.keras.optimizers.schedules.LearningRateSchedule` in optimizers since the default behavior is that the learning rate will become a variable placed on a particular parameter server and requested by all other parameter servers in each step.\n",
+ "2. Avoid creating a hotspot variable that is required by all parameter servers in a single step, by both:\n",
+ "\n",
+ " 1) Using a constant learning rate or subclass `tf.keras.optimizers.schedules.LearningRateSchedule` in optimizers. This is because the default behavior is that the learning rate will become a variable placed on a particular parameter server, and requested by all other parameter servers in each step); and\n",
+ "\n",
+ " 2) Using a `tf.keras.optimizers.legacy.Optimizer` (the standard `tf.keras.optimizers.Optimizer`s could still lead to hotspot variables).\n",
"3. Shuffle your large vocabularies before passing them to Keras preprocessing layers.\n",
"\n",
"Another possible reason for performance issues is the coordinator. The implementation of `schedule`/`join` is Python-based and thus may have threading overhead. Also, the latency between the coordinator and the workers can be large. If this is the case:\n",
@@ -1348,7 +1373,7 @@
"source": [
"### Custom training loop specifics\n",
"\n",
- "- `ClusterCoordinator.schedule` doesn't support visitation guarantees for a dataset.\n",
+ "- `ClusterCoordinator.schedule` doesn't support visitation guarantees for a dataset in general, although a visitation guarantee for evaluation is possible through `Model.fit/.evaluate`. See [Enabling exactly-once evaluation](#exactly_once_evaluation).\n",
"- When `ClusterCoordinator.create_per_worker_dataset` is used with a callable as input, the whole dataset must be created inside the function passed to it.\n",
"- `tf.data.Options` is ignored in a dataset created by `ClusterCoordinator.create_per_worker_dataset`."
]
@@ -1357,9 +1382,7 @@
"metadata": {
"accelerator": "GPU",
"colab": {
- "collapsed_sections": [],
"name": "parameter_server_training.ipynb",
- "provenance": [],
"toc_visible": true
},
"kernelspec": {
diff --git a/site/en/tutorials/distribute/save_and_load.ipynb b/site/en/tutorials/distribute/save_and_load.ipynb
index 517c43e5969..c53a9b8bf0b 100644
--- a/site/en/tutorials/distribute/save_and_load.ipynb
+++ b/site/en/tutorials/distribute/save_and_load.ipynb
@@ -77,7 +77,6 @@
"\n",
"Caution: TensorFlow models are code and it is important to be careful with untrusted code. Learn more in [Using TensorFlow securely](https://github.com/tensorflow/tensorflow/blob/master/SECURITY.md).\n",
"\n"
-
]
},
{
@@ -190,8 +189,8 @@
"Now that you have a simple model to work with, let's explore the saving/loading APIs. \n",
"There are two kinds of APIs available:\n",
"\n",
- "* High-level (Keras): `Model.save` and `tf.keras.models.load_model`\n",
- "* Low-level: `tf.saved_model.save` and `tf.saved_model.load`\n"
+ "* High-level (Keras): `Model.save` and `tf.keras.models.load_model` (`.keras` zip archive format)\n",
+ "* Low-level: `tf.saved_model.save` and `tf.saved_model.load` (TF SavedModel format)\n"
]
},
{
@@ -220,7 +219,7 @@
},
"outputs": [],
"source": [
- "keras_model_path = '/tmp/keras_save'\n",
+ "keras_model_path = '/tmp/keras_save.keras'\n",
"model.save(keras_model_path)"
]
},
@@ -251,7 +250,7 @@
"id": "gYAnskzorda-"
},
"source": [
- "After restoring the model, you can continue training on it, even without needing to call `Model.compile` again, since it was already compiled before saving. The model is saved in TensorFlow's standard `SavedModel` proto format. For more information, please refer to [the guide to `SavedModel` format](../../guide/saved_model.ipynb).\n",
+ "After restoring the model, you can continue training on it, even without needing to call `Model.compile` again, since it was already compiled before saving. The model is saved a Keras zip archive format, marked by the `.keras` extension. For more information, please refer to [the guide on Keras saving](https://www.tensorflow.org/guide/keras/save_and_serialize).\n",
"\n",
"Now, restore the model and train it using a `tf.distribute.Strategy`:"
]
@@ -465,12 +464,12 @@
"model = get_model()\n",
"\n",
"# Saving the model using Keras `Model.save`\n",
- "model.save(keras_model_path)\n",
+ "model.save(saved_model_path)\n",
"\n",
"another_strategy = tf.distribute.MirroredStrategy()\n",
"# Loading the model using the lower-level API\n",
"with another_strategy.scope():\n",
- " loaded = tf.saved_model.load(keras_model_path)"
+ " loaded = tf.saved_model.load(saved_model_path)"
]
},
{
@@ -554,7 +553,7 @@
"\n",
"my_model = SubclassedModel()\n",
"try:\n",
- " my_model.save(keras_model_path)\n",
+ " my_model.save(saved_model_path)\n",
"except ValueError as e:\n",
" print(f'{type(e).__name__}: ', *e.args)"
]
@@ -635,7 +634,7 @@
"my_model.fit(dataset, epochs=2)\n",
"\n",
"print(my_model.save_spec() is None)\n",
- "my_model.save(keras_model_path)"
+ "my_model.save(saved_model_path)"
]
}
],
diff --git a/site/en/tutorials/estimator/keras_model_to_estimator.ipynb b/site/en/tutorials/estimator/keras_model_to_estimator.ipynb
index e41380908f5..3fc0ae384b2 100644
--- a/site/en/tutorials/estimator/keras_model_to_estimator.ipynb
+++ b/site/en/tutorials/estimator/keras_model_to_estimator.ipynb
@@ -68,7 +68,7 @@
"id": "Dhcq8Ds4mCtm"
},
"source": [
- "> Warning: Estimators are not recommended for new code. Estimators run `v1.Session`-style code which is more difficult to write correctly, and can behave unexpectedly, especially when combined with TF 2 code. Estimators do fall under our [compatibility guarantees](https://tensorflow.org/guide/versions), but will receive no fixes other than security vulnerabilities. See the [migration guide](https://tensorflow.org/guide/migrate) for details."
+ "> Warning: TensorFlow 2.15 included the final release of the `tf-estimator` package. Estimators will not be available in TensorFlow 2.16 or after. See the [migration guide](https://tensorflow.org/guide/migrate/migrating_estimator) for more information about how to convert off of Estimators."
]
},
{
@@ -272,7 +272,6 @@
"colab": {
"collapsed_sections": [],
"name": "keras_model_to_estimator.ipynb",
- "provenance": [],
"toc_visible": true
},
"kernelspec": {
diff --git a/site/en/tutorials/estimator/linear.ipynb b/site/en/tutorials/estimator/linear.ipynb
index ea46d41ede1..701e7a2c03f 100644
--- a/site/en/tutorials/estimator/linear.ipynb
+++ b/site/en/tutorials/estimator/linear.ipynb
@@ -61,7 +61,7 @@
"id": "JOccPOFMm5Tc"
},
"source": [
- "> Warning: Estimators are not recommended for new code. Estimators run `v1.Session`-style code which is more difficult to write correctly, and can behave unexpectedly, especially when combined with TF 2 code. Estimators do fall under our [compatibility guarantees](https://tensorflow.org/guide/versions), but will receive no fixes other than security vulnerabilities. See the [migration guide](https://tensorflow.org/guide/migrate) for details."
+ "> Warning: TensorFlow 2.15 included the final release of the `tf-estimator` package. Estimators will not be available in TensorFlow 2.16 or after. See the [migration guide](https://tensorflow.org/guide/migrate/migrating_estimator) for more information about how to convert off of Estimators."
]
},
{
@@ -293,14 +293,31 @@
"pd.concat([dftrain, y_train], axis=1).groupby('sex').survived.mean().plot(kind='barh').set_xlabel('% survive')"
]
},
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "qCHvgeorEsHa"
+ },
+ "source": [
+ "## Feature Engineering for the Model"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "Dhcq8Ds4mCtm"
+ },
+ "source": [
+ "> Warning: The tf.feature_columns module described in this tutorial is not recommended for new code. Keras preprocessing layers cover this functionality, for migration instructions see the [Migrating feature columns guide](https://www.tensorflow.org/guide/migrate/migrating_feature_columns). The tf.feature_columns module was designed for use with TF1 Estimators. It does fall under our [compatibility guarantees](https://tensorflow.org/guide/versions), but will receive no fixes other than security vulnerabilities."
+ ]
+ },
{
"cell_type": "markdown",
"metadata": {
"id": "VqDKQLZn8L-B"
},
"source": [
- "## Feature Engineering for the Model\n",
- "Estimators use a system called [feature columns](https://www.tensorflow.org/guide/feature_columns) to describe how the model should interpret each of the raw input features. An Estimator expects a vector of numeric inputs, and *feature columns* describe how the model should convert each feature.\n",
+ "Estimators use a system called [feature columns](https://www.tensorflow.org/tutorials/structured_data/feature_columns) to describe how the model should interpret each of the raw input features. An Estimator expects a vector of numeric inputs, and *feature columns* describe how the model should convert each feature.\n",
"\n",
"Selecting and crafting the right set of feature columns is key to learning an effective model. A feature column can be either one of the raw inputs in the original features `dict` (a *base feature column*), or any new columns created using transformations defined over one or multiple base columns (a *derived feature columns*).\n",
"\n",
@@ -583,7 +600,6 @@
"colab": {
"collapsed_sections": [],
"name": "linear.ipynb",
- "provenance": [],
"toc_visible": true
},
"kernelspec": {
diff --git a/site/en/tutorials/estimator/premade.ipynb b/site/en/tutorials/estimator/premade.ipynb
index a34096ea2b8..dc81847c7cd 100644
--- a/site/en/tutorials/estimator/premade.ipynb
+++ b/site/en/tutorials/estimator/premade.ipynb
@@ -68,7 +68,7 @@
"id": "stQiPWL6ni6_"
},
"source": [
- "> Warning: Estimators are not recommended for new code. Estimators run `v1.Session`-style code which is more difficult to write correctly, and can behave unexpectedly, especially when combined with TF 2 code. Estimators do fall under [compatibility guarantees](https://tensorflow.org/guide/versions), but will receive no fixes other than security vulnerabilities. See the [migration guide](https://tensorflow.org/guide/migrate) for details."
+ "> Warning: TensorFlow 2.15 included the final release of the `tf-estimator` package. Estimators will not be available in TensorFlow 2.16 or after. See the [migration guide](https://tensorflow.org/guide/migrate/migrating_estimator) for more information about how to convert off of Estimators."
]
},
{
diff --git a/site/en/tutorials/generative/autoencoder.ipynb b/site/en/tutorials/generative/autoencoder.ipynb
index b68b8e934a7..1b2a6fcd2a8 100644
--- a/site/en/tutorials/generative/autoencoder.ipynb
+++ b/site/en/tutorials/generative/autoencoder.ipynb
@@ -6,9 +6,16 @@
"id": "Ndo4ERqnwQOU"
},
"source": [
- "##### Copyright 2020 The TensorFlow Authors."
+ "##### Copyright 2024 The TensorFlow Authors."
]
},
+ {
+ "metadata": {
+ "id": "13rwRG5Jec7n"
+ },
+ "cell_type": "markdown",
+ "source": []
+ },
{
"cell_type": "code",
"execution_count": null,
@@ -76,7 +83,7 @@
"source": [
"This tutorial introduces autoencoders with three examples: the basics, image denoising, and anomaly detection.\n",
"\n",
- "An autoencoder is a special type of neural network that is trained to copy its input to its output. For example, given an image of a handwritten digit, an autoencoder first encodes the image into a lower dimensional latent representation, then decodes the latent representation back to an image. An autoencoder learns to compress the data while minimizing the reconstruction error. \n",
+ "An autoencoder is a special type of neural network that is trained to copy its input to its output. For example, given an image of a handwritten digit, an autoencoder first encodes the image into a lower dimensional latent representation, then decodes the latent representation back to an image. An autoencoder learns to compress the data while minimizing the reconstruction error.\n",
"\n",
"To learn more about autoencoders, please consider reading chapter 14 from [Deep Learning](https://www.deeplearningbook.org/) by Ian Goodfellow, Yoshua Bengio, and Aaron Courville."
]
@@ -117,7 +124,7 @@
},
"source": [
"## Load the dataset\n",
- "To start, you will train the basic autoencoder using the Fashion MNIST dataset. Each image in this dataset is 28x28 pixels. "
+ "To start, you will train the basic autoencoder using the Fashion MNIST dataset. Each image in this dataset is 28x28 pixels."
]
},
{
@@ -159,27 +166,29 @@
},
"outputs": [],
"source": [
- "latent_dim = 64 \n",
- "\n",
"class Autoencoder(Model):\n",
- " def __init__(self, latent_dim):\n",
+ " def __init__(self, latent_dim, shape):\n",
" super(Autoencoder, self).__init__()\n",
- " self.latent_dim = latent_dim \n",
+ " self.latent_dim = latent_dim\n",
+ " self.shape = shape\n",
" self.encoder = tf.keras.Sequential([\n",
" layers.Flatten(),\n",
" layers.Dense(latent_dim, activation='relu'),\n",
" ])\n",
" self.decoder = tf.keras.Sequential([\n",
- " layers.Dense(784, activation='sigmoid'),\n",
- " layers.Reshape((28, 28))\n",
+ " layers.Dense(tf.math.reduce_prod(shape).numpy(), activation='sigmoid'),\n",
+ " layers.Reshape(shape)\n",
" ])\n",
"\n",
" def call(self, x):\n",
" encoded = self.encoder(x)\n",
" decoded = self.decoder(encoded)\n",
" return decoded\n",
- " \n",
- "autoencoder = Autoencoder(latent_dim) "
+ "\n",
+ "\n",
+ "shape = x_test.shape[1:]\n",
+ "latent_dim = 64\n",
+ "autoencoder = Autoencoder(latent_dim, shape)\n"
]
},
{
@@ -329,8 +338,8 @@
"outputs": [],
"source": [
"noise_factor = 0.2\n",
- "x_train_noisy = x_train + noise_factor * tf.random.normal(shape=x_train.shape) \n",
- "x_test_noisy = x_test + noise_factor * tf.random.normal(shape=x_test.shape) \n",
+ "x_train_noisy = x_train + noise_factor * tf.random.normal(shape=x_train.shape)\n",
+ "x_test_noisy = x_test + noise_factor * tf.random.normal(shape=x_test.shape)\n",
"\n",
"x_train_noisy = tf.clip_by_value(x_train_noisy, clip_value_min=0., clip_value_max=1.)\n",
"x_test_noisy = tf.clip_by_value(x_test_noisy, clip_value_min=0., clip_value_max=1.)"
@@ -655,7 +664,7 @@
"id": "wVcTBDo-CqFS"
},
"source": [
- "Plot a normal ECG. "
+ "Plot a normal ECG."
]
},
{
@@ -719,12 +728,12 @@
" layers.Dense(32, activation=\"relu\"),\n",
" layers.Dense(16, activation=\"relu\"),\n",
" layers.Dense(8, activation=\"relu\")])\n",
- " \n",
+ "\n",
" self.decoder = tf.keras.Sequential([\n",
" layers.Dense(16, activation=\"relu\"),\n",
" layers.Dense(32, activation=\"relu\"),\n",
" layers.Dense(140, activation=\"sigmoid\")])\n",
- " \n",
+ "\n",
" def call(self, x):\n",
" encoded = self.encoder(x)\n",
" decoded = self.decoder(encoded)\n",
@@ -761,8 +770,8 @@
},
"outputs": [],
"source": [
- "history = autoencoder.fit(normal_train_data, normal_train_data, \n",
- " epochs=20, \n",
+ "history = autoencoder.fit(normal_train_data, normal_train_data,\n",
+ " epochs=20,\n",
" batch_size=512,\n",
" validation_data=(test_data, test_data),\n",
" shuffle=True)"
@@ -906,7 +915,7 @@
"id": "uEGlA1Be50Nj"
},
"source": [
- "Note: There are other strategies you could use to select a threshold value above which test examples should be classified as anomalous, the correct approach will depend on your dataset. You can learn more with the links at the end of this tutorial. "
+ "Note: There are other strategies you could use to select a threshold value above which test examples should be classified as anomalous, the correct approach will depend on your dataset. You can learn more with the links at the end of this tutorial."
]
},
{
@@ -915,7 +924,7 @@
"id": "zpLSDAeb51D_"
},
"source": [
- "If you examine the reconstruction error for the anomalous examples in the test set, you'll notice most have greater reconstruction error than the threshold. By varing the threshold, you can adjust the [precision](https://developers.google.com/machine-learning/glossary#precision) and [recall](https://developers.google.com/machine-learning/glossary#recall) of your classifier. "
+ "If you examine the reconstruction error for the anomalous examples in the test set, you'll notice most have greater reconstruction error than the threshold. By varing the threshold, you can adjust the [precision](https://developers.google.com/machine-learning/glossary#precision) and [recall](https://developers.google.com/machine-learning/glossary#recall) of your classifier."
]
},
{
@@ -990,8 +999,18 @@
"metadata": {
"accelerator": "GPU",
"colab": {
- "collapsed_sections": [],
- "name": "autoencoder.ipynb",
+ "gpuType": "T4",
+ "private_outputs": true,
+ "provenance": [
+ {
+ "file_id": "17gKB2bKebV2DzoYIMFzyEXA5uDnwWOvT",
+ "timestamp": 1712793165979
+ },
+ {
+ "file_id": "https://github.com/tensorflow/docs/blob/master/site/en/tutorials/generative/autoencoder.ipynb",
+ "timestamp": 1712792176273
+ }
+ ],
"toc_visible": true
},
"kernelspec": {
diff --git a/site/en/tutorials/generative/cyclegan.ipynb b/site/en/tutorials/generative/cyclegan.ipynb
index 4dae5b77791..c872c4a28b5 100644
--- a/site/en/tutorials/generative/cyclegan.ipynb
+++ b/site/en/tutorials/generative/cyclegan.ipynb
@@ -154,7 +154,7 @@
"This is similar to what was done in [pix2pix](https://www.tensorflow.org/tutorials/generative/pix2pix#load_the_dataset)\n",
"\n",
"* In random jittering, the image is resized to `286 x 286` and then randomly cropped to `256 x 256`.\n",
- "* In random mirroring, the image is randomly flipped horizontally i.e. left to right."
+ "* In random mirroring, the image is randomly flipped horizontally i.e., left to right."
]
},
{
@@ -634,7 +634,7 @@
"source": [
"## Training\n",
"\n",
- "Note: This example model is trained for fewer epochs (40) than the paper (200) to keep training time reasonable for this tutorial. Predictions may be less accurate. "
+ "Note: This example model is trained for fewer epochs (10) than the paper (200) to keep training time reasonable for this tutorial. The generated images will have much lower quality."
]
},
{
@@ -645,7 +645,7 @@
},
"outputs": [],
"source": [
- "EPOCHS = 40"
+ "EPOCHS = 10"
]
},
{
@@ -830,7 +830,6 @@
"colab": {
"collapsed_sections": [],
"name": "cyclegan.ipynb",
- "provenance": [],
"toc_visible": true
},
"kernelspec": {
diff --git a/site/en/tutorials/generative/data_compression.ipynb b/site/en/tutorials/generative/data_compression.ipynb
index f8a5fe3d201..f756f088acd 100644
--- a/site/en/tutorials/generative/data_compression.ipynb
+++ b/site/en/tutorials/generative/data_compression.ipynb
@@ -104,8 +104,11 @@
},
"outputs": [],
"source": [
+ "%%bash\n",
"# Installs the latest version of TFC compatible with the installed TF version.\n",
- "!pip install tensorflow-compression~=$(pip show tensorflow | perl -p -0777 -e 's/.*Version: (\\d\\.\\d).*/\\1.0/sg')\n"
+ "\n",
+ "read MAJOR MINOR <<< \"$(pip show tensorflow | perl -p -0777 -e 's/.*Version: (\\d+)\\.(\\d+).*/\\1 \\2/sg')\"\n",
+ "pip install \"tensorflow-compression<$MAJOR.$(($MINOR+1))\"\n"
]
},
{
@@ -818,13 +821,67 @@
{
"cell_type": "markdown",
"metadata": {
- "id": "3ELLMAN1OwMQ"
+ "id": "3ELLMANN1OwMQ"
},
"source": [
"The strings begin to get much shorter now, on the order of one byte per digit. However, this comes at a cost. More digits are becoming unrecognizable.\n",
"\n",
"This demonstrates that this model is agnostic to human perceptions of error, it just measures the absolute deviation in terms of pixel values. To achieve a better perceived image quality, we would need to replace the pixel loss with a perceptual loss."
]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "v9cWHtH0LP_r"
+ },
+ "source": [
+ "## Use the decoder as a generative model.\n",
+ "\n",
+ "If we feed the decoder random bits, this will effectively sample from the distribution that the model learned to represent digits.\n",
+ "\n",
+ "First, re-instantiate the compressor/decompressor without a sanity check that would detect if the input string isn't completely decoded."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "qnic8YsM0_ke"
+ },
+ "outputs": [],
+ "source": [
+ "compressor, decompressor = make_mnist_codec(trainer, decode_sanity_check=False)\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "86uc9_Is1eeo"
+ },
+ "source": [
+ "Now, feed long enough random strings into the decompressor so that it can decode/sample digits from them."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "o4fP7BkqKCHY"
+ },
+ "outputs": [],
+ "source": [
+ "import os\n",
+ "\n",
+ "strings = tf.constant([os.urandom(8) for _ in range(16)])\n",
+ "samples = decompressor(strings)\n",
+ "\n",
+ "fig, axes = plt.subplots(4, 4, sharex=True, sharey=True, figsize=(5, 5))\n",
+ "axes = axes.ravel()\n",
+ "for i in range(len(axes)):\n",
+ " axes[i].imshow(tf.squeeze(samples[i]))\n",
+ " axes[i].axis(\"off\")\n",
+ "plt.subplots_adjust(wspace=0, hspace=0, left=0, right=1, bottom=0, top=1)\n"
+ ]
}
],
"metadata": {
diff --git a/site/en/tutorials/generative/pix2pix.ipynb b/site/en/tutorials/generative/pix2pix.ipynb
index 5c97053c50b..50af6ff13d9 100644
--- a/site/en/tutorials/generative/pix2pix.ipynb
+++ b/site/en/tutorials/generative/pix2pix.ipynb
@@ -72,13 +72,13 @@
"source": [
"This tutorial demonstrates how to build and train a conditional generative adversarial network (cGAN) called pix2pix that learns a mapping from input images to output images, as described in [Image-to-image translation with conditional adversarial networks](https://arxiv.org/abs/1611.07004) by Isola et al. (2017). pix2pix is not application specific—it can be applied to a wide range of tasks, including synthesizing photos from label maps, generating colorized photos from black and white images, turning Google Maps photos into aerial images, and even transforming sketches into photos.\n",
"\n",
- "In this example, your network will generate images of building facades using the [CMP Facade Database](http://cmp.felk.cvut.cz/~tylecr1/facade/) provided by the [Center for Machine Perception](http://cmp.felk.cvut.cz/) at the [Czech Technical University in Prague](https://www.cvut.cz/). To keep it short, you will use a [preprocessed copy]((https://people.eecs.berkeley.edu/~tinghuiz/projects/pix2pix/datasets/)) of this dataset created by the pix2pix authors.\n",
+ "In this example, your network will generate images of building facades using the [CMP Facade Database](http://cmp.felk.cvut.cz/~tylecr1/facade/) provided by the [Center for Machine Perception](http://cmp.felk.cvut.cz/) at the [Czech Technical University in Prague](https://www.cvut.cz/). To keep it short, you will use a [preprocessed copy](https://efrosgans.eecs.berkeley.edu/pix2pix/datasets/) of this dataset created by the pix2pix authors.\n",
"\n",
"In the pix2pix cGAN, you condition on input images and generate corresponding output images. cGANs were first proposed in [Conditional Generative Adversarial Nets](https://arxiv.org/abs/1411.1784) (Mirza and Osindero, 2014)\n",
"\n",
"The architecture of your network will contain:\n",
"\n",
- "- A generator with a [U-Net]([U-Net](https://arxiv.org/abs/1505.04597))-based architecture.\n",
+ "- A generator with a [U-Net](https://arxiv.org/abs/1505.04597)-based architecture.\n",
"- A discriminator represented by a convolutional PatchGAN classifier (proposed in the [pix2pix paper](https://arxiv.org/abs/1611.07004)).\n",
"\n",
"Note that each epoch can take around 15 seconds on a single V100 GPU.\n",
@@ -125,7 +125,7 @@
"source": [
"## Load the dataset\n",
"\n",
- "Download the CMP Facade Database data (30MB). Additional datasets are available in the same format [here](http://efrosgans.eecs.berkeley.edu/pix2pix/datasets/). In Colab you can select other datasets from the drop-down menu. Note that some of the other datasets are significantly larger (`edges2handbags` is 8GB). "
+ "Download the CMP Facade Database data (30MB). Additional datasets are available in the same format [here](http://efrosgans.eecs.berkeley.edu/pix2pix/datasets/). In Colab you can select other datasets from the drop-down menu. Note that some of the other datasets are significantly larger (`edges2handbags` is 8GB in size). "
]
},
{
@@ -156,7 +156,9 @@
"\n",
"path_to_zip = pathlib.Path(path_to_zip)\n",
"\n",
- "PATH = path_to_zip.parent/dataset_name"
+ "extraction_dir = f'{dataset_name}_extracted/{dataset_name}'\n",
+ "\n",
+ "PATH = path_to_zip.parent/extraction_dir"
]
},
{
@@ -280,7 +282,7 @@
"\n",
"1. Resize each `256 x 256` image to a larger height and width—`286 x 286`.\n",
"2. Randomly crop it back to `256 x 256`.\n",
- "3. Randomly flip the image horizontally i.e. left to right (random mirroring).\n",
+ "3. Randomly flip the image horizontally i.e., left to right (random mirroring).\n",
"4. Normalize the images to the `[-1, 1]` range."
]
},
@@ -490,7 +492,7 @@
"source": [
"## Build the generator\n",
"\n",
- "The generator of your pix2pix cGAN is a _modified_ [U-Net](https://arxiv.org/abs/1505.04597). A U-Net consists of an encoder (downsampler) and decoder (upsampler). (You can find out more about it in the [Image segmentation](https://www.tensorflow.org/tutorials/images/segmentation) tutorial and on the [U-Net project website](https://lmb.informatik.uni-freiburg.de/people/ronneber/u-net/).)\n",
+ "The generator of your pix2pix cGAN is a _modified_ [U-Net](https://arxiv.org/abs/1505.04597). A U-Net consists of an encoder (downsampler) and decoder (upsampler). (You can find out more about it in the [Image segmentation](../images/segmentation.ipynb) tutorial and on the [U-Net project website](https://lmb.informatik.uni-freiburg.de/people/ronneber/u-net/).)\n",
"\n",
"- Each block in the encoder is: Convolution -> Batch normalization -> Leaky ReLU\n",
"- Each block in the decoder is: Transposed convolution -> Batch normalization -> Dropout (applied to the first 3 blocks) -> ReLU\n",
@@ -1007,8 +1009,7 @@
"id": "Rb0QQFHF-JfS"
},
"source": [
- "Note: The `training=True` is intentional here since\n",
- "you want the batch statistics, while running the model on the test dataset. If you use `training=False`, you get the accumulated statistics learned from the training dataset (which you don't want)."
+ "Note: The `training=True` is intentional here since you want the batch statistics, while running the model on the test dataset. If you use `training=False`, you get the accumulated statistics learned from the training dataset (which you don't want)."
]
},
{
@@ -1181,7 +1182,8 @@
"\n",
"If you work on a local machine, you would launch a separate TensorBoard process. When working in a notebook, launch the viewer before starting the training to monitor with TensorBoard.\n",
"\n",
- "To launch the viewer paste the following into a code-cell:"
+ "Launch the TensorBoard viewer (Sorry, this doesn't\n",
+ "display on tensorflow.org):"
]
},
{
@@ -1199,72 +1201,30 @@
{
"cell_type": "markdown",
"metadata": {
- "id": "Pe0-8Bzg22ox"
- },
- "source": [
- "Finally, run the training loop:"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "id": "a1zZmKmvOH85"
- },
- "outputs": [],
- "source": [
- "fit(train_dataset, test_dataset, steps=40000)"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {
- "id": "oeq9sByu86-B"
- },
- "source": [
- "If you want to share the TensorBoard results _publicly_, you can upload the logs to [TensorBoard.dev](https://tensorboard.dev/) by copying the following into a code-cell.\n",
- "\n",
- "Note: This requires a Google account.\n",
- "\n",
- "```\n",
- "!tensorboard dev upload --logdir {log_dir}\n",
- "```"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {
- "id": "l-kT7WHRKz-E"
+ "id": "fyjixlMlBybN"
},
"source": [
- "Caution: This command does not terminate. It's designed to continuously upload the results of long-running experiments. Once your data is uploaded you need to stop it using the \"interrupt execution\" option in your notebook tool."
+ "You can view the [results of a previous run](https://tensorboard.dev/experiment/lZ0C6FONROaUMfjYkVyJqw) of this notebook on [TensorBoard.dev](https://tensorboard.dev/)."
]
},
{
"cell_type": "markdown",
"metadata": {
- "id": "-lGhS_LfwQoL"
+ "id": "Pe0-8Bzg22ox"
},
"source": [
- "You can view the [results of a previous run](https://tensorboard.dev/experiment/lZ0C6FONROaUMfjYkVyJqw) of this notebook on [TensorBoard.dev](https://tensorboard.dev/).\n",
- "\n",
- "TensorBoard.dev is a managed experience for hosting, tracking, and sharing ML experiments with everyone.\n",
- "\n",
- "It can also included inline using an `
![](\"https://www.tensorflow.org/images/tf_logo_32px.png\"){kind=logo}
View on TensorFlow.org\n",
+ " ![](\"https://www.tensorflow.org/images/colab_logo_32px.png\"){kind=logo}
Run in Google Colab\n",
+ " ![](\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\")
View source on GitHub\n",
+ " ![](\"https://www.tensorflow.org/images/download_logo_32px.png\"){kind=logo}
Download notebook\n",
+ " 
+
+* Building non-traditional models or layers that do not fully fit the
+ structures supported by high-level APIs
+* Building custom layers, losses, models, and optimizers within Keras
+* Implementing new optimization techniques to expedite convergence during
+ training
+* Creating custom metrics for performance evaluation
+* Designing highly-configurable training loops with support for features like
+ batching, cross-validation, and distribution strategies
+
+### Build frameworks and tools
+
+The TensorFlow Core APIs can also serve as the building blocks for new
+high-level frameworks. Here are some examples of tools and frameworks that are
+created with the low-level APIs:
+
+
+* [Keras](https://keras.io): deep learning for humans
+* [TensorFlow Model Optimization Toolkit](https://www.tensorflow.org/model_optimization):
+ a suite of tools to optimize ML models for deployment and execution
+* [TensorFlow Graphics](https://www.tensorflow.org/graphics): a library for
+ making useful graphics functions widely accessible
+
+### Build for scientific computing
+
+The TensorFlow Core APIs can also be applied outside the realm of machine
+learning. Here are a few general-purpose use cases of TensorFlow for scientific
+computing:
+
+
+* Physics simulations for solid mechanics and
+ [fluid dynamics](https://arxiv.org/abs/2108.11076) problems
+* Graphics rendering applications like
+ [ray tracing](https://github.com/BachiLi/redner)
+* Solving
+ [constrained optimization problems](https://github.com/google-research/tensorflow_constrained_optimization/blob/master/README.md)
+
+## Core API components
+
+Here are some of the fundamental components that comprise TensorFlow Core’s low-
+level APIs. Note that this is not an all-encompassing list:
+
+
+
+* Data structures : `tf.Tensor`, `tf.Variable`, `tf.TensorArray`
+* Primitive APIs: `tf.shape`,
+ [slicing](https://www.tensorflow.org/guide/tensor_slicing), `tf.concat`,
+ `tf.bitwise`
+* Numerical: `tf.math`, `tf.linalg`, `tf.random`
+* Functional components: `tf.function`, `tf.GradientTape`
+* Distribution: [DTensor](https://www.tensorflow.org/guide/dtensor_overview)
+* Export: `tf.saved_model`
+
+## Next steps
+
+The *Build with Core* documentation provides tutorials of basic machine learning
+concepts from scratch. The tutorials in this section help you get comfortable
+with writing low-level code with Core APIs that you can then apply to more
+complex use cases of your own.
+
+Note: You should not use the Core APIs to simply re-implement high-level APIs,
+and it is possible to use high-level APIs, such as Keras, with the Core APIs.
+
+To get started using and learning more about the Core APIs, check out the
+[Quickstart for TensorFlow Core](https://www.tensorflow.org/guide/core/quickstart_core).
diff --git a/site/en/guide/core/logistic_regression_core.ipynb b/site/en/guide/core/logistic_regression_core.ipynb
new file mode 100644
index 00000000000..5a9af324ad5
--- /dev/null
+++ b/site/en/guide/core/logistic_regression_core.ipynb
@@ -0,0 +1,935 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "FhGuhbZ6M5tl"
+ },
+ "source": [
+ "##### Copyright 2022 The TensorFlow Authors."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "cellView": "form",
+ "id": "AwOEIRJC6Une"
+ },
+ "outputs": [],
+ "source": [
+ "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+ "# you may not use this file except in compliance with the License.\n",
+ "# You may obtain a copy of the License at\n",
+ "#\n",
+ "# https://www.apache.org/licenses/LICENSE-2.0\n",
+ "#\n",
+ "# Unless required by applicable law or agreed to in writing, software\n",
+ "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+ "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+ "# See the License for the specific language governing permissions and\n",
+ "# limitations under the License."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "EIdT9iu_Z4Rb"
+ },
+ "source": [
+ "# Logistic regression for binary classification with Core APIs"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "bBIlTPscrIT9"
+ },
+ "source": [
+ "![\"A](\"https://www.tensorflow.org/images/dtensor/dtensor_layout_rank2.png\")
\n"
+ "![\"A](\"https://www.tensorflow.org/images/dtensor/dtensor_layout_hybrid.png\")
\n"
+ "![](\"https://www.tensorflow.org/images/hub_logo_32px.png\"){kind=logo}
See TF Hub model![\"Common](\"https://upload.wikimedia.org/wikipedia/commons/thumb/a/a9/Common_Blackbird.jpg/1200px-Common_Blackbird.jpg\")
|\n",
+ "|:--:|\n",
+ "| *By ![\"drawing\"/](\"https://github.com/tensorflow/tfjs-models/raw/master/pose-detection/assets/combined_squat_dance.gif\")
\n",
+ "\n",
+ "*Images downloaded from Pexels (https://www.pexels.com/)\n",
+ "\n",
+ "This Colab walks you through the details of how to load MoveNet, and run inference on the input image and video below.\n",
+ "\n",
+ "Note: check out the [live demo](https://storage.googleapis.com/tfjs-models/demos/pose-detection/index.html?model=movenet) for how the model works!"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "10_zkgbZBkIE"
+ },
+ "source": [
+ "# Human Pose Estimation with MoveNet"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "9u_VGR6_BmbZ"
+ },
+ "source": [
+ "## Visualization libraries & Imports"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "TtcwSIcgbIVN"
+ },
+ "outputs": [],
+ "source": [
+ "!pip install -q imageio\n",
+ "!pip install -q opencv-python\n",
+ "!pip install -q git+https://github.com/tensorflow/docs"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "9BLeJv-pCCld"
+ },
+ "outputs": [],
+ "source": [
+ "import tensorflow as tf\n",
+ "import tensorflow_hub as hub\n",
+ "from tensorflow_docs.vis import embed\n",
+ "import numpy as np\n",
+ "import cv2\n",
+ "\n",
+ "# Import matplotlib libraries\n",
+ "from matplotlib import pyplot as plt\n",
+ "from matplotlib.collections import LineCollection\n",
+ "import matplotlib.patches as patches\n",
+ "\n",
+ "# Some modules to display an animation using imageio.\n",
+ "import imageio\n",
+ "from IPython.display import HTML, display"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "cellView": "form",
+ "id": "bEJBMeRb3YUy"
+ },
+ "outputs": [],
+ "source": [
+ "#@title Helper functions for visualization\n",
+ "\n",
+ "# Dictionary that maps from joint names to keypoint indices.\n",
+ "KEYPOINT_DICT = {\n",
+ " 'nose': 0,\n",
+ " 'left_eye': 1,\n",
+ " 'right_eye': 2,\n",
+ " 'left_ear': 3,\n",
+ " 'right_ear': 4,\n",
+ " 'left_shoulder': 5,\n",
+ " 'right_shoulder': 6,\n",
+ " 'left_elbow': 7,\n",
+ " 'right_elbow': 8,\n",
+ " 'left_wrist': 9,\n",
+ " 'right_wrist': 10,\n",
+ " 'left_hip': 11,\n",
+ " 'right_hip': 12,\n",
+ " 'left_knee': 13,\n",
+ " 'right_knee': 14,\n",
+ " 'left_ankle': 15,\n",
+ " 'right_ankle': 16\n",
+ "}\n",
+ "\n",
+ "# Maps bones to a matplotlib color name.\n",
+ "KEYPOINT_EDGE_INDS_TO_COLOR = {\n",
+ " (0, 1): 'm',\n",
+ " (0, 2): 'c',\n",
+ " (1, 3): 'm',\n",
+ " (2, 4): 'c',\n",
+ " (0, 5): 'm',\n",
+ " (0, 6): 'c',\n",
+ " (5, 7): 'm',\n",
+ " (7, 9): 'm',\n",
+ " (6, 8): 'c',\n",
+ " (8, 10): 'c',\n",
+ " (5, 6): 'y',\n",
+ " (5, 11): 'm',\n",
+ " (6, 12): 'c',\n",
+ " (11, 12): 'y',\n",
+ " (11, 13): 'm',\n",
+ " (13, 15): 'm',\n",
+ " (12, 14): 'c',\n",
+ " (14, 16): 'c'\n",
+ "}\n",
+ "\n",
+ "def _keypoints_and_edges_for_display(keypoints_with_scores,\n",
+ " height,\n",
+ " width,\n",
+ " keypoint_threshold=0.11):\n",
+ " \"\"\"Returns high confidence keypoints and edges for visualization.\n",
+ "\n",
+ " Args:\n",
+ " keypoints_with_scores: A numpy array with shape [1, 1, 17, 3] representing\n",
+ " the keypoint coordinates and scores returned from the MoveNet model.\n",
+ " height: height of the image in pixels.\n",
+ " width: width of the image in pixels.\n",
+ " keypoint_threshold: minimum confidence score for a keypoint to be\n",
+ " visualized.\n",
+ "\n",
+ " Returns:\n",
+ " A (keypoints_xy, edges_xy, edge_colors) containing:\n",
+ " * the coordinates of all keypoints of all detected entities;\n",
+ " * the coordinates of all skeleton edges of all detected entities;\n",
+ " * the colors in which the edges should be plotted.\n",
+ " \"\"\"\n",
+ " keypoints_all = []\n",
+ " keypoint_edges_all = []\n",
+ " edge_colors = []\n",
+ " num_instances, _, _, _ = keypoints_with_scores.shape\n",
+ " for idx in range(num_instances):\n",
+ " kpts_x = keypoints_with_scores[0, idx, :, 1]\n",
+ " kpts_y = keypoints_with_scores[0, idx, :, 0]\n",
+ " kpts_scores = keypoints_with_scores[0, idx, :, 2]\n",
+ " kpts_absolute_xy = np.stack(\n",
+ " [width * np.array(kpts_x), height * np.array(kpts_y)], axis=-1)\n",
+ " kpts_above_thresh_absolute = kpts_absolute_xy[\n",
+ " kpts_scores > keypoint_threshold, :]\n",
+ " keypoints_all.append(kpts_above_thresh_absolute)\n",
+ "\n",
+ " for edge_pair, color in KEYPOINT_EDGE_INDS_TO_COLOR.items():\n",
+ " if (kpts_scores[edge_pair[0]] > keypoint_threshold and\n",
+ " kpts_scores[edge_pair[1]] > keypoint_threshold):\n",
+ " x_start = kpts_absolute_xy[edge_pair[0], 0]\n",
+ " y_start = kpts_absolute_xy[edge_pair[0], 1]\n",
+ " x_end = kpts_absolute_xy[edge_pair[1], 0]\n",
+ " y_end = kpts_absolute_xy[edge_pair[1], 1]\n",
+ " line_seg = np.array([[x_start, y_start], [x_end, y_end]])\n",
+ " keypoint_edges_all.append(line_seg)\n",
+ " edge_colors.append(color)\n",
+ " if keypoints_all:\n",
+ " keypoints_xy = np.concatenate(keypoints_all, axis=0)\n",
+ " else:\n",
+ " keypoints_xy = np.zeros((0, 17, 2))\n",
+ "\n",
+ " if keypoint_edges_all:\n",
+ " edges_xy = np.stack(keypoint_edges_all, axis=0)\n",
+ " else:\n",
+ " edges_xy = np.zeros((0, 2, 2))\n",
+ " return keypoints_xy, edges_xy, edge_colors\n",
+ "\n",
+ "\n",
+ "def draw_prediction_on_image(\n",
+ " image, keypoints_with_scores, crop_region=None, close_figure=False,\n",
+ " output_image_height=None):\n",
+ " \"\"\"Draws the keypoint predictions on image.\n",
+ "\n",
+ " Args:\n",
+ " image: A numpy array with shape [height, width, channel] representing the\n",
+ " pixel values of the input image.\n",
+ " keypoints_with_scores: A numpy array with shape [1, 1, 17, 3] representing\n",
+ " the keypoint coordinates and scores returned from the MoveNet model.\n",
+ " crop_region: A dictionary that defines the coordinates of the bounding box\n",
+ " of the crop region in normalized coordinates (see the init_crop_region\n",
+ " function below for more detail). If provided, this function will also\n",
+ " draw the bounding box on the image.\n",
+ " output_image_height: An integer indicating the height of the output image.\n",
+ " Note that the image aspect ratio will be the same as the input image.\n",
+ "\n",
+ " Returns:\n",
+ " A numpy array with shape [out_height, out_width, channel] representing the\n",
+ " image overlaid with keypoint predictions.\n",
+ " \"\"\"\n",
+ " height, width, channel = image.shape\n",
+ " aspect_ratio = float(width) / height\n",
+ " fig, ax = plt.subplots(figsize=(12 * aspect_ratio, 12))\n",
+ " # To remove the huge white borders\n",
+ " fig.tight_layout(pad=0)\n",
+ " ax.margins(0)\n",
+ " ax.set_yticklabels([])\n",
+ " ax.set_xticklabels([])\n",
+ " plt.axis('off')\n",
+ "\n",
+ " im = ax.imshow(image)\n",
+ " line_segments = LineCollection([], linewidths=(4), linestyle='solid')\n",
+ " ax.add_collection(line_segments)\n",
+ " # Turn off tick labels\n",
+ " scat = ax.scatter([], [], s=60, color='#FF1493', zorder=3)\n",
+ "\n",
+ " (keypoint_locs, keypoint_edges,\n",
+ " edge_colors) = _keypoints_and_edges_for_display(\n",
+ " keypoints_with_scores, height, width)\n",
+ "\n",
+ " line_segments.set_segments(keypoint_edges)\n",
+ " line_segments.set_color(edge_colors)\n",
+ " if keypoint_edges.shape[0]:\n",
+ " line_segments.set_segments(keypoint_edges)\n",
+ " line_segments.set_color(edge_colors)\n",
+ " if keypoint_locs.shape[0]:\n",
+ " scat.set_offsets(keypoint_locs)\n",
+ "\n",
+ " if crop_region is not None:\n",
+ " xmin = max(crop_region['x_min'] * width, 0.0)\n",
+ " ymin = max(crop_region['y_min'] * height, 0.0)\n",
+ " rec_width = min(crop_region['x_max'], 0.99) * width - xmin\n",
+ " rec_height = min(crop_region['y_max'], 0.99) * height - ymin\n",
+ " rect = patches.Rectangle(\n",
+ " (xmin,ymin),rec_width,rec_height,\n",
+ " linewidth=1,edgecolor='b',facecolor='none')\n",
+ " ax.add_patch(rect)\n",
+ "\n",
+ " fig.canvas.draw()\n",
+ " image_from_plot = np.frombuffer(fig.canvas.tostring_rgb(), dtype=np.uint8)\n",
+ " image_from_plot = image_from_plot.reshape(\n",
+ " fig.canvas.get_width_height()[::-1] + (3,))\n",
+ " plt.close(fig)\n",
+ " if output_image_height is not None:\n",
+ " output_image_width = int(output_image_height / height * width)\n",
+ " image_from_plot = cv2.resize(\n",
+ " image_from_plot, dsize=(output_image_width, output_image_height),\n",
+ " interpolation=cv2.INTER_CUBIC)\n",
+ " return image_from_plot\n",
+ "\n",
+ "def to_gif(images, duration):\n",
+ " \"\"\"Converts image sequence (4D numpy array) to gif.\"\"\"\n",
+ " imageio.mimsave('./animation.gif', images, duration=duration)\n",
+ " return embed.embed_file('./animation.gif')\n",
+ "\n",
+ "def progress(value, max=100):\n",
+ " return HTML(\"\"\"\n",
+ " 
@@ -186,9 +186,9 @@ communication, see:
### Code
-* [WorkerService API definition](https://www.tensorflow.org/code/tensorflow/core/protobuf/worker_service.proto)
-* [Worker interface](https://www.tensorflow.org/code/tensorflow/core/distributed_runtime/worker_interface.h)
-* [Remote rendezvous (for Send and Recv implementations)](https://www.tensorflow.org/code/tensorflow/core/distributed_runtime/rpc/rpc_rendezvous_mgr.h)
+* [WorkerService API definition](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/protobuf/worker_service.proto)
+* [Worker interface](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/distributed_runtime/worker_interface.h)
+* [Remote rendezvous (for Send and Recv implementations)](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/distributed_runtime/rpc/rpc_rendezvous_mgr.h)
## Kernel Implementations
@@ -199,7 +199,7 @@ Many of the operation kernels are implemented using Eigen::Tensor, which uses
C++ templates to generate efficient parallel code for multicore CPUs and GPUs;
however, we liberally use libraries like cuDNN where a more efficient kernel
implementation is possible. We have also implemented
-[quantization](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/g3doc/performance/post_training_quantization.md), which enables
+[quantization](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/lite/g3doc/performance/post_training_quantization.md), which enables
faster inference in environments such as mobile devices and high-throughput
datacenter applications, and use the
[gemmlowp](https://github.com/google/gemmlowp) low-precision matrix library to
@@ -215,4 +215,4 @@ experimental implementation of automatic kernel fusion.
### Code
-* [`OpKernel` interface](https://www.tensorflow.org/code/tensorflow/core/framework/op_kernel.h)
+* [`OpKernel` interface](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/framework/op_kernel.h)
diff --git a/site/en/r1/guide/extend/bindings.md b/site/en/r1/guide/extend/bindings.md
index 9c10e90840f..7daa2212106 100644
--- a/site/en/r1/guide/extend/bindings.md
+++ b/site/en/r1/guide/extend/bindings.md
@@ -112,11 +112,11 @@ There are a few ways to get a list of the `OpDef`s for the registered ops:
to interpret the `OpDef` messages.
- The C++ function `OpRegistry::Global()->GetRegisteredOps()` returns the same
list of all registered `OpDef`s (defined in
- [`tensorflow/core/framework/op.h`](https://www.tensorflow.org/code/tensorflow/core/framework/op.h)). This can be used to write the generator
+ [`tensorflow/core/framework/op.h`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/framework/op.h)). This can be used to write the generator
in C++ (particularly useful for languages that do not have protocol buffer
support).
- The ASCII-serialized version of that list is periodically checked in to
- [`tensorflow/core/ops/ops.pbtxt`](https://www.tensorflow.org/code/tensorflow/core/ops/ops.pbtxt) by an automated process.
+ [`tensorflow/core/ops/ops.pbtxt`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/ops/ops.pbtxt) by an automated process.
The `OpDef` specifies the following:
@@ -159,7 +159,7 @@ between the generated code and the `OpDef`s checked into the repository, but is
useful for languages where code is expected to be generated ahead of time like
`go get` for Go and `cargo ops` for Rust. At the other end of the spectrum, for
some languages the code could be generated dynamically from
-[`tensorflow/core/ops/ops.pbtxt`](https://www.tensorflow.org/code/tensorflow/core/ops/ops.pbtxt).
+[`tensorflow/core/ops/ops.pbtxt`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/ops/ops.pbtxt).
#### Handling Constants
@@ -228,4 +228,4 @@ At this time, support for gradients, functions and control flow operations ("if"
and "while") is not available in languages other than Python. This will be
updated when the [C API] provides necessary support.
-[C API]: https://www.tensorflow.org/code/tensorflow/c/c_api.h
+[C API]: https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/c/c_api.h
diff --git a/site/en/r1/guide/extend/filesystem.md b/site/en/r1/guide/extend/filesystem.md
index 4d34c07102e..2d6ea0c4645 100644
--- a/site/en/r1/guide/extend/filesystem.md
+++ b/site/en/r1/guide/extend/filesystem.md
@@ -54,7 +54,7 @@ To implement a custom filesystem plugin, you must do the following:
### The FileSystem interface
The `FileSystem` interface is an abstract C++ interface defined in
-[file_system.h](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/platform/file_system.h).
+[file_system.h](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/platform/file_system.h).
An implementation of the `FileSystem` interface should implement all relevant
the methods defined by the interface. Implementing the interface requires
defining operations such as creating `RandomAccessFile`, `WritableFile`, and
@@ -70,26 +70,26 @@ involves calling `stat()` on the file and then returns the filesize as reported
by the return of the stat object. Similarly, for the `HDFSFileSystem`
implementation, these calls simply delegate to the `libHDFS` implementation of
similar functionality, such as `hdfsDelete` for
-[DeleteFile](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/platform/hadoop/hadoop_file_system.cc#L386).
+[DeleteFile](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/platform/hadoop/hadoop_file_system.cc#L386).
We suggest looking through these code examples to get an idea of how different
filesystem implementations call their existing libraries. Examples include:
* [POSIX
- plugin](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/platform/posix/posix_file_system.h)
+ plugin](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/platform/posix/posix_file_system.h)
* [HDFS
- plugin](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/platform/hadoop/hadoop_file_system.h)
+ plugin](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/platform/hadoop/hadoop_file_system.h)
* [GCS
- plugin](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/platform/cloud/gcs_file_system.h)
+ plugin](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/platform/cloud/gcs_file_system.h)
* [S3
- plugin](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/platform/s3/s3_file_system.h)
+ plugin](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/platform/s3/s3_file_system.h)
#### The File interfaces
Beyond operations that allow you to query and manipulate files and directories
in a filesystem, the `FileSystem` interface requires you to implement factories
that return implementations of abstract objects such as the
-[RandomAccessFile](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/platform/file_system.h#L223),
+[RandomAccessFile](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/platform/file_system.h#L223),
the `WritableFile`, so that TensorFlow code and read and write to files in that
`FileSystem` implementation.
@@ -224,7 +224,7 @@ it will use the `FooBarFileSystem` implementation.
Next, you must build a shared object containing this implementation. An example
of doing so using bazel's `cc_binary` rule can be found
-[here](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/BUILD#L244),
+[here](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/python/BUILD#L244),
but you may use any build system to do so. See the section on [building the op library](../extend/op.md#build_the_op_library) for similar
instructions.
@@ -236,7 +236,7 @@ passing the path to the shared object. Calling this in your client program loads
the shared object in the process, thus registering your implementation as
available for any file operations going through the `FileSystem` interface. You
can see
-[test_file_system.py](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/framework/file_system_test.py)
+[test_file_system.py](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/python/framework/file_system_test.py)
for an example.
## What goes through this interface?
diff --git a/site/en/r1/guide/extend/formats.md b/site/en/r1/guide/extend/formats.md
index 3b7b4aafbd6..bdebee5487d 100644
--- a/site/en/r1/guide/extend/formats.md
+++ b/site/en/r1/guide/extend/formats.md
@@ -28,11 +28,11 @@ individual records in a file. There are several examples of "reader" datasets
that are already built into TensorFlow:
* `tf.data.TFRecordDataset`
- ([source in `kernels/data/reader_dataset_ops.cc`](https://www.tensorflow.org/code/tensorflow/core/kernels/data/reader_dataset_ops.cc))
+ ([source in `kernels/data/reader_dataset_ops.cc`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/kernels/data/reader_dataset_ops.cc))
* `tf.data.FixedLengthRecordDataset`
- ([source in `kernels/data/reader_dataset_ops.cc`](https://www.tensorflow.org/code/tensorflow/core/kernels/data/reader_dataset_ops.cc))
+ ([source in `kernels/data/reader_dataset_ops.cc`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/kernels/data/reader_dataset_ops.cc))
* `tf.data.TextLineDataset`
- ([source in `kernels/data/reader_dataset_ops.cc`](https://www.tensorflow.org/code/tensorflow/core/kernels/data/reader_dataset_ops.cc))
+ ([source in `kernels/data/reader_dataset_ops.cc`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/kernels/data/reader_dataset_ops.cc))
Each of these implementations comprises three related classes:
@@ -279,7 +279,7 @@ if __name__ == "__main__":
```
You can see some examples of `Dataset` wrapper classes in
-[`tensorflow/python/data/ops/dataset_ops.py`](https://www.tensorflow.org/code/tensorflow/python/data/ops/dataset_ops.py).
+[`tensorflow/python/data/ops/dataset_ops.py`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/python/data/ops/dataset_ops.py).
## Writing an Op for a record format
@@ -297,7 +297,7 @@ Examples of Ops useful for decoding records:
Note that it can be useful to use multiple Ops to decode a particular record
format. For example, you may have an image saved as a string in
-[a `tf.train.Example` protocol buffer](https://www.tensorflow.org/code/tensorflow/core/example/example.proto).
+[a `tf.train.Example` protocol buffer](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/example/example.proto).
Depending on the format of that image, you might take the corresponding output
from a `tf.parse_single_example` op and call `tf.image.decode_jpeg`,
`tf.image.decode_png`, or `tf.decode_raw`. It is common to take the output
diff --git a/site/en/r1/guide/extend/model_files.md b/site/en/r1/guide/extend/model_files.md
index 30e73a5169e..e590fcf1f27 100644
--- a/site/en/r1/guide/extend/model_files.md
+++ b/site/en/r1/guide/extend/model_files.md
@@ -28,7 +28,7 @@ by calling `as_graph_def()`, which returns a `GraphDef` object.
The GraphDef class is an object created by the ProtoBuf library from the
definition in
-[tensorflow/core/framework/graph.proto](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/graph.proto). The protobuf tools parse
+[tensorflow/core/framework/graph.proto](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/framework/graph.proto). The protobuf tools parse
this text file, and generate the code to load, store, and manipulate graph
definitions. If you see a standalone TensorFlow file representing a model, it's
likely to contain a serialized version of one of these `GraphDef` objects
@@ -87,7 +87,7 @@ for node in graph_def.node
```
Each node is a `NodeDef` object, defined in
-[tensorflow/core/framework/node_def.proto](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/node_def.proto). These
+[tensorflow/core/framework/node_def.proto](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/framework/node_def.proto). These
are the fundamental building blocks of TensorFlow graphs, with each one defining
a single operation along with its input connections. Here are the members of a
`NodeDef`, and what they mean.
@@ -107,7 +107,7 @@ This defines what operation to run, for example `"Add"`, `"MatMul"`, or
`"Conv2D"`. When a graph is run, this op name is looked up in a registry to
find an implementation. The registry is populated by calls to the
`REGISTER_OP()` macro, like those in
-[tensorflow/core/ops/nn_ops.cc](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/ops/nn_ops.cc).
+[tensorflow/core/ops/nn_ops.cc](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/ops/nn_ops.cc).
### `input`
@@ -133,7 +133,7 @@ size of filters for convolutions, or the values of constant ops. Because there
can be so many different types of attribute values, from strings, to ints, to
arrays of tensor values, there's a separate protobuf file defining the data
structure that holds them, in
-[tensorflow/core/framework/attr_value.proto](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/attr_value.proto).
+[tensorflow/core/framework/attr_value.proto](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/framework/attr_value.proto).
Each attribute has a unique name string, and the expected attributes are listed
when the operation is defined. If an attribute isn't present in a node, but it
@@ -151,7 +151,7 @@ the file format during training. Instead, they're held in separate checkpoint
files, and there are `Variable` ops in the graph that load the latest values
when they're initialized. It's often not very convenient to have separate files
when you're deploying to production, so there's the
-[freeze_graph.py](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/tools/freeze_graph.py) script that takes a graph definition and a set
+[freeze_graph.py](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/python/tools/freeze_graph.py) script that takes a graph definition and a set
of checkpoints and freezes them together into a single file.
What this does is load the `GraphDef`, pull in the values for all the variables
@@ -167,7 +167,7 @@ the most common problems is extracting and interpreting the weight values. A
common way to store them, for example in graphs created by the freeze_graph
script, is as `Const` ops containing the weights as `Tensors`. These are
defined in
-[tensorflow/core/framework/tensor.proto](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/tensor.proto), and contain information
+[tensorflow/core/framework/tensor.proto](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/framework/tensor.proto), and contain information
about the size and type of the data, as well as the values themselves. In
Python, you get a `TensorProto` object from a `NodeDef` representing a `Const`
op by calling something like `some_node_def.attr['value'].tensor`.
diff --git a/site/en/r1/guide/extend/op.md b/site/en/r1/guide/extend/op.md
index d006a6251d0..186d9c28c04 100644
--- a/site/en/r1/guide/extend/op.md
+++ b/site/en/r1/guide/extend/op.md
@@ -47,7 +47,7 @@ To incorporate your custom op you'll need to:
test the op in C++. If you define gradients, you can verify them with the
Python `tf.test.compute_gradient_error`.
See
- [`relu_op_test.py`](https://www.tensorflow.org/code/tensorflow/python/kernel_tests/relu_op_test.py) as
+ [`relu_op_test.py`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/python/kernel_tests/relu_op_test.py) as
an example that tests the forward functions of Relu-like operators and
their gradients.
@@ -155,17 +155,17 @@ REGISTER_KERNEL_BUILDER(Name("ZeroOut").Device(DEVICE_CPU), ZeroOutOp);
> Important: Instances of your OpKernel may be accessed concurrently.
> Your `Compute` method must be thread-safe. Guard any access to class
> members with a mutex. Or better yet, don't share state via class members!
-> Consider using a [`ResourceMgr`](https://www.tensorflow.org/code/tensorflow/core/framework/resource_mgr.h)
+> Consider using a [`ResourceMgr`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/framework/resource_mgr.h)
> to keep track of op state.
### Multi-threaded CPU kernels
To write a multi-threaded CPU kernel, the Shard function in
-[`work_sharder.h`](https://www.tensorflow.org/code/tensorflow/core/util/work_sharder.h)
+[`work_sharder.h`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/util/work_sharder.h)
can be used. This function shards a computation function across the
threads configured to be used for intra-op threading (see
intra_op_parallelism_threads in
-[`config.proto`](https://www.tensorflow.org/code/tensorflow/core/protobuf/config.proto)).
+[`config.proto`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/protobuf/config.proto)).
### GPU kernels
@@ -348,12 +348,13 @@ g++ -std=c++11 -shared zero_out.cc -o zero_out.so -fPIC ${TF_CFLAGS[@]} ${TF_LFL
On macOS, the additional flag "-undefined dynamic_lookup" is required when
building the `.so` file.
-> Note on `gcc` version `>=5`: gcc uses the new C++
-> [ABI](https://gcc.gnu.org/gcc-5/changes.html#libstdcxx) since version `5`. The binary pip
-> packages available on the TensorFlow website are built with `gcc4` that uses
-> the older ABI. If you compile your op library with `gcc>=5`, add
-> `-D_GLIBCXX_USE_CXX11_ABI=0` to the command line to make the library
-> compatible with the older abi.
+> Note on `gcc` version `>=5`: gcc uses the new C++
+> [ABI](https://gcc.gnu.org/gcc-5/changes.html#libstdcxx) since version `5`.
+> TensorFlow 2.8 and earlier were built with `gcc4` that uses the older ABI. If
+> you are using these versions of TensorFlow and are trying to compile your op
+> library with `gcc>=5`, add `-D_GLIBCXX_USE_CXX11_ABI=0` to the command line to
+> make the library compatible with the older ABI. TensorFlow 2.9+ packages are
+> compatible with the newer ABI by default.
### Compile the op using bazel (TensorFlow source installation)
@@ -485,13 +486,13 @@ This asserts that the input is a vector, and returns having set the
* The `context`, which can either be an `OpKernelContext` or
`OpKernelConstruction` pointer (see
- [`tensorflow/core/framework/op_kernel.h`](https://www.tensorflow.org/code/tensorflow/core/framework/op_kernel.h)),
+ [`tensorflow/core/framework/op_kernel.h`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/framework/op_kernel.h)),
for its `SetStatus()` method.
* The condition. For example, there are functions for validating the shape
of a tensor in
- [`tensorflow/core/framework/tensor_shape.h`](https://www.tensorflow.org/code/tensorflow/core/framework/tensor_shape.h)
+ [`tensorflow/core/framework/tensor_shape.h`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/framework/tensor_shape.h)
* The error itself, which is represented by a `Status` object, see
- [`tensorflow/core/lib/core/status.h`](https://www.tensorflow.org/code/tensorflow/core/lib/core/status.h). A
+ [`tensorflow/core/lib/core/status.h`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/lib/core/status.h). A
`Status` has both a type (frequently `InvalidArgument`, but see the list of
types) and a message. Functions for constructing an error may be found in
[`tensorflow/core/lib/core/errors.h`][validation-macros].
@@ -632,7 +633,7 @@ define an attr with constraints, you can use the following `
-Next, try it out on your own images by supplying the --image= argument, e.g.
+Next, try it out on your own images by supplying the --image= argument, e.g.,
```bash
bazel-bin/tensorflow/examples/label_image/label_image --image=my_image.png
```
-If you look inside the [`tensorflow/examples/label_image/main.cc`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/examples/label_image/main.cc)
+If you look inside the [`tensorflow/examples/label_image/main.cc`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/examples/label_image/main.cc)
file, you can find out
how it works. We hope this code will help you integrate TensorFlow into
your own applications, so we will walk step by step through the main functions:
@@ -164,7 +164,7 @@ training. If you have a graph that you've trained yourself, you'll just need
to adjust the values to match whatever you used during your training process.
You can see how they're applied to an image in the
-[`ReadTensorFromImageFile()`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/examples/label_image/main.cc#L88)
+[`ReadTensorFromImageFile()`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/examples/label_image/main.cc#L88)
function.
```C++
@@ -334,7 +334,7 @@ The `PrintTopLabels()` function takes those sorted results, and prints them out
friendly way. The `CheckTopLabel()` function is very similar, but just makes sure that
the top label is the one we expect, for debugging purposes.
-At the end, [`main()`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/examples/label_image/main.cc#L252)
+At the end, [`main()`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/examples/label_image/main.cc#L252)
ties together all of these calls.
```C++
diff --git a/site/en/r1/tutorials/images/transfer_learning.ipynb b/site/en/r1/tutorials/images/transfer_learning.ipynb
index 25779babd17..232ffa40bdd 100644
--- a/site/en/r1/tutorials/images/transfer_learning.ipynb
+++ b/site/en/r1/tutorials/images/transfer_learning.ipynb
@@ -148,7 +148,7 @@
},
"outputs": [],
"source": [
- "zip_file = tf.keras.utils.get_file(origin=\"https://storage.googleapis.com/mledu-datasets/cats_and_dogs_filtered.zip\",\n",
+ "zip_file = tf.keras.utils.get_file(origin=\"https://download.mlcc.google.com/mledu-datasets/cats_and_dogs_filtered.zip\",\n",
" fname=\"cats_and_dogs_filtered.zip\", extract=True)\n",
"base_dir, _ = os.path.splitext(zip_file)"
]
diff --git a/site/en/r1/tutorials/keras/save_and_restore_models.ipynb b/site/en/r1/tutorials/keras/save_and_restore_models.ipynb
index e9d112bd3f3..04cc94417a9 100644
--- a/site/en/r1/tutorials/keras/save_and_restore_models.ipynb
+++ b/site/en/r1/tutorials/keras/save_and_restore_models.ipynb
@@ -115,7 +115,7 @@
"\n",
"Sharing this data helps others understand how the model works and try it themselves with new data.\n",
"\n",
- "Caution: Be careful with untrusted code—TensorFlow models are code. See [Using TensorFlow Securely](https://github.com/tensorflow/tensorflow/blob/master/SECURITY.md) for details.\n",
+ "Caution: Be careful with untrusted code—TensorFlow models are code. See [Using TensorFlow Securely](https://github.com/tensorflow/tensorflow/blob/r1.15/SECURITY.md) for details.\n",
"\n",
"### Options\n",
"\n",
diff --git a/site/en/r1/tutorials/load_data/tf_records.ipynb b/site/en/r1/tutorials/load_data/tf_records.ipynb
index 64631cf0eab..45635034c69 100644
--- a/site/en/r1/tutorials/load_data/tf_records.ipynb
+++ b/site/en/r1/tutorials/load_data/tf_records.ipynb
@@ -141,7 +141,7 @@
"source": [
"Fundamentally a `tf.Example` is a `{\"string\": tf.train.Feature}` mapping.\n",
"\n",
- "The `tf.train.Feature` message type can accept one of the following three types (See the [`.proto` file](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/example/feature.proto) for reference). Most other generic types can be coerced into one of these.\n",
+ "The `tf.train.Feature` message type can accept one of the following three types (See the [`.proto` file](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/example/feature.proto) for reference). Most other generic types can be coerced into one of these.\n",
"\n",
"1. `tf.train.BytesList` (the following types can be coerced)\n",
"\n",
@@ -276,7 +276,7 @@
"\n",
"1. We create a map (dictionary) from the feature name string to the encoded feature value produced in #1.\n",
"\n",
- "1. The map produced in #2 is converted to a [`Features` message](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/example/feature.proto#L85)."
+ "1. The map produced in #2 is converted to a [`Features` message](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/example/feature.proto#L85)."
]
},
{
@@ -365,7 +365,7 @@
"id": "XftzX9CN_uGT"
},
"source": [
- "For example, suppose we have a single observation from the dataset, `[False, 4, bytes('goat'), 0.9876]`. We can create and print the `tf.Example` message for this observation using `create_message()`. Each single observation will be written as a `Features` message as per the above. Note that the `tf.Example` [message](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/example/example.proto#L88) is just a wrapper around the `Features` message."
+ "For example, suppose we have a single observation from the dataset, `[False, 4, bytes('goat'), 0.9876]`. We can create and print the `tf.Example` message for this observation using `create_message()`. Each single observation will be written as a `Features` message as per the above. Note that the `tf.Example` [message](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/example/example.proto#L88) is just a wrapper around the `Features` message."
]
},
{
@@ -632,7 +632,7 @@
"source": [
"We can also read the TFRecord file using the `tf.data.TFRecordDataset` class.\n",
"\n",
- "More information on consuming TFRecord files using `tf.data` can be found [here](https://www.tensorflow.org/r1/guide/datasets#consuming_tfrecord_data).\n",
+ "More information on consuming TFRecord files using `tf.data` can be found [here](https://www.tensorflow.org/guide/data#consuming_tfrecord_data).\n",
"\n",
"Using `TFRecordDataset`s can be useful for standardizing input data and optimizing performance."
]
diff --git a/site/en/r1/tutorials/representation/kernel_methods.md b/site/en/r1/tutorials/representation/kernel_methods.md
index 67adc4951c6..227fe81d515 100644
--- a/site/en/r1/tutorials/representation/kernel_methods.md
+++ b/site/en/r1/tutorials/representation/kernel_methods.md
@@ -24,7 +24,7 @@ following sources for an introduction:
Currently, TensorFlow supports explicit kernel mappings for dense features only;
TensorFlow will provide support for sparse features at a later release.
-This tutorial uses [tf.contrib.learn](https://www.tensorflow.org/code/tensorflow/contrib/learn/python/learn)
+This tutorial uses [tf.contrib.learn](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/contrib/learn/python/learn)
(TensorFlow's high-level Machine Learning API) Estimators for our ML models.
If you are not familiar with this API, The [Estimator guide](../../guide/estimators.md)
is a good place to start. We will use the MNIST dataset. The tutorial consists
@@ -131,7 +131,7 @@ In addition to experimenting with the (training) batch size and the number of
training steps, there are a couple other parameters that can be tuned as well.
For instance, you can change the optimization method used to minimize the loss
by explicitly selecting another optimizer from the collection of
-[available optimizers](https://www.tensorflow.org/code/tensorflow/python/training).
+[available optimizers](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/python/training).
As an example, the following code constructs a LinearClassifier estimator that
uses the Follow-The-Regularized-Leader (FTRL) optimization strategy with a
specific learning rate and L2-regularization.
diff --git a/site/en/r1/tutorials/representation/linear.md b/site/en/r1/tutorials/representation/linear.md
index 5516672b34a..d996a13bc1f 100644
--- a/site/en/r1/tutorials/representation/linear.md
+++ b/site/en/r1/tutorials/representation/linear.md
@@ -12,7 +12,7 @@ those tools. It explains:
Read this overview to decide whether the Estimator's linear model tools might
be useful to you. Then work through the
-[Estimator wide and deep learning tutorial](https://github.com/tensorflow/models/tree/master/official/r1/wide_deep)
+[Estimator wide and deep learning tutorial](https://github.com/tensorflow/models/tree/r1.15/official/r1/wide_deep)
to give it a try. This overview uses code samples from the tutorial, but the
tutorial walks through the code in greater detail.
@@ -177,7 +177,7 @@ the name of a `FeatureColumn`. Each key's value is a tensor containing the
values of that feature for all data instances. See
[Premade Estimators](../../guide/premade_estimators.md#input_fn) for a
more comprehensive look at input functions, and `input_fn` in the
-[wide and deep learning tutorial](https://github.com/tensorflow/models/tree/master/official/r1/wide_deep)
+[wide and deep learning tutorial](https://github.com/tensorflow/models/tree/r1.15/official/r1/wide_deep)
for an example implementation of an input function.
The input function is passed to the `train()` and `evaluate()` calls that
@@ -236,4 +236,4 @@ e = tf.estimator.DNNLinearCombinedClassifier(
dnn_hidden_units=[100, 50])
```
For more information, see the
-[wide and deep learning tutorial](https://github.com/tensorflow/models/tree/master/official/r1/wide_deep).
+[wide and deep learning tutorial](https://github.com/tensorflow/models/tree/r1.15/official/r1/wide_deep).
diff --git a/site/en/r1/tutorials/representation/unicode.ipynb b/site/en/r1/tutorials/representation/unicode.ipynb
index 98aaacff5b9..f76977c3c92 100644
--- a/site/en/r1/tutorials/representation/unicode.ipynb
+++ b/site/en/r1/tutorials/representation/unicode.ipynb
@@ -136,7 +136,7 @@
"id": "jsMPnjb6UDJ1"
},
"source": [
- "Note: When using python to construct strings, the handling of unicode differs betweeen v2 and v3. In v2, unicode strings are indicated by the \"u\" prefix, as above. In v3, strings are unicode-encoded by default."
+ "Note: When using python to construct strings, the handling of unicode differs between v2 and v3. In v2, unicode strings are indicated by the \"u\" prefix, as above. In v3, strings are unicode-encoded by default."
]
},
{
@@ -425,7 +425,7 @@
"source": [
"### Character substrings\n",
"\n",
- "Similarly, the `tf.strings.substr` operation accepts the \"`unit`\" parameter, and uses it to determine what kind of offsets the \"`pos`\" and \"`len`\" paremeters contain."
+ "Similarly, the `tf.strings.substr` operation accepts the \"`unit`\" parameter, and uses it to determine what kind of offsets the \"`pos`\" and \"`len`\" parameters contain."
]
},
{
@@ -587,7 +587,7 @@
"id": "CapnbShuGU8i"
},
"source": [
- "First, we decode the sentences into character codepoints, and find the script identifeir for each character."
+ "First, we decode the sentences into character codepoints, and find the script identifier for each character."
]
},
{
diff --git a/site/en/r1/tutorials/representation/word2vec.md b/site/en/r1/tutorials/representation/word2vec.md
index f6a27c68f3c..517a5dbc5c5 100644
--- a/site/en/r1/tutorials/representation/word2vec.md
+++ b/site/en/r1/tutorials/representation/word2vec.md
@@ -36,7 +36,7 @@ like to get your hands dirty with the details.
Image and audio processing systems work with rich, high-dimensional datasets
encoded as vectors of the individual raw pixel-intensities for image data, or
-e.g. power spectral density coefficients for audio data. For tasks like object
+e.g., power spectral density coefficients for audio data. For tasks like object
or speech recognition we know that all the information required to successfully
perform the task is encoded in the data (because humans can perform these tasks
from the raw data). However, natural language processing systems traditionally
@@ -109,7 +109,7 @@ $$
where \\(\text{score}(w_t, h)\\) computes the compatibility of word \\(w_t\\)
with the context \\(h\\) (a dot product is commonly used). We train this model
by maximizing its [log-likelihood](https://en.wikipedia.org/wiki/Likelihood_function)
-on the training set, i.e. by maximizing
+on the training set, i.e., by maximizing
$$
\begin{align}
@@ -176,7 +176,7 @@ As an example, let's consider the dataset
We first form a dataset of words and the contexts in which they appear. We
could define 'context' in any way that makes sense, and in fact people have
looked at syntactic contexts (i.e. the syntactic dependents of the current
-target word, see e.g.
+target word, see e.g.,
[Levy et al.](https://levyomer.files.wordpress.com/2014/04/dependency-based-word-embeddings-acl-2014.pdf)),
words-to-the-left of the target, words-to-the-right of the target, etc. For now,
let's stick to the vanilla definition and define 'context' as the window
@@ -204,7 +204,7 @@ where the goal is to predict `the` from `quick`. We select `num_noise` number
of noisy (contrastive) examples by drawing from some noise distribution,
typically the unigram distribution, \\(P(w)\\). For simplicity let's say
`num_noise=1` and we select `sheep` as a noisy example. Next we compute the
-loss for this pair of observed and noisy examples, i.e. the objective at time
+loss for this pair of observed and noisy examples, i.e., the objective at time
step \\(t\\) becomes
$$J^{(t)}_\text{NEG} = \log Q_\theta(D=1 | \text{the, quick}) +
@@ -212,7 +212,7 @@ $$J^{(t)}_\text{NEG} = \log Q_\theta(D=1 | \text{the, quick}) +
The goal is to make an update to the embedding parameters \\(\theta\\) to improve
(in this case, maximize) this objective function. We do this by deriving the
-gradient of the loss with respect to the embedding parameters \\(\theta\\), i.e.
+gradient of the loss with respect to the embedding parameters \\(\theta\\), i.e.,
\\(\frac{\partial}{\partial \theta} J_\text{NEG}\\) (luckily TensorFlow provides
easy helper functions for doing this!). We then perform an update to the
embeddings by taking a small step in the direction of the gradient. When this
@@ -227,7 +227,7 @@ When we inspect these visualizations it becomes apparent that the vectors
capture some general, and in fact quite useful, semantic information about
words and their relationships to one another. It was very interesting when we
first discovered that certain directions in the induced vector space specialize
-towards certain semantic relationships, e.g. *male-female*, *verb tense* and
+towards certain semantic relationships, e.g., *male-female*, *verb tense* and
even *country-capital* relationships between words, as illustrated in the figure
below (see also for example
[Mikolov et al., 2013](https://www.aclweb.org/anthology/N13-1090)).
@@ -327,7 +327,7 @@ for inputs, labels in generate_batch(...):
```
See the full example code in
-[tensorflow/examples/tutorials/word2vec/word2vec_basic.py](https://www.tensorflow.org/code/tensorflow/examples/tutorials/word2vec/word2vec_basic.py).
+[tensorflow/examples/tutorials/word2vec/word2vec_basic.py](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/examples/tutorials/word2vec/word2vec_basic.py).
## Visualizing the learned embeddings
@@ -341,7 +341,7 @@ t-SNE.
Et voila! As expected, words that are similar end up clustering nearby each
other. For a more heavyweight implementation of word2vec that showcases more of
the advanced features of TensorFlow, see the implementation in
-[models/tutorials/embedding/word2vec.py](https://github.com/tensorflow/models/tree/master/research/tutorials/embedding/word2vec.py).
+[models/tutorials/embedding/word2vec.py](https://github.com/tensorflow/models/tree/r1.15/research/tutorials/embedding/word2vec.py).
## Evaluating embeddings: analogical reasoning
@@ -357,7 +357,7 @@ Download the dataset for this task from
To see how we do this evaluation, have a look at the `build_eval_graph()` and
`eval()` functions in
-[models/tutorials/embedding/word2vec.py](https://github.com/tensorflow/models/tree/master/research/tutorials/embedding/word2vec.py).
+[models/tutorials/embedding/word2vec.py](https://github.com/tensorflow/models/tree/r1.15/research/tutorials/embedding/word2vec.py).
The choice of hyperparameters can strongly influence the accuracy on this task.
To achieve state-of-the-art performance on this task requires training over a
diff --git a/site/en/r1/tutorials/sequences/audio_recognition.md b/site/en/r1/tutorials/sequences/audio_recognition.md
index 8ad71b88a3c..0388514ec92 100644
--- a/site/en/r1/tutorials/sequences/audio_recognition.md
+++ b/site/en/r1/tutorials/sequences/audio_recognition.md
@@ -159,9 +159,9 @@ accuracy. If the training accuracy increases but the validation doesn't, that's
a sign that overfitting is occurring, and your model is only learning things
about the training clips, not broader patterns that generalize.
-## Tensorboard
+## TensorBoard
-A good way to visualize how the training is progressing is using Tensorboard. By
+A good way to visualize how the training is progressing is using TensorBoard. By
default, the script saves out events to /tmp/retrain_logs, and you can load
these by running:
diff --git a/site/en/r1/tutorials/sequences/recurrent.md b/site/en/r1/tutorials/sequences/recurrent.md
index 6654795d944..e7c1f8c0b16 100644
--- a/site/en/r1/tutorials/sequences/recurrent.md
+++ b/site/en/r1/tutorials/sequences/recurrent.md
@@ -2,7 +2,7 @@
## Introduction
-See [Understanding LSTM Networks](https://colah.github.io/posts/2015-08-Understanding-LSTMs/){:.external}
+See [Understanding LSTM Networks](https://colah.github.io/posts/2015-08-Understanding-LSTMs/)
for an introduction to recurrent neural networks and LSTMs.
## Language Modeling
diff --git a/site/en/r1/tutorials/sequences/recurrent_quickdraw.md b/site/en/r1/tutorials/sequences/recurrent_quickdraw.md
index 435076f629c..d6a85377d17 100644
--- a/site/en/r1/tutorials/sequences/recurrent_quickdraw.md
+++ b/site/en/r1/tutorials/sequences/recurrent_quickdraw.md
@@ -109,7 +109,7 @@ This download will take a while and download a bit more than 23GB of data.
To convert the `ndjson` files to
[TFRecord](../../api_guides/python/python_io.md#TFRecords_Format_Details) files containing
-[`tf.train.Example`](https://www.tensorflow.org/code/tensorflow/core/example/example.proto)
+[`tf.train.Example`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/example/example.proto)
protos run the following command.
```shell
@@ -213,7 +213,7 @@ screen coordinates and normalize the size such that the drawing has unit height.
Finally, we compute the differences between consecutive points and store these
as a `VarLenFeature` in a
-[tensorflow.Example](https://www.tensorflow.org/code/tensorflow/core/example/example.proto)
+[tensorflow.Example](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/example/example.proto)
under the key `ink`. In addition we store the `class_index` as a single entry
`FixedLengthFeature` and the `shape` of the `ink` as a `FixedLengthFeature` of
length 2.
diff --git a/site/en/r1/tutorials/sequences/text_generation.ipynb b/site/en/r1/tutorials/sequences/text_generation.ipynb
index 1350df2a486..84d942c8bd0 100644
--- a/site/en/r1/tutorials/sequences/text_generation.ipynb
+++ b/site/en/r1/tutorials/sequences/text_generation.ipynb
@@ -77,9 +77,9 @@
"id": "BwpJ5IffzRG6"
},
"source": [
- "This tutorial demonstrates how to generate text using a character-based RNN. We will work with a dataset of Shakespeare's writing from Andrej Karpathy's [The Unreasonable Effectiveness of Recurrent Neural Networks](http://karpathy.github.io/2015/05/21/rnn-effectiveness/). Given a sequence of characters from this data (\"Shakespear\"), train a model to predict the next character in the sequence (\"e\"). Longer sequences of text can be generated by calling the model repeatedly.\n",
+ "This tutorial demonstrates how to generate text using a character-based RNN. You will work with a dataset of Shakespeare's writing from Andrej Karpathy's [The Unreasonable Effectiveness of Recurrent Neural Networks](http://karpathy.github.io/2015/05/21/rnn-effectiveness/). Given a sequence of characters from this data (\"Shakespear\"), train a model to predict the next character in the sequence (\"e\"). Longer sequences of text can be generated by calling the model repeatedly.\n",
"\n",
- "Note: Enable GPU acceleration to execute this notebook faster. In Colab: *Runtime > Change runtime type > Hardware acclerator > GPU*. If running locally make sure TensorFlow version >= 1.11.\n",
+ "Note: Enable GPU acceleration to execute this notebook faster. In Colab: *Runtime > Change runtime type > Hardware accelerator > GPU*. If running locally make sure TensorFlow version >= 1.11.\n",
"\n",
"This tutorial includes runnable code implemented using [tf.keras](https://www.tensorflow.org/programmers_guide/keras) and [eager execution](https://www.tensorflow.org/programmers_guide/eager). The following is sample output when the model in this tutorial trained for 30 epochs, and started with the string \"Q\":\n",
"\n",
@@ -98,7 +98,7 @@
"To watch the next way with his father with his face?\n",
"\n",
"ESCALUS:\n",
- "The cause why then we are all resolved more sons.\n",
+ "The cause why then us all resolved more sons.\n",
"\n",
"VOLUMNIA:\n",
"O, no, no, no, no, no, no, no, no, no, no, no, no, no, no, no, no, no, no, no, no, it is no sin it should be dead,\n",
@@ -248,7 +248,7 @@
"source": [
"### Vectorize the text\n",
"\n",
- "Before training, we need to map strings to a numerical representation. Create two lookup tables: one mapping characters to numbers, and another for numbers to characters."
+ "Before training, you need to map strings to a numerical representation. Create two lookup tables: one mapping characters to numbers, and another for numbers to characters."
]
},
{
@@ -272,7 +272,7 @@
"id": "tZfqhkYCymwX"
},
"source": [
- "Now we have an integer representation for each character. Notice that we mapped the character as indexes from 0 to `len(unique)`."
+ "Now you have an integer representation for each character. Notice that you mapped the character as indexes from 0 to `len(unique)`."
]
},
{
@@ -316,7 +316,7 @@
"id": "wssHQ1oGymwe"
},
"source": [
- "Given a character, or a sequence of characters, what is the most probable next character? This is the task we're training the model to perform. The input to the model will be a sequence of characters, and we train the model to predict the output—the following character at each time step.\n",
+ "Given a character, or a sequence of characters, what is the most probable next character? This is the task you are training the model to perform. The input to the model will be a sequence of characters, and you train the model to predict the output—the following character at each time step.\n",
"\n",
"Since RNNs maintain an internal state that depends on the previously seen elements, given all the characters computed until this moment, what is the next character?\n"
]
@@ -346,7 +346,7 @@
},
"outputs": [],
"source": [
- "# The maximum length sentence we want for a single input in characters\n",
+ "# The maximum length sentence you want for a single input in characters\n",
"seq_length = 100\n",
"examples_per_epoch = len(text)//seq_length\n",
"\n",
@@ -458,7 +458,7 @@
"source": [
"### Create training batches\n",
"\n",
- "We used `tf.data` to split the text into manageable sequences. But before feeding this data into the model, we need to shuffle the data and pack it into batches."
+ "You used `tf.data` to split the text into manageable sequences. But before feeding this data into the model, you need to shuffle the data and pack it into batches."
]
},
{
@@ -650,7 +650,7 @@
"id": "uwv0gEkURfx1"
},
"source": [
- "To get actual predictions from the model we need to sample from the output distribution, to get actual character indices. This distribution is defined by the logits over the character vocabulary.\n",
+ "To get actual predictions from the model you need to sample from the output distribution, to get actual character indices. This distribution is defined by the logits over the character vocabulary.\n",
"\n",
"Note: It is important to _sample_ from this distribution as taking the _argmax_ of the distribution can easily get the model stuck in a loop.\n",
"\n",
@@ -746,7 +746,7 @@
"source": [
"The standard `tf.keras.losses.sparse_categorical_crossentropy` loss function works in this case because it is applied across the last dimension of the predictions.\n",
"\n",
- "Because our model returns logits, we need to set the `from_logits` flag.\n"
+ "Because our model returns logits, you need to set the `from_logits` flag.\n"
]
},
{
@@ -771,7 +771,7 @@
"id": "jeOXriLcymww"
},
"source": [
- "Configure the training procedure using the `tf.keras.Model.compile` method. We'll use `tf.train.AdamOptimizer` with default arguments and the loss function."
+ "Configure the training procedure using the `tf.keras.Model.compile` method. You'll use `tf.train.AdamOptimizer` with default arguments and the loss function."
]
},
{
@@ -891,7 +891,7 @@
"\n",
"Because of the way the RNN state is passed from timestep to timestep, the model only accepts a fixed batch size once built.\n",
"\n",
- "To run the model with a different `batch_size`, we need to rebuild the model and restore the weights from the checkpoint.\n"
+ "To run the model with a different `batch_size`, you need to rebuild the model and restore the weights from the checkpoint.\n"
]
},
{
@@ -992,7 +992,7 @@
" predictions = predictions / temperature\n",
" predicted_id = tf.multinomial(predictions, num_samples=1)[-1,0].numpy()\n",
"\n",
- " # We pass the predicted word as the next input to the model\n",
+ " # You pass the predicted word as the next input to the model\n",
" # along with the previous hidden state\n",
" input_eval = tf.expand_dims([predicted_id], 0)\n",
"\n",
@@ -1035,11 +1035,11 @@
"\n",
"So now that you've seen how to run the model manually let's unpack the training loop, and implement it ourselves. This gives a starting point, for example, to implement _curriculum learning_ to help stabilize the model's open-loop output.\n",
"\n",
- "We will use `tf.GradientTape` to track the gradients. You can learn more about this approach by reading the [eager execution guide](https://www.tensorflow.org/r1/guide/eager).\n",
+ "You will use `tf.GradientTape` to track the gradients. You can learn more about this approach by reading the [eager execution guide](https://www.tensorflow.org/r1/guide/eager).\n",
"\n",
"The procedure works as follows:\n",
"\n",
- "* First, initialize the RNN state. We do this by calling the `tf.keras.Model.reset_states` method.\n",
+ "* First, initialize the RNN state. You do this by calling the `tf.keras.Model.reset_states` method.\n",
"\n",
"* Next, iterate over the dataset (batch by batch) and calculate the *predictions* associated with each.\n",
"\n",
diff --git a/site/en/tutorials/_index.yaml b/site/en/tutorials/_index.yaml
index c38a23ddb35..0d09f04c5c7 100644
--- a/site/en/tutorials/_index.yaml
+++ b/site/en/tutorials/_index.yaml
@@ -16,8 +16,9 @@ landing_page:
- description: >