diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 62770eb..228ee93 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,15 +29,16 @@ jobs: python -m pip install --upgrade pip # cpu version of pytorch pip install torch==1.6.0+cpu -f https://download.pytorch.org/whl/torch_stable.html + # Install dependencies for docs and tests pip install stable_baselines3[extra,tests,docs] # Install master version pip install git+https://github.com/DLR-RM/stable-baselines3 pip install . # Use headless version pip install opencv-python-headless - # - name: Build the doc - # run: | - # make doc + - name: Build the doc + run: | + make doc - name: Type check run: | make type diff --git a/Makefile b/Makefile index 639d266..d740b60 100644 --- a/Makefile +++ b/Makefile @@ -28,6 +28,11 @@ check-codestyle: commit-checks: format type lint +doc: + cd docs && make html + +spelling: + cd docs && make spelling # PyPi package release release: @@ -41,4 +46,4 @@ test-release: python setup.py bdist_wheel twine upload --repository-url https://test.pypi.org/legacy/ dist/* -.PHONY: lint format check-codestyle commit-checks +.PHONY: lint format check-codestyle commit-checks doc spelling diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..47f98cd --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = -W # make warnings fatal +SPHINXBUILD = sphinx-build +SPHINXPROJ = StableBaselines +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) \ No newline at end of file diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..1427a79 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,25 @@ +## Stable Baselines3 Documentation + +This folder contains documentation for the RL baselines. + + +### Build the Documentation + +#### Install Sphinx and Theme + +``` +pip install sphinx sphinx-autobuild sphinx-rtd-theme +``` + +#### Building the Docs + +In the `docs/` folder: +``` +make html +``` + +if you want to building each time a file is changed: + +``` +sphinx-autobuild . _build/html +``` diff --git a/docs/_static/css/baselines_theme.css b/docs/_static/css/baselines_theme.css new file mode 100644 index 0000000..450864e --- /dev/null +++ b/docs/_static/css/baselines_theme.css @@ -0,0 +1,61 @@ +/* Main colors adapted from pytorch doc */ +:root{ + --main-bg-color: #343A40; + --link-color: #FD7E14; +} + +/* Header fonts y */ +h1, h2, .rst-content .toctree-wrapper p.caption, h3, h4, h5, h6, legend, p.caption { + font-family: "Lato","proxima-nova","Helvetica Neue",Arial,sans-serif; +} + + +/* Docs background */ +.wy-side-nav-search{ + background-color: var(--main-bg-color); +} + +/* Mobile version */ +.wy-nav-top{ + background-color: var(--main-bg-color); +} + +/* Change link colors (except for the menu) */ +a { + color: var(--link-color); +} + +a:hover { + color: #4F778F; +} + +.wy-menu a { + color: #b3b3b3; +} + +.wy-menu a:hover { + color: #b3b3b3; +} + +a.icon.icon-home { + color: #b3b3b3; +} + +.version{ + color: var(--link-color) !important; +} + + +/* Make code blocks have a background */ +.codeblock,pre.literal-block,.rst-content .literal-block,.rst-content pre.literal-block,div[class^='highlight'] { + background: #f8f8f8;; +} + +/* Change style of types in the docstrings .rst-content .field-list */ +.field-list .xref.py.docutils, .field-list code.docutils, .field-list .docutils.literal.notranslate +{ + border: None; + padding-left: 0; + padding-right: 0; + color: #404040; +} diff --git a/docs/_static/img/colab-badge.svg b/docs/_static/img/colab-badge.svg new file mode 100644 index 0000000..c08066e --- /dev/null +++ b/docs/_static/img/colab-badge.svg @@ -0,0 +1 @@ + Open in ColabOpen in Colab diff --git a/docs/_static/img/colab.svg b/docs/_static/img/colab.svg new file mode 100644 index 0000000..c2d30e9 --- /dev/null +++ b/docs/_static/img/colab.svg @@ -0,0 +1,7 @@ + + + + + + + \ No newline at end of file diff --git a/docs/_static/img/logo.png b/docs/_static/img/logo.png new file mode 100644 index 0000000..7cec1c3 Binary files /dev/null and b/docs/_static/img/logo.png differ diff --git a/docs/_static/img/logo.png~ b/docs/_static/img/logo.png~ new file mode 100644 index 0000000..ef592f5 Binary files /dev/null and b/docs/_static/img/logo.png~ differ diff --git a/docs/conda_env.yml b/docs/conda_env.yml new file mode 100644 index 0000000..35e604d --- /dev/null +++ b/docs/conda_env.yml @@ -0,0 +1,18 @@ +name: root +channels: + - pytorch + - defaults +dependencies: + - cpuonly=1.0=0 + - pip=20.2 + - python=3.6 + - pytorch=1.5.0=py3.6_cpu_0 + - pip: + - gym>=0.17.2 + - cloudpickle + - opencv-python-headless + - pandas + - numpy + - matplotlib + - sphinx_autodoc_typehints + - stable-baselines3>=0.9.0 diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..ed733df --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,219 @@ +# -*- coding: utf-8 -*- +# +# Configuration file for the Sphinx documentation builder. +# +# This file does only contain a selection of the most common options. For a +# full list see the documentation: +# http://www.sphinx-doc.org/en/master/config + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +import os +import sys +from unittest.mock import MagicMock + +# We CANNOT enable 'sphinxcontrib.spelling' because ReadTheDocs.org does not support +# PyEnchant. +try: + import sphinxcontrib.spelling # noqa: F401 + + enable_spell_check = True +except ImportError: + enable_spell_check = False + +# source code directory, relative to this file, for sphinx-autobuild +sys.path.insert(0, os.path.abspath("..")) + + +class Mock(MagicMock): + __subclasses__ = [] + + @classmethod + def __getattr__(cls, name): + return MagicMock() + + +# Mock modules that requires C modules +# Note: because of that we cannot test examples using CI +# 'torch', 'torch.nn', 'torch.nn.functional', +# DO not mock modules for now, we will need to do that for read the docs later +MOCK_MODULES = [] +sys.modules.update((mod_name, Mock()) for mod_name in MOCK_MODULES) + +# Read version from file +version_file = os.path.join(os.path.dirname(__file__), "../sb3_contrib", "version.txt") +with open(version_file, "r") as file_handler: + __version__ = file_handler.read().strip() + +# -- Project information ----------------------------------------------------- + +project = "Stable Baselines3 - Contrib" +copyright = "2020, Stable Baselines3" +author = "Stable Baselines3 Contributors" + +# The short X.Y version +version = "master (" + __version__ + " )" +# The full version, including alpha/beta/rc tags +release = __version__ + + +# -- General configuration --------------------------------------------------- + +# If your documentation needs a minimal Sphinx version, state it here. +# +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + "sphinx.ext.autodoc", + "sphinx_autodoc_typehints", + "sphinx.ext.autosummary", + "sphinx.ext.mathjax", + "sphinx.ext.ifconfig", + "sphinx.ext.viewcode", + # 'sphinx.ext.intersphinx', + # 'sphinx.ext.doctest' +] + +if enable_spell_check: + extensions.append("sphinxcontrib.spelling") + +# Add any paths that contain templates here, relative to this directory. +templates_path = ["_templates"] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = ".rst" + +# The master toctree document. +master_doc = "index" + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path . +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = "sphinx" + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. + +# Fix for read the docs +on_rtd = os.environ.get("READTHEDOCS") == "True" +if on_rtd: + html_theme = "default" +else: + html_theme = "sphinx_rtd_theme" + +html_logo = "_static/img/logo.png" + + +def setup(app): + app.add_css_file("css/baselines_theme.css") + + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +# html_theme_options = {} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ["_static"] + +# Custom sidebar templates, must be a dictionary that maps document names +# to template names. +# +# The default sidebars (for documents that don't match any pattern) are +# defined by theme itself. Builtin themes are using these templates by +# default: ``['localtoc.html', 'relations.html', 'sourcelink.html', +# 'searchbox.html']``. +# +# html_sidebars = {} + + +# -- Options for HTMLHelp output --------------------------------------------- + +# Output file base name for HTML help builder. +htmlhelp_basename = "StableBaselines3doc" + + +# -- Options for LaTeX output ------------------------------------------------ + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, "StableBaselines3.tex", "Stable Baselines3 Documentation", "Stable Baselines3 Contributors", "manual"), +] + + +# -- Options for manual page output ------------------------------------------ + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [(master_doc, "stablebaselines3", "Stable Baselines3 Documentation", [author], 1)] + + +# -- Options for Texinfo output ---------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + ( + master_doc, + "StableBaselines3", + "Stable Baselines3 Documentation", + author, + "StableBaselines3", + "One line description of project.", + "Miscellaneous", + ), +] + + +# -- Extension configuration ------------------------------------------------- + +# Example configuration for intersphinx: refer to the Python standard library. +# intersphinx_mapping = { +# 'python': ('https://docs.python.org/3/', None), +# 'numpy': ('http://docs.scipy.org/doc/numpy/', None), +# 'torch': ('http://pytorch.org/docs/master/', None), +# } diff --git a/docs/guide/algos.rst b/docs/guide/algos.rst new file mode 100644 index 0000000..1d863ea --- /dev/null +++ b/docs/guide/algos.rst @@ -0,0 +1,30 @@ +RL Algorithms +============= + +This table displays the rl algorithms that are implemented in the Stable Baselines3 contrib project, +along with some useful characteristics: support for discrete/continuous actions, multiprocessing. + +.. +.. ============ =========== ============ ================= =============== ================ +.. Name ``Box`` ``Discrete`` ``MultiDiscrete`` ``MultiBinary`` Multi Processing +.. ============ =========== ============ ================= =============== ================ +.. A2C ✔️ ✔️ ✔️ ✔️ ✔️ +.. DDPG ✔️ ❌ ❌ ❌ ❌ +.. DQN ❌ ✔️ ❌ ❌ ❌ +.. PPO ✔️ ✔️ ✔️ ✔️ ✔️ +.. SAC ✔️ ❌ ❌ ❌ ❌ +.. TD3 ✔️ ❌ ❌ ❌ ❌ +.. ============ =========== ============ ================= =============== ================ + + +.. .. note:: +.. Non-array spaces such as ``Dict`` or ``Tuple`` are not currently supported by any algorithm. +.. +.. Actions ``gym.spaces``: +.. +.. - ``Box``: A N-dimensional box that contains every point in the action +.. space. +.. - ``Discrete``: A list of possible actions, where each timestep only +.. one of the actions can be used. +.. - ``MultiDiscrete``: A list of possible actions, where each timestep only one action of each discrete set can be used. +.. - ``MultiBinary``: A list of possible actions, where each timestep any of the actions can be used in any combination. diff --git a/docs/guide/examples.rst b/docs/guide/examples.rst new file mode 100644 index 0000000..65b2d08 --- /dev/null +++ b/docs/guide/examples.rst @@ -0,0 +1,40 @@ +.. _examples: + +Examples +======== + +WIP + +.. PyBullet: Normalizing input features +.. ------------------------------------ +.. +.. Normalizing input features may be essential to successful training of an RL agent +.. (by default, images are scaled but not other types of input), +.. for instance when training on `PyBullet `__ environments. For that, a wrapper exists and +.. will compute a running average and standard deviation of input features (it can do the same for rewards). +.. + +.. .. note:: +.. +.. you need to install pybullet with ``pip install pybullet`` +.. +.. +.. .. image:: ../_static/img/colab-badge.svg +.. :target: https://colab.research.google.com/github/Stable-Baselines-Team/rl-colab-notebooks/blob/sb3/pybullet.ipynb +.. +.. +.. .. code-block:: python +.. +.. import gym +.. import pybullet_envs +.. +.. from stable_baselines3.common.vec_env import DummyVecEnv, VecNormalize +.. from stable_baselines3 import PPO +.. +.. env = DummyVecEnv([lambda: gym.make("HalfCheetahBulletEnv-v0")]) +.. # Automatically normalize the input features and reward +.. env = VecNormalize(env, norm_obs=True, norm_reward=True, +.. clip_obs=10.) +.. +.. model = PPO('MlpPolicy', env) +.. model.learn(total_timesteps=2000) diff --git a/docs/guide/install.rst b/docs/guide/install.rst new file mode 100644 index 0000000..dcffed8 --- /dev/null +++ b/docs/guide/install.rst @@ -0,0 +1,37 @@ +.. _install: + +Installation +============ + +Prerequisites +------------- + +Please read `Stable-Baselines3 installation guide `_ first. + + +Stable Release +~~~~~~~~~~~~~~ +To install Stable Baselines3 contrib with pip, execute: + +.. code-block:: bash + + pip install sb3-contrib + + +Bleeding-edge version +--------------------- + +.. code-block:: bash + + pip install git+https://github.com/Stable-Baselines-Team/stable-baselines3-contrib/ + + +Development version +------------------- + +To contribute to Stable-Baselines3, with support for running tests and building the documentation. + +.. code-block:: bash + + git clone https://github.com/Stable-Baselines-Team/stable-baselines3-contrib/ && cd stable-baselines3-contrib + pip install -e . diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 0000000..b77a85f --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,78 @@ +.. Stable Baselines3 documentation master file, created by + sphinx-quickstart on Thu Sep 26 11:06:54 2019. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to Stable Baselines3 Contrib docs! +========================================== + +Contrib package for `Stable Baselines3 `_ - Experimental code. + + +Github repository: https://github.com/Stable-Baselines-Team/stable-baselines3-contrib + +SB3 repository: https://github.com/DLR-RM/stable-baselines3 + +RL Baselines3 Zoo (collection of pre-trained agents): https://github.com/DLR-RM/rl-baselines3-zoo + +RL Baselines3 Zoo also offers a simple interface to train, evaluate agents and do hyperparameter tuning. + + +.. toctree:: + :maxdepth: 2 + :caption: User Guide + + guide/install + guide/algos + guide/examples + + +.. .. toctree:: +.. :maxdepth: 1 +.. :caption: RL Algorithms +.. +.. modules/a2c + +.. .. toctree:: +.. :maxdepth: 1 +.. :caption: Common +.. +.. common/atari_wrappers + +.. toctree:: + :maxdepth: 1 + :caption: Misc + + misc/changelog + + +Citing Stable Baselines3 +------------------------ +To cite this project in publications: + +.. code-block:: bibtex + + @misc{stable-baselines3, + author = {Raffin, Antonin and Hill, Ashley and Ernestus, Maximilian and Gleave, Adam and Kanervisto, Anssi and Dormann, Noah}, + title = {Stable Baselines3}, + year = {2019}, + publisher = {GitHub}, + journal = {GitHub repository}, + howpublished = {\url{https://github.com/DLR-RM/stable-baselines3}}, + } + +Contributing +------------ + +To any interested in making the rl baselines better, there are still some improvements +that need to be done. +You can check issues in the `repo `_. + +If you want to contribute, please read `CONTRIBUTING.md `_ first. + +Indices and tables +------------------- + +* :ref:`genindex` +* :ref:`search` +* :ref:`modindex` diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 0000000..22b5fff --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,36 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=_build +set SPHINXPROJ=StableBaselines + +if "%1" == "" goto help + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% + +:end +popd diff --git a/docs/misc/changelog.rst b/docs/misc/changelog.rst new file mode 100644 index 0000000..6cb9a38 --- /dev/null +++ b/docs/misc/changelog.rst @@ -0,0 +1,42 @@ +.. _changelog: + +Changelog +========== + + +Pre-Release 0.10.0a0 (WIP) +------------------------------ + +Breaking Changes: +^^^^^^^^^^^^^^^^^ + +New Features: +^^^^^^^^^^^^^ + +Bug Fixes: +^^^^^^^^^^ + +Deprecations: +^^^^^^^^^^^^^ + +Others: +^^^^^^^ + +Documentation: +^^^^^^^^^^^^^^ + + +Maintainers +----------- + +Stable-Baselines3 is currently maintained by `Antonin Raffin`_ (aka `@araffin`_), `Ashley Hill`_ (aka @hill-a), +`Maximilian Ernestus`_ (aka @erniejunior), `Adam Gleave`_ (`@AdamGleave`_) and `Anssi Kanervisto`_ (aka `@Miffyli`_). + +.. _Ashley Hill: https://github.com/hill-a +.. _Antonin Raffin: https://araffin.github.io/ +.. _Maximilian Ernestus: https://github.com/erniejunior +.. _Adam Gleave: https://gleave.me/ +.. _@araffin: https://github.com/araffin +.. _@AdamGleave: https://github.com/adamgleave +.. _Anssi Kanervisto: https://github.com/Miffyli +.. _@Miffyli: https://github.com/Miffyli diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt new file mode 100644 index 0000000..42669bf --- /dev/null +++ b/docs/spelling_wordlist.txt @@ -0,0 +1,121 @@ +py +env +atari +argparse +Argparse +TensorFlow +feedforward +envs +VecEnv +pretrain +petrained +tf +th +nn +np +str +mujoco +cpu +ndarray +ndarrays +timestep +timesteps +stepsize +dataset +adam +fn +normalisation +Kullback +Leibler +boolean +deserialized +pretrained +minibatch +subprocesses +ArgumentParser +Tensorflow +Gaussian +approximator +minibatches +hyperparameters +hyperparameter +vectorized +rl +colab +dataloader +npz +datasets +vf +logits +num +Utils +backpropagate +prepend +NaN +preprocessing +Cloudpickle +async +multiprocess +tensorflow +mlp +cnn +neglogp +tanh +coef +repo +Huber +params +ppo +arxiv +Arxiv +func +DQN +Uhlenbeck +Ornstein +multithread +cancelled +Tensorboard +parallelize +customising +serializable +Multiprocessed +cartpole +toolset +lstm +rescale +ffmpeg +avconv +unnormalized +Github +pre +preprocess +backend +attr +preprocess +Antonin +Raffin +araffin +Homebrew +Numpy +Theano +rollout +kfac +Piecewise +csv +nvidia +visdom +tensorboard +preprocessed +namespace +sklearn +GoalEnv +Torchy +pytorch +dicts +optimizers +Deprecations +forkserver +cuda +Polyak +gSDE +rollouts diff --git a/sb3_contrib/version.txt b/sb3_contrib/version.txt index 63aaf3d..37f1777 100644 --- a/sb3_contrib/version.txt +++ b/sb3_contrib/version.txt @@ -1 +1 @@ -0.9.0a2 +0.10.0a0