diff --git a/.flake8 b/.flake8 new file mode 100644 index 00000000000..ec083d68034 --- /dev/null +++ b/.flake8 @@ -0,0 +1,5 @@ +[flake8] +ignore = W293,E301,E271,E265,W291,E722,E302,C901,E225,E128,E122,E226,E231 +max-line-length = 160 +exclude = tests/* +max-complexity = 10 diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index 70971c53b5a..d5be139ad02 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -1,8 +1,21 @@ # When making commits that are strictly formatting/style changes, add the -# commit hash here, so git blame can ignore the change. -# See docs for more details: +# commit hash here, so git blame can ignore the change. See docs for more +# details: # https://git-scm.com/docs/git-config#Documentation/git-config.txt-blameignoreRevsFile - +# +# +# You should be able to execute either +# ./tools/configure-git-blame-ignore-revs.bat or +# ./tools/configure-git-blame-ignore-revs.sh +# # Example entries: -# # initial black-format +# +# # initial black-format # # rename something internal +6e748726282d1acb9a4f9f264ee679c474c4b8f5 # Apply pygrade --36plus on IPython/core/tests/test_inputtransformer.py. +0233e65d8086d0ec34acb8685b7a5411633f0899 # apply pyupgrade to IPython/extensions/tests/test_autoreload.py +a6a7e4dd7e51b892147895006d3a2a6c34b79ae6 # apply black to IPython/extensions/tests/test_autoreload.py +c5ca5a8f25432dfd6b9eccbbe446a8348bf37cfa # apply pyupgrade to IPython/extensions/autoreload.py +50624b84ccdece781750f5eb635a9efbf2fe30d6 # apply black to IPython/extensions/autoreload.py +b7aaa47412b96379198705955004930c57f9d74a # apply pyupgrade to IPython/extensions/autoreload.py +9c7476a88af3e567426b412f1b3c778401d8f6aa # apply black to IPython/extensions/autoreload.py diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md new file mode 100644 index 00000000000..2a6d4877c68 --- /dev/null +++ b/.github/ISSUE_TEMPLATE.md @@ -0,0 +1,16 @@ +--- +name: Bug report / Question / Feature +about: Anything related to IPython itsel +title: '' +labels: '' +assignees: '' + +--- + + diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 00000000000..f18fb39392f --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,39 @@ +name: Build docs + +on: [push, pull_request] + +permissions: + contents: read + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: 3.x + - name: Install Graphviz + run: | + sudo apt-get update + sudo apt-get install graphviz + - name: Install Python dependencies + run: | + python -m pip install --upgrade pip setuptools coverage rstvalidator + pip install -r docs/requirements.txt + - name: Build docs + run: | + python -m rstvalidator long_description.rst + python tools/fixup_whats_new_pr.py + make -C docs/ html SPHINXOPTS="-W" \ + PYTHON="coverage run -a" \ + SPHINXBUILD="coverage run -a -m sphinx.cmd.build" + - name: Generate coverage xml + run: | + coverage combine `find . -name .coverage\*` && coverage xml + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v2 + with: + name: Docs diff --git a/.github/workflows/downstream.yml b/.github/workflows/downstream.yml new file mode 100644 index 00000000000..e6206ae71f1 --- /dev/null +++ b/.github/workflows/downstream.yml @@ -0,0 +1,52 @@ +name: Run Downstream tests + +on: + push: + pull_request: + # Run weekly on Monday at 1:23 UTC + schedule: + - cron: '23 1 * * 1' + workflow_dispatch: + +permissions: + contents: read + +jobs: + test: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest] + python-version: ["3.9"] + include: + - os: macos-latest + python-version: "3.9" + + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Update Python installer + run: | + python -m pip install --upgrade pip setuptools wheel + - name: Install ipykernel + run: | + cd .. + git clone https://github.com/ipython/ipykernel + cd ipykernel + pip install -e .[test] + cd .. + - name: Install and update Python dependencies + run: | + python -m pip install --upgrade -e file://$PWD#egg=ipython[test] + # we must install IPython after ipykernel to get the right versions. + python -m pip install --upgrade --upgrade-strategy eager flaky ipyparallel + python -m pip install --upgrade 'pytest<7' + - name: pytest + env: + COLUMNS: 120 + run: | + cd ../ipykernel + pytest diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml new file mode 100644 index 00000000000..c7fa22c7210 --- /dev/null +++ b/.github/workflows/mypy.yml @@ -0,0 +1,41 @@ +name: Run MyPy + +on: + push: + branches: [ main, 7.x] + pull_request: + branches: [ main, 7.x] + +permissions: + contents: read + +jobs: + build: + + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.x"] + + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install mypy pyflakes flake8 + - name: Lint with mypy + run: | + set -e + mypy -p IPython.terminal + mypy -p IPython.core.magics + mypy -p IPython.core.guarded_eval + mypy -p IPython.core.completer + - name: Lint with pyflakes + run: | + set -e + flake8 IPython/core/magics/script.py + flake8 IPython/core/magics/packaging.py diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml new file mode 100644 index 00000000000..fc1f19e0912 --- /dev/null +++ b/.github/workflows/python-package.yml @@ -0,0 +1,39 @@ +# This workflow will install Python dependencies, run tests and lint with a variety of Python versions +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: Python package + +permissions: + contents: read + +on: + push: + branches: [ main, 7.x ] + pull_request: + branches: [ main, 7.x ] + +jobs: + formatting: + + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: 3.x + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install darker==1.5.1 black==22.10.0 + - name: Lint with darker + run: | + darker -r 60625f241f298b5039cb2debc365db38aa7bb522 --check --diff . || ( + echo "Changes need auto-formatting. Run:" + echo " darker -r 60625f241f298b5039cb2debc365db38aa7bb522" + echo "then commit and push changes to fix." + exit 1 + ) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 00000000000..73968555c4e --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,83 @@ +name: Run tests + +on: + push: + branches: + - main + - '*.x' + pull_request: + # Run weekly on Monday at 1:23 UTC + schedule: + - cron: '23 1 * * 1' + workflow_dispatch: + + +jobs: + test: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest] + python-version: ["3.8", "3.9", "3.10", "3.11"] + deps: [test_extra] + # Test all on ubuntu, test ends on macos + include: + - os: macos-latest + python-version: "3.8" + deps: test_extra + - os: macos-latest + python-version: "3.11" + deps: test_extra + # Tests minimal dependencies set + - os: ubuntu-latest + python-version: "3.11" + deps: test + # Tests latest development Python version + - os: ubuntu-latest + python-version: "3.12-dev" + deps: test + # Installing optional dependencies stuff takes ages on PyPy + - os: ubuntu-latest + python-version: "pypy-3.8" + deps: test + - os: windows-latest + python-version: "pypy-3.8" + deps: test + - os: macos-latest + python-version: "pypy-3.8" + deps: test + + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + cache: pip + - name: Install latex + if: runner.os == 'Linux' && matrix.deps == 'test_extra' + run: echo "disable latex for now, issues in mirros" #sudo apt-get -yq -o Acquire::Retries=3 --no-install-suggests --no-install-recommends install texlive dvipng + - name: Install and update Python dependencies + run: | + python -m pip install --upgrade pip setuptools wheel build + python -m pip install --upgrade -e .[${{ matrix.deps }}] + python -m pip install --upgrade check-manifest pytest-cov + - name: Try building with Python build + if: runner.os != 'Windows' # setup.py does not support sdist on Windows + run: | + python -m build + shasum -a 256 dist/* + - name: Check manifest + if: runner.os != 'Windows' # setup.py does not support sdist on Windows + run: check-manifest + - name: pytest + env: + COLUMNS: 120 + run: | + pytest --color=yes -raXxs ${{ startsWith(matrix.python-version, 'pypy') && ' ' || '--cov --cov-report=xml' }} + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v3 + with: + name: Test + files: /home/runner/work/ipython/ipython/coverage.xml diff --git a/.gitignore b/.gitignore index 1fc0e22a320..3b6963b6317 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ docs/man/*.gz docs/source/api/generated docs/source/config/options docs/source/config/shortcuts/*.csv +docs/source/savefig docs/source/interactive/magics-generated.txt docs/gh-pages jupyter_notebook/notebook/static/mathjax @@ -23,8 +24,16 @@ __pycache__ .cache .coverage *.swp -.vscode .pytest_cache .python-version venv*/ +.mypy_cache/ + +# jetbrains ide stuff +*.iml .idea/ + +# vscode ide stuff +*.code-workspace +.history +.vscode diff --git a/.mailmap b/.mailmap index 8d4757e6865..ab05ba24ba2 100644 --- a/.mailmap +++ b/.mailmap @@ -2,6 +2,9 @@ A. J. Holyoake ajholyoake Alok Singh Alok Singh <8325708+alok@users.noreply.github.com> Aaron Culich Aaron Culich Aron Ahmadia ahmadia +Arthur Svistunov <18216480+madbird1304@users.noreply.github.com> +Arthur Svistunov <18216480+madbird1304@users.noreply.github.com> +Adam Hackbarth Benjamin Ragan-Kelley Benjamin Ragan-Kelley Min RK Benjamin Ragan-Kelley MinRK @@ -14,6 +17,8 @@ Brian E. Granger Brian Granger Brian E. Granger Brian Granger <> Brian E. Granger bgranger <> Brian E. Granger bgranger +Blazej Michalik <6691643+MrMino@users.noreply.github.com> +Blazej Michalik Christoph Gohlke cgohlke Cyrille Rossant rossant Damián Avila damianavila @@ -26,6 +31,7 @@ Dav Clark Dav Clark David Hirschfeld dhirschfeld David P. Sanders David P. Sanders David Warde-Farley David Warde-Farley <> +Dan Green-Leipciger Doug Blank Doug Blank Eugene Van den Bulke Eugene Van den Bulke Evan Patterson @@ -141,6 +147,9 @@ Silvia Vinyes silviav12 Srinivas Reddy Thatiparthy Srinivas Reddy Thatiparthy Sylvain Corlay Sylvain Corlay sylvain.corlay +Samuel Gaist +Richard Shadrach +Juan Luis Cano Rodríguez Tamir Bahar Tamir Bahar Ted Drain TD22057 Théophile Studer Théophile Studer diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000000..6af0afb1d23 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,16 @@ +# See https://pre-commit.com for more information +# See https://pre-commit.com/hooks.html for more hooks +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v3.2.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + - id: check-added-large-files + +- repo: https://github.com/akaihola/darker + rev: 1.3.1 + hooks: + - id: darker + diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index cccdfb3db14..00000000000 --- a/.travis.yml +++ /dev/null @@ -1,122 +0,0 @@ -# http://travis-ci.org/#!/ipython/ipython -language: python -os: linux - -addons: - apt: - packages: - - graphviz - -python: - - 3.6 - -sudo: false - -env: - global: - - PATH=$TRAVIS_BUILD_DIR/pandoc:$PATH - -group: edge - -before_install: - - | - # install Python on macOS - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then - env | sort - if ! which python$TRAVIS_PYTHON_VERSION; then - HOMEBREW_NO_AUTO_UPDATE=1 brew tap carreau/homebrew-python-frameworks - HOMEBREW_NO_AUTO_UPDATE=1 brew cask install python-framework-${TRAVIS_PYTHON_VERSION/./} - fi - python3 -m pip install virtualenv - python3 -m virtualenv -p $(which python$TRAVIS_PYTHON_VERSION) ~/travis-env - source ~/travis-env/bin/activate - fi - - python --version - -install: - - pip install pip --upgrade - - pip install setuptools --upgrade - - pip install -e file://$PWD#egg=ipython[test] --upgrade - - pip install trio curio --upgrade --upgrade-strategy eager - - pip install pytest matplotlib - - pip install codecov check-manifest --upgrade - -script: - - check-manifest - - | - if [[ "$TRAVIS_PYTHON_VERSION" == "nightly" ]]; then - # on nightly fake parso known the grammar - cp /home/travis/virtualenv/python3.8-dev/lib/python3.8/site-packages/parso/python/grammar37.txt /home/travis/virtualenv/python3.8-dev/lib/python3.8/site-packages/parso/python/grammar38.txt - fi - - cd /tmp && iptest --coverage xml && cd - - - pytest IPython - # On the latest Python (on Linux) only, make sure that the docs build. - - | - if [[ "$TRAVIS_PYTHON_VERSION" == "3.7" ]] && [[ "$TRAVIS_OS_NAME" == "linux" ]]; then - pip install -r docs/requirements.txt - python tools/fixup_whats_new_pr.py - make -C docs/ html SPHINXOPTS="-W" - fi - -after_success: - - cp /tmp/ipy_coverage.xml ./ - - cp /tmp/.coverage ./ - - codecov - -matrix: - include: - - arch: amd64 - python: "3.7" - dist: xenial - sudo: true - - arch: arm64 - python: "3.7" - dist: xenial - env: ARM64=True - sudo: true - - arch: amd64 - python: "3.8-dev" - dist: xenial - sudo: true - - arch: amd64 - python: "3.7-dev" - dist: xenial - sudo: true - - arch: amd64 - python: "nightly" - dist: xenial - sudo: true - - arch: arm64 - python: "nightly" - dist: bionic - env: ARM64=True - sudo: true - - os: osx - language: generic - python: 3.6 - env: TRAVIS_PYTHON_VERSION=3.6 - - os: osx - language: generic - python: 3.7 - env: TRAVIS_PYTHON_VERSION=3.7 - allow_failures: - - python: nightly - -before_deploy: - - rm -rf dist/ - - python setup.py sdist - - python setup.py bdist_wheel - -deploy: - provider: releases - api_key: - secure: Y/Ae9tYs5aoBU8bDjN2YrwGG6tCbezj/h3Lcmtx8HQavSbBgXnhnZVRb2snOKD7auqnqjfT/7QMm4ZyKvaOEgyggGktKqEKYHC8KOZ7yp8I5/UMDtk6j9TnXpSqqBxPiud4MDV76SfRYEQiaDoG4tGGvSfPJ9KcNjKrNvSyyxns= - file: dist/* - file_glob: true - skip_cleanup: true - on: - repo: ipython/ipython - all_branches: true # Backports are released from e.g. 5.x branch - tags: true - python: 3.6 # Any version should work, but we only need one - condition: $TRAVIS_OS_NAME = "linux" diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3aecb233319..164757fb350 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -33,7 +33,7 @@ When opening a new Issue, please take the following steps: 1. Search GitHub and/or Google for your issue to avoid duplicate reports. Keyword searches for your error messages are most helpful. -2. If possible, try updating to master and reproducing your issue, +2. If possible, try updating to main and reproducing your issue, because we may have already fixed it. 3. Try to include a minimal reproducible test case. 4. Include relevant system information. Start with the output of: @@ -53,7 +53,7 @@ Some guidelines on contributing to IPython: Review and discussion can begin well before the work is complete, and the more discussion the better. The worst case is that the PR is closed. -* Pull Requests should generally be made against master +* Pull Requests should generally be made against main * Pull Requests should be tested, if feasible: - bugfixes should include regression tests. - new behavior should at least get minimal exercise. @@ -66,30 +66,53 @@ Some guidelines on contributing to IPython: If you're making functional changes, you can clean up the specific pieces of code you're working on. -[Travis](http://travis-ci.org/#!/ipython/ipython) does a pretty good job testing -IPython and Pull Requests, but it may make sense to manually perform tests, +[GitHub Actions](https://github.com/ipython/ipython/actions/workflows/test.yml) does +a pretty good job testing IPython and Pull Requests, +but it may make sense to manually perform tests, particularly for PRs that affect `IPython.parallel` or Windows. For more detailed information, see our [GitHub Workflow](https://github.com/ipython/ipython/wiki/Dev:-GitHub-workflow). ## Running Tests -All the tests can by running +All the tests can be run by using ```shell -iptest +pytest ``` All the tests for a single module (for example **test_alias**) can be run by using the fully qualified path to the module. ```shell -iptest IPython.core.tests.test_alias +pytest IPython/core/tests/test_alias.py ``` -Only a single test (for example **test_alias_lifecycle**) within a single file can be run by adding the specific test after a `:` at the end: +Only a single test (for example **test_alias_lifecycle**) within a single file can be run by adding the specific test after a `::` at the end: ```shell -iptest IPython.core.tests.test_alias:test_alias_lifecycle +pytest IPython/core/tests/test_alias.py::test_alias_lifecycle ``` -For convenience, the full path to a file can often be used instead of the module path on unix systems. For example we can run all the tests by using +## Code style + +* Before committing, run `darker -r 60625f241f298b5039cb2debc365db38aa7bb522 ` to apply selective `black` formatting on modified regions using [darker](https://github.com/akaihola/darker). +* For newly added modules or refactors, please enable static typing analysis with `mypy` for the modified module by adding the file path in [`mypy.yml`](https://github.com/ipython/ipython/blob/main/.github/workflows/mypy.yml) workflow. +* As described in the pull requests section, please avoid excessive formatting changes; if a formatting-only commit is necessary, consider adding its hash to [`.git-blame-ignore-revs`](https://github.com/ipython/ipython/blob/main/.git-blame-ignore-revs) file. + +## Documentation + +Sphinx documentation can be built locally using standard sphinx `make` commands. To build HTML documentation from the root of the project, execute: + +```shell +pip install -r docs/requirements.txt # only needed once +make -C docs/ html SPHINXOPTS="-W" +``` + +To force update of the API documentation, precede the `make` command with: + +```shell +python3 docs/autogen_api.py +``` + +Similarly, to force-update the configuration, run: + ```shell -iptest IPython/core/tests/test_alias.py +python3 docs/autogen_config.py ``` diff --git a/IPython/__init__.py b/IPython/__init__.py index 4fb77107680..7d3799ab363 100644 --- a/IPython/__init__.py +++ b/IPython/__init__.py @@ -1,4 +1,4 @@ -# encoding: utf-8 +# PYTHON_ARGCOMPLETE_OK """ IPython: tools for interactive and parallel computing in Python. @@ -19,7 +19,6 @@ # Imports #----------------------------------------------------------------------------- -import os import sys #----------------------------------------------------------------------------- @@ -27,24 +26,22 @@ #----------------------------------------------------------------------------- # Don't forget to also update setup.py when this changes! -if sys.version_info < (3, 6): +if sys.version_info < (3, 8): raise ImportError( -""" -IPython 7.10+ supports Python 3.6 and above. + """ +IPython 8+ supports Python 3.8 and above, following NEP 29. When using Python 2.7, please install IPython 5.x LTS Long Term Support version. Python 3.3 and 3.4 were supported up to IPython 6.x. Python 3.5 was supported with IPython 7.0 to 7.9. +Python 3.6 was supported with IPython up to 7.16. +Python 3.7 was still supported with the 7.x branch. See IPython `README.rst` file for more information: - https://github.com/ipython/ipython/blob/master/README.rst - -""") + https://github.com/ipython/ipython/blob/main/README.rst -# Make it easy to import extensions - they are always directly on pythonpath. -# Therefore, non-IPython modules can be added to extensions directory. -# This should probably be in ipapp.py. -sys.path.append(os.path.join(os.path.dirname(__file__), "extensions")) +""" + ) #----------------------------------------------------------------------------- # Setup the top level names @@ -56,7 +53,6 @@ from .terminal.embed import embed from .core.interactiveshell import InteractiveShell -from .testing import test from .utils.sysinfo import sys_info from .utils.frame import extract_module_locals @@ -65,23 +61,26 @@ __license__ = release.license __version__ = release.version version_info = release.version_info +# list of CVEs that should have been patched in this release. +# this is informational and should not be relied upon. +__patched_cves__ = {"CVE-2022-21699", "CVE-2023-24816"} + def embed_kernel(module=None, local_ns=None, **kwargs): """Embed and start an IPython kernel in a given scope. - + If you don't want the kernel to initialize the namespace from the scope of the surrounding function, and/or you want to load full IPython configuration, you probably want `IPython.start_kernel()` instead. - + Parameters ---------- module : types.ModuleType, optional The module to load into IPython globals (default: caller) local_ns : dict, optional The namespace to load into IPython user namespace (default: caller) - - kwargs : various, optional + **kwargs : various, optional Further keyword args are relayed to the IPKernelApp constructor, allowing configuration of the Kernel. Will only have an effect on the first embed_kernel call for a given process. @@ -99,26 +98,25 @@ def embed_kernel(module=None, local_ns=None, **kwargs): def start_ipython(argv=None, **kwargs): """Launch a normal IPython instance (as opposed to embedded) - + `IPython.embed()` puts a shell in a particular calling scope, such as a function or method for debugging purposes, which is often not desirable. - + `start_ipython()` does full, regular IPython initialization, including loading startup files, configuration, etc. much of which is skipped by `embed()`. - + This is a public API method, and will survive implementation changes. - + Parameters ---------- - argv : list or None, optional If unspecified or None, IPython will parse command-line options from sys.argv. To prevent any command-line parsing, pass an empty list: `argv=[]`. user_ns : dict, optional specify this dictionary to initialize the IPython user namespace with particular values. - kwargs : various, optional + **kwargs : various, optional Any other kwargs will be passed to the Application constructor, such as `config`. """ @@ -127,26 +125,32 @@ def start_ipython(argv=None, **kwargs): def start_kernel(argv=None, **kwargs): """Launch a normal IPython kernel instance (as opposed to embedded) - + `IPython.embed_kernel()` puts a shell in a particular calling scope, such as a function or method for debugging purposes, which is often not desirable. - + `start_kernel()` does full, regular IPython initialization, including loading startup files, configuration, etc. much of which is skipped by `embed()`. - + Parameters ---------- - argv : list or None, optional If unspecified or None, IPython will parse command-line options from sys.argv. To prevent any command-line parsing, pass an empty list: `argv=[]`. user_ns : dict, optional specify this dictionary to initialize the IPython user namespace with particular values. - kwargs : various, optional + **kwargs : various, optional Any other kwargs will be passed to the Application constructor, such as `config`. """ - from IPython.kernel.zmq.kernelapp import launch_new_instance + import warnings + + warnings.warn( + "start_kernel is deprecated since IPython 8.0, use from `ipykernel.kernelapp.launch_new_instance`", + DeprecationWarning, + stacklevel=2, + ) + from ipykernel.kernelapp import launch_new_instance return launch_new_instance(argv=argv, **kwargs) diff --git a/IPython/__main__.py b/IPython/__main__.py index d5123f33a20..8e9f989a82c 100644 --- a/IPython/__main__.py +++ b/IPython/__main__.py @@ -1,3 +1,4 @@ +# PYTHON_ARGCOMPLETE_OK # encoding: utf-8 """Terminal-based IPython entry point. """ diff --git a/IPython/config.py b/IPython/config.py deleted file mode 100644 index 964f46f10ac..00000000000 --- a/IPython/config.py +++ /dev/null @@ -1,19 +0,0 @@ -""" -Shim to maintain backwards compatibility with old IPython.config imports. -""" -# Copyright (c) IPython Development Team. -# Distributed under the terms of the Modified BSD License. - -import sys -from warnings import warn - -from .utils.shimmodule import ShimModule, ShimWarning - -warn("The `IPython.config` package has been deprecated since IPython 4.0. " - "You should import from traitlets.config instead.", ShimWarning) - - -# Unconditionally insert the shim into sys.modules so that further import calls -# trigger the custom attribute access above - -sys.modules['IPython.config'] = ShimModule(src='IPython.config', mirror='traitlets.config') diff --git a/IPython/conftest.py b/IPython/conftest.py index 8b2af8c020a..abf61314798 100644 --- a/IPython/conftest.py +++ b/IPython/conftest.py @@ -1,14 +1,32 @@ -import types -import sys import builtins +import inspect import os -import pytest import pathlib import shutil +import sys +import types + +import pytest + +# Must register before it gets imported +pytest.register_assert_rewrite("IPython.testing.tools") from .testing import tools +def pytest_collection_modifyitems(items): + """This function is automatically run by pytest passing all collected test + functions. + + We use it to add asyncio marker to all async tests and assert we don't use + test functions that are async generators which wouldn't make sense. + """ + for item in items: + if inspect.iscoroutinefunction(item.obj): + item.add_marker("asyncio") + assert not inspect.isasyncgenfunction(item.obj) + + def get_ipython(): from .terminal.interactiveshell import TerminalInteractiveShell if TerminalInteractiveShell._instance: @@ -29,7 +47,7 @@ def work_path(): if path.exists(): raise ValueError('IPython dir temporary path already exists ! Did previous test run exit successfully ?') path.mkdir() - yield + yield shutil.rmtree(str(path.resolve())) diff --git a/IPython/core/application.py b/IPython/core/application.py index 93639d88e2c..e0a8174f153 100644 --- a/IPython/core/application.py +++ b/IPython/core/application.py @@ -14,12 +14,13 @@ import atexit from copy import deepcopy -import glob import logging import os import shutil import sys +from pathlib import Path + from traitlets.config.application import Application, catch_config_error from traitlets.config.loader import ConfigFileNotFound, PyFileConfigLoader from IPython.core import release, crashhandler @@ -31,10 +32,10 @@ default, observe, ) -if os.name == 'nt': - programdata = os.environ.get('PROGRAMDATA', None) - if programdata: - SYSTEM_CONFIG_DIRS = [os.path.join(programdata, 'ipython')] +if os.name == "nt": + programdata = os.environ.get("PROGRAMDATA", None) + if programdata is not None: + SYSTEM_CONFIG_DIRS = [str(Path(programdata) / "ipython")] else: # PROGRAMDATA is not defined by default on XP. SYSTEM_CONFIG_DIRS = [] else: @@ -64,27 +65,49 @@ # aliases and flags -base_aliases = { - 'profile-dir' : 'ProfileDir.location', - 'profile' : 'BaseIPythonApplication.profile', - 'ipython-dir' : 'BaseIPythonApplication.ipython_dir', - 'log-level' : 'Application.log_level', - 'config' : 'BaseIPythonApplication.extra_config_file', -} - -base_flags = dict( - debug = ({'Application' : {'log_level' : logging.DEBUG}}, - "set log level to logging.DEBUG (maximize logging output)"), - quiet = ({'Application' : {'log_level' : logging.CRITICAL}}, - "set log level to logging.CRITICAL (minimize logging output)"), - init = ({'BaseIPythonApplication' : { - 'copy_config_files' : True, - 'auto_create' : True} - }, """Initialize profile with default config files. This is equivalent +base_aliases = {} +if isinstance(Application.aliases, dict): + # traitlets 5 + base_aliases.update(Application.aliases) +base_aliases.update( + { + "profile-dir": "ProfileDir.location", + "profile": "BaseIPythonApplication.profile", + "ipython-dir": "BaseIPythonApplication.ipython_dir", + "log-level": "Application.log_level", + "config": "BaseIPythonApplication.extra_config_file", + } +) + +base_flags = dict() +if isinstance(Application.flags, dict): + # traitlets 5 + base_flags.update(Application.flags) +base_flags.update( + dict( + debug=( + {"Application": {"log_level": logging.DEBUG}}, + "set log level to logging.DEBUG (maximize logging output)", + ), + quiet=( + {"Application": {"log_level": logging.CRITICAL}}, + "set log level to logging.CRITICAL (minimize logging output)", + ), + init=( + { + "BaseIPythonApplication": { + "copy_config_files": True, + "auto_create": True, + } + }, + """Initialize profile with default config files. This is equivalent to running `ipython profile create ` prior to startup. - """) + """, + ), + ) ) + class ProfileAwareConfigLoader(PyFileConfigLoader): """A Python file config loader that is aware of IPython profiles.""" def load_subconfig(self, fname, path=None, profile=None): @@ -100,9 +123,8 @@ def load_subconfig(self, fname, path=None, profile=None): return super(ProfileAwareConfigLoader, self).load_subconfig(fname, path=path) class BaseIPythonApplication(Application): - - name = u'ipython' - description = Unicode(u'IPython: an enhanced interactive Python shell.') + name = "ipython" + description = "IPython: an enhanced interactive Python shell." version = Unicode(release.version) aliases = base_aliases @@ -133,7 +155,7 @@ def _config_file_name_changed(self, change): config_file_paths = List(Unicode()) @default('config_file_paths') def _config_file_paths_default(self): - return [os.getcwd()] + return [] extra_config_file = Unicode( help="""Path to an extra config file to load. @@ -161,6 +183,17 @@ def _profile_changed(self, change): get_ipython_package_dir(), u'config', u'profile', change['new'] ) + add_ipython_dir_to_sys_path = Bool( + False, + """Should the IPython profile directory be added to sys path ? + + This option was non-existing before IPython 8.0, and ipython_dir was added to + sys path to allow import of extensions present there. This was historical + baggage from when pip did not exist. This now default to false, + but can be set to true for legacy reasons. + """, + ).tag(config=True) + ipython_dir = Unicode( help=""" The name of the IPython directory. This directory is used for logging @@ -232,16 +265,6 @@ def __init__(self, **kwargs): # Various stages of Application creation #------------------------------------------------------------------------- - deprecated_subcommands = {} - - def initialize_subcommand(self, subc, argv=None): - if subc in self.deprecated_subcommands: - self.log.warning("Subcommand `ipython {sub}` is deprecated and will be removed " - "in future versions.".format(sub=subc)) - self.log.warning("You likely want to use `jupyter {sub}` in the " - "future".format(sub=subc)) - return super(BaseIPythonApplication, self).initialize_subcommand(subc, argv) - def init_crash_handler(self): """Create a crash handler, typically setting sys.excepthook to it.""" self.crash_handler = self.crash_handler_class(self) @@ -252,7 +275,7 @@ def unset_crashhandler(): def excepthook(self, etype, evalue, tb): """this is sys.excepthook after init_crashhandler - + set self.verbose_crash=True to use our full crashhandler, instead of a regular traceback with a short message (crash_handler_lite) """ @@ -270,21 +293,24 @@ def _ipython_dir_changed(self, change): str_old = os.path.abspath(old) if str_old in sys.path: sys.path.remove(str_old) - str_path = os.path.abspath(new) - sys.path.append(str_path) - ensure_dir_exists(new) - readme = os.path.join(new, 'README') - readme_src = os.path.join(get_ipython_package_dir(), u'config', u'profile', 'README') - if not os.path.exists(readme) and os.path.exists(readme_src): - shutil.copy(readme_src, readme) - for d in ('extensions', 'nbextensions'): - path = os.path.join(new, d) - try: - ensure_dir_exists(path) - except OSError as e: - # this will not be EEXIST - self.log.error("couldn't create path %s: %s", path, e) - self.log.debug("IPYTHONDIR set to: %s" % new) + if self.add_ipython_dir_to_sys_path: + str_path = os.path.abspath(new) + sys.path.append(str_path) + ensure_dir_exists(new) + readme = os.path.join(new, "README") + readme_src = os.path.join( + get_ipython_package_dir(), "config", "profile", "README" + ) + if not os.path.exists(readme) and os.path.exists(readme_src): + shutil.copy(readme_src, readme) + for d in ("extensions", "nbextensions"): + path = os.path.join(new, d) + try: + ensure_dir_exists(path) + except OSError as e: + # this will not be EEXIST + self.log.error("couldn't create path %s: %s", path, e) + self.log.debug("IPYTHONDIR set to: %s", new) def load_config_file(self, suppress_errors=IPYTHON_SUPPRESS_CONFIG_ERRORS): """Load the config file. @@ -374,7 +400,7 @@ def init_profile_dir(self): self.log.fatal("Profile %r not found."%self.profile) self.exit(1) else: - self.log.debug("Using existing profile dir: %r"%p.location) + self.log.debug("Using existing profile dir: %r", p.location) else: location = self.config.ProfileDir.location # location is fully specified @@ -394,7 +420,7 @@ def init_profile_dir(self): self.log.fatal("Profile directory %r not found."%location) self.exit(1) else: - self.log.info("Using existing profile dir: %r"%location) + self.log.debug("Using existing profile dir: %r", p.location) # if profile_dir is specified explicitly, set profile name dir_name = os.path.basename(p.location) if dir_name.startswith('profile_'): @@ -409,14 +435,15 @@ def init_config_files(self): self.config_file_paths.extend(ENV_CONFIG_DIRS) self.config_file_paths.extend(SYSTEM_CONFIG_DIRS) # copy config files - path = self.builtin_profile_dir + path = Path(self.builtin_profile_dir) if self.copy_config_files: src = self.profile cfg = self.config_file_name - if path and os.path.exists(os.path.join(path, cfg)): - self.log.warning("Staging %r from %s into %r [overwrite=%s]"%( - cfg, src, self.profile_dir.location, self.overwrite) + if path and (path / cfg).exists(): + self.log.warning( + "Staging %r from %s into %r [overwrite=%s]" + % (cfg, src, self.profile_dir.location, self.overwrite) ) self.profile_dir.copy_config_file(cfg, path=path, overwrite=self.overwrite) else: @@ -425,9 +452,9 @@ def init_config_files(self): # Still stage *bundled* config files, but not generated ones # This is necessary for `ipython profile=sympy` to load the profile # on the first go - files = glob.glob(os.path.join(path, '*.py')) + files = path.glob("*.py") for fullpath in files: - cfg = os.path.basename(fullpath) + cfg = fullpath.name if self.profile_dir.copy_config_file(cfg, path=path, overwrite=False): # file was copied self.log.warning("Staging bundled %s from %s into %r"%( @@ -438,11 +465,10 @@ def init_config_files(self): def stage_default_config_file(self): """auto generate default config file, and stage it into the profile.""" s = self.generate_config_file() - fname = os.path.join(self.profile_dir.location, self.config_file_name) - if self.overwrite or not os.path.exists(fname): - self.log.warning("Generating default config file: %r"%(fname)) - with open(fname, 'w') as f: - f.write(s) + config_file = Path(self.profile_dir.location) / self.config_file_name + if self.overwrite or not config_file.exists(): + self.log.warning("Generating default config file: %r", (config_file)) + config_file.write_text(s, encoding="utf-8") @catch_config_error def initialize(self, argv=None): diff --git a/IPython/core/async_helpers.py b/IPython/core/async_helpers.py index fb4cc193250..0e7db0bb54d 100644 --- a/IPython/core/async_helpers.py +++ b/IPython/core/async_helpers.py @@ -12,27 +12,89 @@ import ast -import sys +import asyncio import inspect -from textwrap import dedent, indent +from functools import wraps +_asyncio_event_loop = None -class _AsyncIORunner: +def get_asyncio_loop(): + """asyncio has deprecated get_event_loop + + Replicate it here, with our desired semantics: + + - always returns a valid, not-closed loop + - not thread-local like asyncio's, + because we only want one loop for IPython + - if called from inside a coroutine (e.g. in ipykernel), + return the running loop + + .. versionadded:: 8.0 + """ + try: + return asyncio.get_running_loop() + except RuntimeError: + # not inside a coroutine, + # track our own global + pass + + # not thread-local like asyncio's, + # because we only track one event loop to run for IPython itself, + # always in the main thread. + global _asyncio_event_loop + if _asyncio_event_loop is None or _asyncio_event_loop.is_closed(): + _asyncio_event_loop = asyncio.new_event_loop() + return _asyncio_event_loop + + +class _AsyncIORunner: def __call__(self, coro): """ Handler for asyncio autoawait """ - import asyncio - - return asyncio.get_event_loop().run_until_complete(coro) + return get_asyncio_loop().run_until_complete(coro) def __str__(self): - return 'asyncio' + return "asyncio" + _asyncio_runner = _AsyncIORunner() +class _AsyncIOProxy: + """Proxy-object for an asyncio + + Any coroutine methods will be wrapped in event_loop.run_ + """ + + def __init__(self, obj, event_loop): + self._obj = obj + self._event_loop = event_loop + + def __repr__(self): + return f"<_AsyncIOProxy({self._obj!r})>" + + def __getattr__(self, key): + attr = getattr(self._obj, key) + if inspect.iscoroutinefunction(attr): + # if it's a coroutine method, + # return a threadsafe wrapper onto the _current_ asyncio loop + @wraps(attr) + def _wrapped(*args, **kwargs): + concurrent_future = asyncio.run_coroutine_threadsafe( + attr(*args, **kwargs), self._event_loop + ) + return asyncio.wrap_future(concurrent_future) + + return _wrapped + else: + return attr + + def __dir__(self): + return dir(self._obj) + + def _curio_runner(coroutine): """ handler for curio autoawait @@ -62,7 +124,6 @@ def _pseudo_sync_runner(coro): See discussion in https://github.com/python-trio/trio/issues/608, Credit to Nathaniel Smith - """ try: coro.send(None) @@ -75,69 +136,6 @@ def _pseudo_sync_runner(coro): ) -def _asyncify(code: str) -> str: - """wrap code in async def definition. - - And setup a bit of context to run it later. - """ - res = dedent( - """ - async def __wrapper__(): - try: - {usercode} - finally: - locals() - """ - ).format(usercode=indent(code, " " * 8)) - return res - - -class _AsyncSyntaxErrorVisitor(ast.NodeVisitor): - """ - Find syntax errors that would be an error in an async repl, but because - the implementation involves wrapping the repl in an async function, it - is erroneously allowed (e.g. yield or return at the top level) - """ - def __init__(self): - if sys.version_info >= (3,8): - raise ValueError('DEPRECATED in Python 3.8+') - self.depth = 0 - super().__init__() - - def generic_visit(self, node): - func_types = (ast.FunctionDef, ast.AsyncFunctionDef) - invalid_types_by_depth = { - 0: (ast.Return, ast.Yield, ast.YieldFrom), - 1: (ast.Nonlocal,) - } - - should_traverse = self.depth < max(invalid_types_by_depth.keys()) - if isinstance(node, func_types) and should_traverse: - self.depth += 1 - super().generic_visit(node) - self.depth -= 1 - elif isinstance(node, invalid_types_by_depth[self.depth]): - raise SyntaxError() - else: - super().generic_visit(node) - - -def _async_parse_cell(cell: str) -> ast.AST: - """ - This is a compatibility shim for pre-3.7 when async outside of a function - is a syntax error at the parse stage. - - It will return an abstract syntax tree parsed as if async and await outside - of a function were not a syntax error. - """ - if sys.version_info < (3, 7): - # Prior to 3.7 you need to asyncify before parse - wrapped_parse_tree = ast.parse(_asyncify(cell)) - return wrapped_parse_tree.body[0].body[0] - else: - return ast.parse(cell) - - def _should_be_async(cell: str) -> bool: """Detect if a block of code need to be wrapped in an `async def` @@ -149,25 +147,10 @@ def _should_be_async(cell: str) -> bool: Not handled yet: If the block of code has a return statement as the top level, it will be seen as async. This is a know limitation. """ - if sys.version_info > (3, 8): - try: - code = compile(cell, "<>", "exec", flags=getattr(ast,'PyCF_ALLOW_TOP_LEVEL_AWAIT', 0x0)) - return inspect.CO_COROUTINE & code.co_flags == inspect.CO_COROUTINE - except (SyntaxError, MemoryError): - return False try: - # we can't limit ourself to ast.parse, as it __accepts__ to parse on - # 3.7+, but just does not _compile_ - code = compile(cell, "<>", "exec") + code = compile( + cell, "<>", "exec", flags=getattr(ast, "PyCF_ALLOW_TOP_LEVEL_AWAIT", 0x0) + ) + return inspect.CO_COROUTINE & code.co_flags == inspect.CO_COROUTINE except (SyntaxError, MemoryError): - try: - parse_tree = _async_parse_cell(cell) - - # Raise a SyntaxError if there are top-level return or yields - v = _AsyncSyntaxErrorVisitor() - v.visit(parse_tree) - - except (SyntaxError, MemoryError): - return False - return True - return False + return False diff --git a/IPython/core/autocall.py b/IPython/core/autocall.py index bab7f859c96..54beec3f58d 100644 --- a/IPython/core/autocall.py +++ b/IPython/core/autocall.py @@ -40,10 +40,10 @@ def __init__(self, ip=None): self._ip = ip def set_ip(self, ip): - """ Will be used to set _ip point to current ipython instance b/f call - + """Will be used to set _ip point to current ipython instance b/f call + Override this method if you don't want this to happen. - + """ self._ip = ip diff --git a/IPython/core/compilerop.py b/IPython/core/compilerop.py index c4771af7303..7799a4fc99e 100644 --- a/IPython/core/compilerop.py +++ b/IPython/core/compilerop.py @@ -73,25 +73,10 @@ class CachingCompiler(codeop.Compile): def __init__(self): codeop.Compile.__init__(self) - # This is ugly, but it must be done this way to allow multiple - # simultaneous ipython instances to coexist. Since Python itself - # directly accesses the data structures in the linecache module, and - # the cache therein is global, we must work with that data structure. - # We must hold a reference to the original checkcache routine and call - # that in our own check_cache() below, but the special IPython cache - # must also be shared by all IPython instances. If we were to hold - # separate caches (one in each CachingCompiler instance), any call made - # by Python itself to linecache.checkcache() would obliterate the - # cached data from the other IPython instances. - if not hasattr(linecache, '_ipython_cache'): - linecache._ipython_cache = {} - if not hasattr(linecache, '_checkcache_ori'): - linecache._checkcache_ori = linecache.checkcache - # Now, we must monkeypatch the linecache directly so that parts of the - # stdlib that call it outside our control go through our codepath - # (otherwise we'd lose our tracebacks). - linecache.checkcache = check_linecache_ipython - + # Caching a dictionary { filename: execution_count } for nicely + # rendered tracebacks. The filename corresponds to the filename + # argument used for the builtins.compile function. + self._filename_map = {} def ast_parse(self, source, filename='', symbol='exec'): """Parse code to an AST with the current compiler flags active. @@ -99,7 +84,7 @@ def ast_parse(self, source, filename='', symbol='exec'): Arguments are exactly the same as ast.parse (in the standard library), and are passed to the built-in compile function.""" return compile(source, filename, symbol, self.flags | PyCF_ONLY_AST, 1) - + def reset_compiler_flags(self): """Reset compiler flags to default state.""" # This value is copied from codeop.Compile.__init__, so if that ever @@ -112,27 +97,84 @@ def compiler_flags(self): """ return self.flags - def cache(self, code, number=0): + def get_code_name(self, raw_code, transformed_code, number): + """Compute filename given the code, and the cell number. + + Parameters + ---------- + raw_code : str + The raw cell code. + transformed_code : str + The executable Python source code to cache and compile. + number : int + A number which forms part of the code's name. Used for the execution + counter. + + Returns + ------- + The computed filename. + """ + return code_name(transformed_code, number) + + def format_code_name(self, name): + """Return a user-friendly label and name for a code block. + + Parameters + ---------- + name : str + The name for the code block returned from get_code_name + + Returns + ------- + A (label, name) pair that can be used in tracebacks, or None if the default formatting should be used. + """ + if name in self._filename_map: + return "Cell", "In[%s]" % self._filename_map[name] + + def cache(self, transformed_code, number=0, raw_code=None): """Make a name for a block of code, and cache the code. Parameters ---------- - code : str - The Python source code to cache. + transformed_code : str + The executable Python source code to cache and compile. number : int - A number which forms part of the code's name. Used for the execution - counter. + A number which forms part of the code's name. Used for the execution + counter. + raw_code : str + The raw code before transformation, if None, set to `transformed_code`. Returns ------- The name of the cached code (as a string). Pass this as the filename argument to compilation, so that tracebacks are correctly hooked up. """ - name = code_name(code, number) - entry = (len(code), time.time(), - [line+'\n' for line in code.splitlines()], name) + if raw_code is None: + raw_code = transformed_code + + name = self.get_code_name(raw_code, transformed_code, number) + + # Save the execution count + self._filename_map[name] = number + + # Since Python 2.5, setting mtime to `None` means the lines will + # never be removed by `linecache.checkcache`. This means all the + # monkeypatching has *never* been necessary, since this code was + # only added in 2010, at which point IPython had already stopped + # supporting Python 2.4. + # + # Note that `linecache.clearcache` and `linecache.updatecache` may + # still remove our code from the cache, but those show explicit + # intent, and we should not try to interfere. Normally the former + # is never called except when out of memory, and the latter is only + # called for lines *not* in the cache. + entry = ( + len(transformed_code), + None, + [line + "\n" for line in transformed_code.splitlines()], + name, + ) linecache.cache[name] = entry - linecache._ipython_cache[name] = entry return name @contextmanager @@ -146,15 +188,27 @@ def extra_flags(self, flags): yield finally: # turn off only the bits we turned on so that something like - # __future__ that set flags stays. + # __future__ that set flags stays. self.flags &= ~turn_on_bits def check_linecache_ipython(*args): - """Call linecache.checkcache() safely protecting our cached values. + """Deprecated since IPython 8.6. Call linecache.checkcache() directly. + + It was already not necessary to call this function directly. If no + CachingCompiler had been created, this function would fail badly. If + an instance had been created, this function would've been monkeypatched + into place. + + As of IPython 8.6, the monkeypatching has gone away entirely. But there + were still internal callers of this function, so maybe external callers + also existed? """ - # First call the original checkcache as intended - linecache._checkcache_ori(*args) - # Then, update back the cache with our data, so that tracebacks related - # to our compiled codes can be produced. - linecache.cache.update(linecache._ipython_cache) + import warnings + + warnings.warn( + "Deprecated Since IPython 8.6, Just call linecache.checkcache() directly.", + DeprecationWarning, + stacklevel=2, + ) + linecache.checkcache() diff --git a/IPython/core/completer.py b/IPython/core/completer.py index 0942798f3b3..f0bbb4e5619 100644 --- a/IPython/core/completer.py +++ b/IPython/core/completer.py @@ -35,13 +35,13 @@ .. code:: - \\greek small letter alpha + \\GREEK SMALL LETTER ALPHA α Only valid Python identifiers will complete. Combining characters (like arrow or dots) are also available, unlike latex they need to be put after the their -counterpart that is to say, `F\\\\vec` is correct, not `\\\\vecF`. +counterpart that is to say, ``F\\\\vec`` is correct, not ``\\\\vecF``. Some browsers are known to display combining characters incorrectly. @@ -50,7 +50,7 @@ It is sometime challenging to know how to type a character, if you are using IPython, or any compatible frontend you can prepend backslash to the character -and press `` to expand it to its latex form. +and press :kbd:`Tab` to expand it to its latex form. .. code:: @@ -59,7 +59,8 @@ Both forward and backward completions can be deactivated by setting the -``Completer.backslash_combining_completions`` option to ``False``. +:std:configtrait:`Completer.backslash_combining_completions` option to +``False``. Experimental @@ -95,11 +96,78 @@ ... myvar[1].bi Tab completion will be able to infer that ``myvar[1]`` is a real number without -executing any code unlike the previously available ``IPCompleter.greedy`` +executing almost any code unlike the deprecated :any:`IPCompleter.greedy` option. Be sure to update :any:`jedi` to the latest stable version or to try the current development version to get better completions. + +Matchers +======== + +All completions routines are implemented using unified *Matchers* API. +The matchers API is provisional and subject to change without notice. + +The built-in matchers include: + +- :any:`IPCompleter.dict_key_matcher`: dictionary key completions, +- :any:`IPCompleter.magic_matcher`: completions for magics, +- :any:`IPCompleter.unicode_name_matcher`, + :any:`IPCompleter.fwd_unicode_matcher` + and :any:`IPCompleter.latex_name_matcher`: see `Forward latex/unicode completion`_, +- :any:`back_unicode_name_matcher` and :any:`back_latex_name_matcher`: see `Backward latex completion`_, +- :any:`IPCompleter.file_matcher`: paths to files and directories, +- :any:`IPCompleter.python_func_kw_matcher` - function keywords, +- :any:`IPCompleter.python_matches` - globals and attributes (v1 API), +- ``IPCompleter.jedi_matcher`` - static analysis with Jedi, +- :any:`IPCompleter.custom_completer_matcher` - pluggable completer with a default + implementation in :any:`InteractiveShell` which uses IPython hooks system + (`complete_command`) with string dispatch (including regular expressions). + Differently to other matchers, ``custom_completer_matcher`` will not suppress + Jedi results to match behaviour in earlier IPython versions. + +Custom matchers can be added by appending to ``IPCompleter.custom_matchers`` list. + +Matcher API +----------- + +Simplifying some details, the ``Matcher`` interface can described as + +.. code-block:: + + MatcherAPIv1 = Callable[[str], list[str]] + MatcherAPIv2 = Callable[[CompletionContext], SimpleMatcherResult] + + Matcher = MatcherAPIv1 | MatcherAPIv2 + +The ``MatcherAPIv1`` reflects the matcher API as available prior to IPython 8.6.0 +and remains supported as a simplest way for generating completions. This is also +currently the only API supported by the IPython hooks system `complete_command`. + +To distinguish between matcher versions ``matcher_api_version`` attribute is used. +More precisely, the API allows to omit ``matcher_api_version`` for v1 Matchers, +and requires a literal ``2`` for v2 Matchers. + +Once the API stabilises future versions may relax the requirement for specifying +``matcher_api_version`` by switching to :any:`functools.singledispatch`, therefore +please do not rely on the presence of ``matcher_api_version`` for any purposes. + +Suppression of competing matchers +--------------------------------- + +By default results from all matchers are combined, in the order determined by +their priority. Matchers can request to suppress results from subsequent +matchers by setting ``suppress`` to ``True`` in the ``MatcherResult``. + +When multiple matchers simultaneously request surpression, the results from of +the matcher with higher priority will be returned. + +Sometimes it is desirable to suppress most but not all other matchers; +this can be achieved by adding a list of identifiers of matchers which +should not be suppressed to ``MatcherResult`` under ``do_not_suppress`` key. + +The suppression behaviour can is user-configurable via +:std:configtrait:`IPCompleter.suppress_competing_matchers`. """ @@ -109,38 +177,74 @@ # Some of this code originated from rlcompleter in the Python standard library # Copyright (C) 2001 Python Software Foundation, www.python.org - -import __main__ +from __future__ import annotations import builtins as builtin_mod +import enum import glob -import time import inspect import itertools import keyword import os import re +import string import sys +import tokenize +import time import unicodedata -import string +import uuid import warnings - +from ast import literal_eval +from collections import defaultdict from contextlib import contextmanager -from importlib import import_module -from typing import Iterator, List, Tuple, Iterable +from dataclasses import dataclass +from functools import cached_property, partial from types import SimpleNamespace - -from traitlets.config.configurable import Configurable +from typing import ( + Iterable, + Iterator, + List, + Tuple, + Union, + Any, + Sequence, + Dict, + Optional, + TYPE_CHECKING, + Set, + Sized, + TypeVar, + Literal, +) + +from IPython.core.guarded_eval import guarded_eval, EvaluationContext from IPython.core.error import TryNext from IPython.core.inputtransformer2 import ESC_MAGIC from IPython.core.latex_symbols import latex_symbols, reverse_latex_symbol from IPython.core.oinspect import InspectColors +from IPython.testing.skipdoctest import skip_doctest from IPython.utils import generics +from IPython.utils.decorators import sphinx_options from IPython.utils.dir2 import dir2, get_real_method +from IPython.utils.docs import GENERATING_DOCUMENTATION +from IPython.utils.path import ensure_dir_exists from IPython.utils.process import arg_split -from traitlets import Bool, Enum, observe, Int +from traitlets import ( + Bool, + Enum, + Int, + List as ListTrait, + Unicode, + Dict as DictTrait, + Union as UnionTrait, + observe, +) +from traitlets.config.configurable import Configurable + +import __main__ # skip module docstests -skip_doctest = True +__skip_doctest__ = True + try: import jedi @@ -150,12 +254,41 @@ JEDI_INSTALLED = True except ImportError: JEDI_INSTALLED = False -#----------------------------------------------------------------------------- + + +if TYPE_CHECKING or GENERATING_DOCUMENTATION and sys.version_info >= (3, 11): + from typing import cast + from typing_extensions import TypedDict, NotRequired, Protocol, TypeAlias, TypeGuard +else: + from typing import Generic + + def cast(type_, obj): + """Workaround for `TypeError: MatcherAPIv2() takes no arguments`""" + return obj + + # do not require on runtime + NotRequired = Tuple # requires Python >=3.11 + TypedDict = Dict # by extension of `NotRequired` requires 3.11 too + Protocol = object # requires Python >=3.8 + TypeAlias = Any # requires Python >=3.10 + TypeGuard = Generic # requires Python >=3.10 +if GENERATING_DOCUMENTATION: + from typing import TypedDict + +# ----------------------------------------------------------------------------- # Globals #----------------------------------------------------------------------------- +# ranges where we have most of the valid unicode names. We could be more finer +# grained but is it worth it for performance While unicode have character in the +# range 0, 0x110000, we seem to have name for about 10% of those. (131808 as I +# write this). With below range we cover them all, with a density of ~67% +# biggest next gap we consider only adds up about 1% density and there are 600 +# gaps that would need hard coding. +_UNICODE_RANGES = [(32, 0x323B0), (0xE0001, 0xE01F0)] + # Public API -__all__ = ['Completer','IPCompleter'] +__all__ = ["Completer", "IPCompleter"] if sys.platform == 'win32': PROTECTABLES = ' ' @@ -166,8 +299,11 @@ # may have trouble processing. MATCHES_LIMIT = 500 -_deprecation_readline_sentinel = object() +# Completion type reported when no type can be inferred. +_UNKNOWN_TYPE = "" +# sentinel value to signal lack of a match +not_found = object() class ProvisionalCompleterWarning(FutureWarning): """ @@ -180,11 +316,11 @@ class ProvisionalCompleterWarning(FutureWarning): warnings.filterwarnings('error', category=ProvisionalCompleterWarning) + +@skip_doctest @contextmanager def provisionalcompleter(action='ignore'): """ - - This context manager has to be used in any place where unstable completer behavior and API may be called. @@ -193,7 +329,9 @@ def provisionalcompleter(action='ignore'): >>> completer.do_experimental_things() # raises. - .. note:: Unstable + .. note:: + + Unstable By using this context manager you agree that the API in use may change without warning, and that you won't complain if they do so. @@ -252,17 +390,17 @@ def expand_user(path:str) -> Tuple[str, bool, str]: Parameters ---------- path : str - String to be expanded. If no ~ is present, the output is the same as the - input. + String to be expanded. If no ~ is present, the output is the same as the + input. Returns ------- newpath : str - Result of ~ expansion in the input path. + Result of ~ expansion in the input path. tilde_expand : bool - Whether any expansion was performed or not. + Whether any expansion was performed or not. tilde_val : str - The value that ~ was replaced with. + The value that ~ was replaced with. """ # Default values tilde_expand = False @@ -337,18 +475,24 @@ def __init__(self, name): self.complete = name self.type = 'crashed' self.name_with_symbols = name - self.signature = '' - self._origin = 'fake' + self.signature = "" + self._origin = "fake" + self.text = "crashed" def __repr__(self): return '' +_JediCompletionLike = Union[jedi.api.Completion, _FakeJediCompletion] + + class Completion: """ - Completion object used and return by IPython completers. + Completion object used and returned by IPython completers. - .. warning:: Unstable + .. warning:: + + Unstable This function is unstable, API may change without warning. It will also raise unless use in proper context manager. @@ -369,11 +513,23 @@ class Completion: __slots__ = ['start', 'end', 'text', 'type', 'signature', '_origin'] - def __init__(self, start: int, end: int, text: str, *, type: str=None, _origin='', signature='') -> None: - warnings.warn("``Completion`` is a provisional API (as of IPython 6.0). " - "It may change without warnings. " - "Use in corresponding context manager.", - category=ProvisionalCompleterWarning, stacklevel=2) + def __init__( + self, + start: int, + end: int, + text: str, + *, + type: Optional[str] = None, + _origin="", + signature="", + ) -> None: + warnings.warn( + "``Completion`` is a provisional API (as of IPython 6.0). " + "It may change without warnings. " + "Use in corresponding context manager.", + category=ProvisionalCompleterWarning, + stacklevel=2, + ) self.start = start self.end = end @@ -386,7 +542,7 @@ def __repr__(self): return '' % \ (self.start, self.end, self.text, self.type or '?', self.signature or '?') - def __eq__(self, other)->Bool: + def __eq__(self, other) -> bool: """ Equality and hash do not hash the type (as some completer may not be able to infer the type), but are use to (partially) de-duplicate @@ -404,6 +560,248 @@ def __hash__(self): return hash((self.start, self.end, self.text)) +class SimpleCompletion: + """Completion item to be included in the dictionary returned by new-style Matcher (API v2). + + .. warning:: + + Provisional + + This class is used to describe the currently supported attributes of + simple completion items, and any additional implementation details + should not be relied on. Additional attributes may be included in + future versions, and meaning of text disambiguated from the current + dual meaning of "text to insert" and "text to used as a label". + """ + + __slots__ = ["text", "type"] + + def __init__(self, text: str, *, type: Optional[str] = None): + self.text = text + self.type = type + + def __repr__(self): + return f"" + + +class _MatcherResultBase(TypedDict): + """Definition of dictionary to be returned by new-style Matcher (API v2).""" + + #: Suffix of the provided ``CompletionContext.token``, if not given defaults to full token. + matched_fragment: NotRequired[str] + + #: Whether to suppress results from all other matchers (True), some + #: matchers (set of identifiers) or none (False); default is False. + suppress: NotRequired[Union[bool, Set[str]]] + + #: Identifiers of matchers which should NOT be suppressed when this matcher + #: requests to suppress all other matchers; defaults to an empty set. + do_not_suppress: NotRequired[Set[str]] + + #: Are completions already ordered and should be left as-is? default is False. + ordered: NotRequired[bool] + + +@sphinx_options(show_inherited_members=True, exclude_inherited_from=["dict"]) +class SimpleMatcherResult(_MatcherResultBase, TypedDict): + """Result of new-style completion matcher.""" + + # note: TypedDict is added again to the inheritance chain + # in order to get __orig_bases__ for documentation + + #: List of candidate completions + completions: Sequence[SimpleCompletion] | Iterator[SimpleCompletion] + + +class _JediMatcherResult(_MatcherResultBase): + """Matching result returned by Jedi (will be processed differently)""" + + #: list of candidate completions + completions: Iterator[_JediCompletionLike] + + +AnyMatcherCompletion = Union[_JediCompletionLike, SimpleCompletion] +AnyCompletion = TypeVar("AnyCompletion", AnyMatcherCompletion, Completion) + + +@dataclass +class CompletionContext: + """Completion context provided as an argument to matchers in the Matcher API v2.""" + + # rationale: many legacy matchers relied on completer state (`self.text_until_cursor`) + # which was not explicitly visible as an argument of the matcher, making any refactor + # prone to errors; by explicitly passing `cursor_position` we can decouple the matchers + # from the completer, and make substituting them in sub-classes easier. + + #: Relevant fragment of code directly preceding the cursor. + #: The extraction of token is implemented via splitter heuristic + #: (following readline behaviour for legacy reasons), which is user configurable + #: (by switching the greedy mode). + token: str + + #: The full available content of the editor or buffer + full_text: str + + #: Cursor position in the line (the same for ``full_text`` and ``text``). + cursor_position: int + + #: Cursor line in ``full_text``. + cursor_line: int + + #: The maximum number of completions that will be used downstream. + #: Matchers can use this information to abort early. + #: The built-in Jedi matcher is currently excepted from this limit. + # If not given, return all possible completions. + limit: Optional[int] + + @cached_property + def text_until_cursor(self) -> str: + return self.line_with_cursor[: self.cursor_position] + + @cached_property + def line_with_cursor(self) -> str: + return self.full_text.split("\n")[self.cursor_line] + + +#: Matcher results for API v2. +MatcherResult = Union[SimpleMatcherResult, _JediMatcherResult] + + +class _MatcherAPIv1Base(Protocol): + def __call__(self, text: str) -> List[str]: + """Call signature.""" + ... + + #: Used to construct the default matcher identifier + __qualname__: str + + +class _MatcherAPIv1Total(_MatcherAPIv1Base, Protocol): + #: API version + matcher_api_version: Optional[Literal[1]] + + def __call__(self, text: str) -> List[str]: + """Call signature.""" + ... + + +#: Protocol describing Matcher API v1. +MatcherAPIv1: TypeAlias = Union[_MatcherAPIv1Base, _MatcherAPIv1Total] + + +class MatcherAPIv2(Protocol): + """Protocol describing Matcher API v2.""" + + #: API version + matcher_api_version: Literal[2] = 2 + + def __call__(self, context: CompletionContext) -> MatcherResult: + """Call signature.""" + ... + + #: Used to construct the default matcher identifier + __qualname__: str + + +Matcher: TypeAlias = Union[MatcherAPIv1, MatcherAPIv2] + + +def _is_matcher_v1(matcher: Matcher) -> TypeGuard[MatcherAPIv1]: + api_version = _get_matcher_api_version(matcher) + return api_version == 1 + + +def _is_matcher_v2(matcher: Matcher) -> TypeGuard[MatcherAPIv2]: + api_version = _get_matcher_api_version(matcher) + return api_version == 2 + + +def _is_sizable(value: Any) -> TypeGuard[Sized]: + """Determines whether objects is sizable""" + return hasattr(value, "__len__") + + +def _is_iterator(value: Any) -> TypeGuard[Iterator]: + """Determines whether objects is sizable""" + return hasattr(value, "__next__") + + +def has_any_completions(result: MatcherResult) -> bool: + """Check if any result includes any completions.""" + completions = result["completions"] + if _is_sizable(completions): + return len(completions) != 0 + if _is_iterator(completions): + try: + old_iterator = completions + first = next(old_iterator) + result["completions"] = cast( + Iterator[SimpleCompletion], + itertools.chain([first], old_iterator), + ) + return True + except StopIteration: + return False + raise ValueError( + "Completions returned by matcher need to be an Iterator or a Sizable" + ) + + +def completion_matcher( + *, + priority: Optional[float] = None, + identifier: Optional[str] = None, + api_version: int = 1, +): + """Adds attributes describing the matcher. + + Parameters + ---------- + priority : Optional[float] + The priority of the matcher, determines the order of execution of matchers. + Higher priority means that the matcher will be executed first. Defaults to 0. + identifier : Optional[str] + identifier of the matcher allowing users to modify the behaviour via traitlets, + and also used to for debugging (will be passed as ``origin`` with the completions). + + Defaults to matcher function's ``__qualname__`` (for example, + ``IPCompleter.file_matcher`` for the built-in matched defined + as a ``file_matcher`` method of the ``IPCompleter`` class). + api_version: Optional[int] + version of the Matcher API used by this matcher. + Currently supported values are 1 and 2. + Defaults to 1. + """ + + def wrapper(func: Matcher): + func.matcher_priority = priority or 0 # type: ignore + func.matcher_identifier = identifier or func.__qualname__ # type: ignore + func.matcher_api_version = api_version # type: ignore + if TYPE_CHECKING: + if api_version == 1: + func = cast(MatcherAPIv1, func) + elif api_version == 2: + func = cast(MatcherAPIv2, func) + return func + + return wrapper + + +def _get_matcher_priority(matcher: Matcher): + return getattr(matcher, "matcher_priority", 0) + + +def _get_matcher_id(matcher: Matcher): + return getattr(matcher, "matcher_identifier", matcher.__qualname__) + + +def _get_matcher_api_version(matcher): + return getattr(matcher, "matcher_api_version", 1) + + +context_matcher = partial(completion_matcher, api_version=2) + + _IC = Iterable[Completion] @@ -411,26 +809,25 @@ def _deduplicate_completions(text: str, completions: _IC)-> _IC: """ Deduplicate a set of completions. - .. warning:: Unstable + .. warning:: + + Unstable This function is unstable, API may change without warning. Parameters ---------- - text: str + text : str text that should be completed. - completions: Iterator[Completion] + completions : Iterator[Completion] iterator over the completions to deduplicate Yields ------ `Completions` objects - - Completions coming from multiple sources, may be different but end up having the same effect when applied to ``text``. If this is the case, this will consider completions as equal and only emit the first encountered. - Not folded in `completions()` yet for debugging purpose, and to detect when the IPython completer does return things that Jedi does not, but should be at some point. @@ -450,23 +847,28 @@ def _deduplicate_completions(text: str, completions: _IC)-> _IC: seen.add(new_text) -def rectify_completions(text: str, completions: _IC, *, _debug=False)->_IC: +def rectify_completions(text: str, completions: _IC, *, _debug: bool = False) -> _IC: """ Rectify a set of completions to all have the same ``start`` and ``end`` - .. warning:: Unstable + .. warning:: + + Unstable This function is unstable, API may change without warning. It will also raise unless use in proper context manager. Parameters ---------- - text: str + text : str text that should be completed. - completions: Iterator[Completion] + completions : Iterator[Completion] iterator over the completions to rectify + _debug : bool + Log failed completion - + Notes + ----- :any:`jedi.api.classes.Completion` s returned by Jedi may not have the same start and end, though the Jupyter Protocol requires them to behave like so. This will readjust the completion to have the same ``start`` and ``end`` by padding both @@ -566,13 +968,45 @@ def split_line(self, line, cursor_pos=None): class Completer(Configurable): - greedy = Bool(False, - help="""Activate greedy completion - PENDING DEPRECTION. this is now mostly taken care of with Jedi. + greedy = Bool( + False, + help="""Activate greedy completion. - This will enable completion on elements of lists, results of function calls, etc., - but can be unsafe because the code is actually evaluated on TAB. - """ + .. deprecated:: 8.8 + Use :std:configtrait:`Completer.evaluation` and :std:configtrait:`Completer.auto_close_dict_keys` instead. + + When enabled in IPython 8.8 or newer, changes configuration as follows: + + - ``Completer.evaluation = 'unsafe'`` + - ``Completer.auto_close_dict_keys = True`` + """, + ).tag(config=True) + + evaluation = Enum( + ("forbidden", "minimal", "limited", "unsafe", "dangerous"), + default_value="limited", + help="""Policy for code evaluation under completion. + + Successive options allow to enable more eager evaluation for better + completion suggestions, including for nested dictionaries, nested lists, + or even results of function calls. + Setting ``unsafe`` or higher can lead to evaluation of arbitrary user + code on :kbd:`Tab` with potentially unwanted or dangerous side effects. + + Allowed values are: + + - ``forbidden``: no evaluation of code is permitted, + - ``minimal``: evaluation of literals and access to built-in namespace; + no item/attribute evaluationm no access to locals/globals, + no evaluation of any operations or comparisons. + - ``limited``: access to all namespaces, evaluation of hard-coded methods + (for example: :any:`dict.keys`, :any:`object.__getattr__`, + :any:`object.__getitem__`) on allow-listed objects (for example: + :any:`dict`, :any:`list`, :any:`tuple`, ``pandas.Series``), + - ``unsafe``: evaluation of all methods and function calls but not of + syntax with side-effects like `del x`, + - ``dangerous``: completely arbitrary evaluation. + """, ).tag(config=True) use_jedi = Bool(default_value=JEDI_INSTALLED, @@ -595,7 +1029,17 @@ class Completer(Configurable): "Includes completion of latex commands, unicode names, and expanding " "unicode characters back to latex commands.").tag(config=True) + auto_close_dict_keys = Bool( + False, + help=""" + Enable auto-closing dictionary keys. + When enabled string keys will be suffixed with a final quote + (matching the opening quote), tuple keys will also receive a + separating comma if needed, and keys which are final will + receive a closing bracket (``]``). + """, + ).tag(config=True) def __init__(self, namespace=None, global_namespace=None, **kwargs): """Create a new completer for the command line. @@ -626,6 +1070,8 @@ def __init__(self, namespace=None, global_namespace=None, **kwargs): else: self.global_namespace = global_namespace + self.custom_matchers = [] + super(Completer, self).__init__(**kwargs) def complete(self, text, state): @@ -658,19 +1104,23 @@ def global_matches(self, text): matches = [] match_append = matches.append n = len(text) - for lst in [keyword.kwlist, - builtin_mod.__dict__.keys(), - self.namespace.keys(), - self.global_namespace.keys()]: + for lst in [ + keyword.kwlist, + builtin_mod.__dict__.keys(), + list(self.namespace.keys()), + list(self.global_namespace.keys()), + ]: for word in lst: if word[:n] == text and word != "__builtins__": match_append(word) snake_case_re = re.compile(r"[^_]+(_[^_]+)+?\Z") - for lst in [self.namespace.keys(), - self.global_namespace.keys()]: - shortened = {"_".join([sub[0] for sub in word.split('_')]) : word - for word in lst if snake_case_re.match(word)} + for lst in [list(self.namespace.keys()), list(self.global_namespace.keys())]: + shortened = { + "_".join([sub[0] for sub in word.split("_")]): word + for word in lst + if snake_case_re.match(word) + } for word in shortened.keys(): if word[:n] == text and word != "__builtins__": match_append(shortened[word]) @@ -689,28 +1139,16 @@ def attr_matches(self, text): with a __getattr__ hook is evaluated. """ + m2 = re.match(r"(.+)\.(\w*)$", self.line_buffer) + if not m2: + return [] + expr, attr = m2.group(1, 2) - # Another option, seems to work great. Catches things like ''. - m = re.match(r"(\S+(\.\w+)*)\.(\w*)$", text) + obj = self._evaluate_expr(expr) - if m: - expr, attr = m.group(1, 3) - elif self.greedy: - m2 = re.match(r"(.+)\.(\w*)$", self.line_buffer) - if not m2: - return [] - expr, attr = m2.group(1,2) - else: + if obj is not_found: return [] - try: - obj = eval(expr, self.namespace) - except: - try: - obj = eval(expr, self.global_namespace) - except: - return [] - if self.limit_to__all__ and hasattr(obj, '__all__'): words = get__all__entries(obj) else: @@ -728,8 +1166,31 @@ def attr_matches(self, text): pass # Build match list to return n = len(attr) - return [u"%s.%s" % (expr, w) for w in words if w[:n] == attr ] + return ["%s.%s" % (expr, w) for w in words if w[:n] == attr] + def _evaluate_expr(self, expr): + obj = not_found + done = False + while not done and expr: + try: + obj = guarded_eval( + expr, + EvaluationContext( + globals=self.global_namespace, + locals=self.namespace, + evaluation=self.evaluation, + ), + ) + done = True + except Exception as e: + if self.debug: + print("Evaluation exception", e) + # trim the expression to remove any invalid prefix + # e.g. user starts `(d[`, so we get `expr = '(d'`, + # where parenthesis is not closed. + # TODO: make this faster by reusing parts of the computation? + expr = expr[1:] + return obj def get__all__entries(obj): """returns the strings in the __all__ attribute""" @@ -741,63 +1202,223 @@ def get__all__entries(obj): return [w for w in words if isinstance(w, str)] -def match_dict_keys(keys: List[str], prefix: str, delims: str): +class _DictKeyState(enum.Flag): + """Represent state of the key match in context of other possible matches. + + - given `d1 = {'a': 1}` completion on `d1['` will yield `{'a': END_OF_ITEM}` as there is no tuple. + - given `d2 = {('a', 'b'): 1}`: `d2['a', '` will yield `{'b': END_OF_TUPLE}` as there is no tuple members to add beyond `'b'`. + - given `d3 = {('a', 'b'): 1}`: `d3['` will yield `{'a': IN_TUPLE}` as `'a'` can be added. + - given `d4 = {'a': 1, ('a', 'b'): 2}`: `d4['` will yield `{'a': END_OF_ITEM & END_OF_TUPLE}` + """ + + BASELINE = 0 + END_OF_ITEM = enum.auto() + END_OF_TUPLE = enum.auto() + IN_TUPLE = enum.auto() + + +def _parse_tokens(c): + """Parse tokens even if there is an error.""" + tokens = [] + token_generator = tokenize.generate_tokens(iter(c.splitlines()).__next__) + while True: + try: + tokens.append(next(token_generator)) + except tokenize.TokenError: + return tokens + except StopIteration: + return tokens + + +def _match_number_in_dict_key_prefix(prefix: str) -> Union[str, None]: + """Match any valid Python numeric literal in a prefix of dictionary keys. + + References: + - https://docs.python.org/3/reference/lexical_analysis.html#numeric-literals + - https://docs.python.org/3/library/tokenize.html + """ + if prefix[-1].isspace(): + # if user typed a space we do not have anything to complete + # even if there was a valid number token before + return None + tokens = _parse_tokens(prefix) + rev_tokens = reversed(tokens) + skip_over = {tokenize.ENDMARKER, tokenize.NEWLINE} + number = None + for token in rev_tokens: + if token.type in skip_over: + continue + if number is None: + if token.type == tokenize.NUMBER: + number = token.string + continue + else: + # we did not match a number + return None + if token.type == tokenize.OP: + if token.string == ",": + break + if token.string in {"+", "-"}: + number = token.string + number + else: + return None + return number + + +_INT_FORMATS = { + "0b": bin, + "0o": oct, + "0x": hex, +} + + +def match_dict_keys( + keys: List[Union[str, bytes, Tuple[Union[str, bytes], ...]]], + prefix: str, + delims: str, + extra_prefix: Optional[Tuple[Union[str, bytes], ...]] = None, +) -> Tuple[str, int, Dict[str, _DictKeyState]]: """Used by dict_key_matches, matching the prefix to a list of keys Parameters - ========== - keys: + ---------- + keys list of keys in dictionary currently being completed. - prefix: - Part of the text already typed by the user. e.g. `mydict[b'fo` - delims: + prefix + Part of the text already typed by the user. E.g. `mydict[b'fo` + delims String of delimiters to consider when finding the current key. + extra_prefix : optional + Part of the text already typed in multi-key index cases. E.g. for + `mydict['foo', "bar", 'b`, this would be `('foo', 'bar')`. Returns - ======= - + ------- A tuple of three elements: ``quote``, ``token_start``, ``matched``, with ``quote`` being the quote that need to be used to close current string. ``token_start`` the position where the replacement should start occurring, - ``matches`` a list of replacement/completion - + ``matches`` a dictionary of replacement/completion keys on keys and values + indicating whether the state. """ + prefix_tuple = extra_prefix if extra_prefix else () + + prefix_tuple_size = sum( + [ + # for pandas, do not count slices as taking space + not isinstance(k, slice) + for k in prefix_tuple + ] + ) + text_serializable_types = (str, bytes, int, float, slice) + + def filter_prefix_tuple(key): + # Reject too short keys + if len(key) <= prefix_tuple_size: + return False + # Reject keys which cannot be serialised to text + for k in key: + if not isinstance(k, text_serializable_types): + return False + # Reject keys that do not match the prefix + for k, pt in zip(key, prefix_tuple): + if k != pt and not isinstance(pt, slice): + return False + # All checks passed! + return True + + filtered_key_is_final: Dict[ + Union[str, bytes, int, float], _DictKeyState + ] = defaultdict(lambda: _DictKeyState.BASELINE) + + for k in keys: + # If at least one of the matches is not final, mark as undetermined. + # This can happen with `d = {111: 'b', (111, 222): 'a'}` where + # `111` appears final on first match but is not final on the second. + + if isinstance(k, tuple): + if filter_prefix_tuple(k): + key_fragment = k[prefix_tuple_size] + filtered_key_is_final[key_fragment] |= ( + _DictKeyState.END_OF_TUPLE + if len(k) == prefix_tuple_size + 1 + else _DictKeyState.IN_TUPLE + ) + elif prefix_tuple_size > 0: + # we are completing a tuple but this key is not a tuple, + # so we should ignore it + pass + else: + if isinstance(k, text_serializable_types): + filtered_key_is_final[k] |= _DictKeyState.END_OF_ITEM + + filtered_keys = filtered_key_is_final.keys() + if not prefix: - return None, 0, [repr(k) for k in keys - if isinstance(k, (str, bytes))] - quote_match = re.search('["\']', prefix) - quote = quote_match.group() - try: - prefix_str = eval(prefix + quote, {}) - except Exception: - return None, 0, [] + return "", 0, {repr(k): v for k, v in filtered_key_is_final.items()} + + quote_match = re.search("(?:\"|')", prefix) + is_user_prefix_numeric = False + + if quote_match: + quote = quote_match.group() + valid_prefix = prefix + quote + try: + prefix_str = literal_eval(valid_prefix) + except Exception: + return "", 0, {} + else: + # If it does not look like a string, let's assume + # we are dealing with a number or variable. + number_match = _match_number_in_dict_key_prefix(prefix) + + # We do not want the key matcher to suggest variable names so we yield: + if number_match is None: + # The alternative would be to assume that user forgort the quote + # and if the substring matches, suggest adding it at the start. + return "", 0, {} + + prefix_str = number_match + is_user_prefix_numeric = True + quote = "" pattern = '[^' + ''.join('\\' + c for c in delims) + ']*$' token_match = re.search(pattern, prefix, re.UNICODE) + assert token_match is not None # silence mypy token_start = token_match.start() token_prefix = token_match.group() - matched = [] - for key in keys: + matched: Dict[str, _DictKeyState] = {} + + str_key: Union[str, bytes] + + for key in filtered_keys: + if isinstance(key, (int, float)): + # User typed a number but this key is not a number. + if not is_user_prefix_numeric: + continue + str_key = str(key) + if isinstance(key, int): + int_base = prefix_str[:2].lower() + # if user typed integer using binary/oct/hex notation: + if int_base in _INT_FORMATS: + int_format = _INT_FORMATS[int_base] + str_key = int_format(key) + else: + # User typed a string but this key is a number. + if is_user_prefix_numeric: + continue + str_key = key try: - if not key.startswith(prefix_str): + if not str_key.startswith(prefix_str): continue - except (AttributeError, TypeError, UnicodeError): + except (AttributeError, TypeError, UnicodeError) as e: # Python 3+ TypeError on b'a'.startswith('a') or vice-versa continue # reformat remainder of key to begin with prefix - rem = key[len(prefix_str):] + rem = str_key[len(prefix_str) :] # force repr wrapped in ' rem_repr = repr(rem + '"') if isinstance(rem, str) else repr(rem + b'"') - if rem_repr.startswith('u') and prefix[0] not in 'uU': - # Found key is unicode, but prefix is Py2 string. - # Therefore attempt to interpret key as string. - try: - rem_repr = repr(rem.encode('ascii') + '"') - except UnicodeEncodeError: - continue - rem_repr = rem_repr[1 + rem_repr.index("'"):-2] if quote == '"': # The entered prefix is quoted with ", @@ -806,19 +1427,19 @@ def match_dict_keys(keys: List[str], prefix: str, delims: str): rem_repr = rem_repr.replace('"', '\\"') # then reinsert prefix from start of token - matched.append('%s%s' % (token_prefix, rem_repr)) + match = "%s%s" % (token_prefix, rem_repr) + + matched[match] = filtered_key_is_final[key] return quote, token_start, matched def cursor_to_position(text:str, line:int, column:int)->int: """ - Convert the (line,column) position of the cursor in text to an offset in a string. Parameters ---------- - text : str The text in which to calculate the cursor offset line : int @@ -826,13 +1447,13 @@ def cursor_to_position(text:str, line:int, column:int)->int: column : int Column of the cursor 0-indexed - Return - ------ - Position of the cursor in ``text``, 0-indexed. + Returns + ------- + Position of the cursor in ``text``, 0-indexed. See Also -------- - position_to_cursor: reciprocal of this function + position_to_cursor : reciprocal of this function """ lines = text.split('\n') @@ -849,23 +1470,20 @@ def position_to_cursor(text:str, offset:int)->Tuple[int, int]: Parameters ---------- - text : str The text in which to calculate the cursor offset offset : int Position of the cursor in ``text``, 0-indexed. - Return - ------ + Returns + ------- (line, column) : (int, int) Line of the cursor; 0-indexed, column of the cursor 0-indexed - See Also -------- cursor_to_position : reciprocal of this function - """ assert 0 <= offset <= len(text) , "0 <= %s <= %s" % (offset , len(text)) @@ -877,15 +1495,30 @@ def position_to_cursor(text:str, offset:int)->Tuple[int, int]: return line, col -def _safe_isinstance(obj, module, class_name): +def _safe_isinstance(obj, module, class_name, *attrs): """Checks if obj is an instance of module.class_name if loaded """ - return (module in sys.modules and - isinstance(obj, getattr(import_module(module), class_name))) + if module in sys.modules: + m = sys.modules[module] + for attr in [class_name, *attrs]: + m = getattr(m, attr) + return isinstance(obj, m) + +@context_matcher() +def back_unicode_name_matcher(context: CompletionContext): + """Match Unicode characters back to Unicode name -def back_unicode_name_matches(text): - u"""Match unicode characters back to unicode name + Same as :any:`back_unicode_name_matches`, but adopted to new Matcher API. + """ + fragment, matches = back_unicode_name_matches(context.text_until_cursor) + return _convert_matcher_v1_result_to_v2( + matches, type="unicode", fragment=fragment, suppress_if_matches=True + ) + + +def back_unicode_name_matches(text: str) -> Tuple[str, Sequence[str]]: + """Match Unicode characters back to Unicode name This does ``☃`` -> ``\\snowman`` @@ -894,52 +1527,77 @@ def back_unicode_name_matches(text): This will not either back-complete standard sequences like \\n, \\b ... - Used on Python 3 only. + .. deprecated:: 8.6 + You can use :meth:`back_unicode_name_matcher` instead. + + Returns + ======= + + Return a tuple with two elements: + + - The Unicode character that was matched (preceded with a backslash), or + empty string, + - a sequence (of 1), name for the match Unicode character, preceded by + backslash, or empty if no match. """ if len(text)<2: - return u'', () + return '', () maybe_slash = text[-2] if maybe_slash != '\\': - return u'', () + return '', () char = text[-1] # no expand on quote for completion in strings. # nor backcomplete standard ascii keys - if char in string.ascii_letters or char in ['"',"'"]: - return u'', () + if char in string.ascii_letters or char in ('"',"'"): + return '', () try : unic = unicodedata.name(char) - return '\\'+char,['\\'+unic] + return '\\'+char,('\\'+unic,) except KeyError: pass - return u'', () + return '', () -def back_latex_name_matches(text:str): + +@context_matcher() +def back_latex_name_matcher(context: CompletionContext): + """Match latex characters back to unicode name + + Same as :any:`back_latex_name_matches`, but adopted to new Matcher API. + """ + fragment, matches = back_latex_name_matches(context.text_until_cursor) + return _convert_matcher_v1_result_to_v2( + matches, type="latex", fragment=fragment, suppress_if_matches=True + ) + + +def back_latex_name_matches(text: str) -> Tuple[str, Sequence[str]]: """Match latex characters back to unicode name This does ``\\ℵ`` -> ``\\aleph`` - Used on Python 3 only. + .. deprecated:: 8.6 + You can use :meth:`back_latex_name_matcher` instead. """ if len(text)<2: - return u'', () + return '', () maybe_slash = text[-2] if maybe_slash != '\\': - return u'', () + return '', () char = text[-1] # no expand on quote for completion in strings. # nor backcomplete standard ascii keys - if char in string.ascii_letters or char in ['"',"'"]: - return u'', () + if char in string.ascii_letters or char in ('"',"'"): + return '', () try : latex = reverse_latex_symbol[char] # '\\' replace the \ as well return '\\'+char,[latex] except KeyError: pass - return u'', () + return '', () def _formatparamchildren(parameter) -> str: @@ -948,18 +1606,15 @@ def _formatparamchildren(parameter) -> str: Jedi does not expose a simple way to get `param=value` from its API. - Parameter - ========= - - parameter: + Parameters + ---------- + parameter Jedi's function `Param` Returns - ======= - + ------- A string like 'a', 'b=1', '*args', '**kwargs' - """ description = parameter.description if not description.startswith('param '): @@ -971,47 +1626,172 @@ def _make_signature(completion)-> str: """ Make the signature from a jedi completion - Parameter - ========= - - completion: jedi.Completion + Parameters + ---------- + completion : jedi.Completion object does not complete a function type Returns - ======= - + ------- a string consisting of the function signature, with the parenthesis but without the function name. example: `(a, *args, b=1, **kwargs)` """ - return '(%s)'% ', '.join([f for f in (_formatparamchildren(p) for p in completion.params) if f]) + # it looks like this might work on jedi 0.17 + if hasattr(completion, 'get_signatures'): + signatures = completion.get_signatures() + if not signatures: + return '(?)' + + c0 = completion.get_signatures()[0] + return '('+c0.to_string().split('(', maxsplit=1)[1] + + return '(%s)'% ', '.join([f for f in (_formatparamchildren(p) for signature in completion.get_signatures() + for p in signature.defined_names()) if f]) + + +_CompleteResult = Dict[str, MatcherResult] + + +DICT_MATCHER_REGEX = re.compile( + r"""(?x) +( # match dict-referring - or any get item object - expression + .+ +) +\[ # open bracket +\s* # and optional whitespace +# Capture any number of serializable objects (e.g. "a", "b", 'c') +# and slices +((?:(?: + (?: # closed string + [uUbB]? # string prefix (r not handled) + (?: + '(?:[^']|(? SimpleMatcherResult: + """Utility to help with transition""" + result = { + "completions": [SimpleCompletion(text=match, type=type) for match in matches], + "suppress": (True if matches else False) if suppress_if_matches else False, + } + if fragment is not None: + result["matched_fragment"] = fragment + return cast(SimpleMatcherResult, result) + class IPCompleter(Completer): """Extension of the completer class with IPython-specific features""" - _names = None - @observe('greedy') def _greedy_changed(self, change): """update the splitter and readline delims when greedy is changed""" - if change['new']: + if change["new"]: + self.evaluation = "unsafe" + self.auto_close_dict_keys = True self.splitter.delims = GREEDY_DELIMS else: + self.evaluation = "limited" + self.auto_close_dict_keys = False self.splitter.delims = DELIMS - dict_keys_only = Bool(False, - help="""Whether to show dict key matches only""") + dict_keys_only = Bool( + False, + help=""" + Whether to show dict key matches only. + + (disables all matchers except for `IPCompleter.dict_key_matcher`). + """, + ) - merge_completions = Bool(True, + suppress_competing_matchers = UnionTrait( + [Bool(allow_none=True), DictTrait(Bool(None, allow_none=True))], + default_value=None, + help=""" + Whether to suppress completions from other *Matchers*. + + When set to ``None`` (default) the matchers will attempt to auto-detect + whether suppression of other matchers is desirable. For example, at + the beginning of a line followed by `%` we expect a magic completion + to be the only applicable option, and after ``my_dict['`` we usually + expect a completion with an existing dictionary key. + + If you want to disable this heuristic and see completions from all matchers, + set ``IPCompleter.suppress_competing_matchers = False``. + To disable the heuristic for specific matchers provide a dictionary mapping: + ``IPCompleter.suppress_competing_matchers = {'IPCompleter.dict_key_matcher': False}``. + + Set ``IPCompleter.suppress_competing_matchers = True`` to limit + completions to the set of matchers with the highest priority; + this is equivalent to ``IPCompleter.merge_completions`` and + can be beneficial for performance, but will sometimes omit relevant + candidates from matchers further down the priority list. + """, + ).tag(config=True) + + merge_completions = Bool( + True, help="""Whether to merge completion results into a single list If False, only the completion results from the first non-empty completer will be returned. - """ + + As of version 8.6.0, setting the value to ``False`` is an alias for: + ``IPCompleter.suppress_competing_matchers = True.``. + """, ).tag(config=True) - omit__names = Enum((0,1,2), default_value=2, + + disable_matchers = ListTrait( + Unicode(), + help="""List of matchers to disable. + + The list should contain matcher identifiers (see :any:`completion_matcher`). + """, + ).tag(config=True) + + omit__names = Enum( + (0, 1, 2), + default_value=2, help="""Instruct the completer to omit private method names Specifically, when completing on ``object.``. @@ -1037,6 +1817,16 @@ def _greedy_changed(self, change): """, ).tag(config=True) + profile_completions = Bool( + default_value=False, + help="If True, emit profiling data for completion subsystem using cProfile." + ).tag(config=True) + + profiler_output_dir = Unicode( + default_value=".completion_profiles", + help="Template for path at which to output profile data for completions." + ).tag(config=True) + @observe('limit_to__all__') def _limit_to_all_changed(self, change): warnings.warn('`IPython.core.IPCompleter.limit_to__all__` configuration ' @@ -1044,42 +1834,41 @@ def _limit_to_all_changed(self, change): 'no effects and then removed in future version of IPython.', UserWarning) - def __init__(self, shell=None, namespace=None, global_namespace=None, - use_readline=_deprecation_readline_sentinel, config=None, **kwargs): + def __init__( + self, shell=None, namespace=None, global_namespace=None, config=None, **kwargs + ): """IPCompleter() -> completer Return a completer object. Parameters ---------- - shell a pointer to the ipython shell itself. This is needed because this completer knows about magic functions, and those can only be accessed via the ipython instance. - namespace : dict, optional an optional dict where completions are performed. - global_namespace : dict, optional secondary optional dict for completions, to handle cases (such as IPython embedded inside functions) where both Python scopes are visible. - - use_readline : bool, optional - DEPRECATED, ignored since IPython 6.0, will have no effects + config : Config + traitlet's config object + **kwargs + passed to super class unmodified. """ self.magic_escape = ESC_MAGIC self.splitter = CompletionSplitter() - if use_readline is not _deprecation_readline_sentinel: - warnings.warn('The `use_readline` parameter is deprecated and ignored since IPython 6.0.', - DeprecationWarning, stacklevel=2) - # _greedy_changed() depends on splitter and readline being defined: - Completer.__init__(self, namespace=namespace, global_namespace=global_namespace, - config=config, **kwargs) + super().__init__( + namespace=namespace, + global_namespace=global_namespace, + config=config, + **kwargs, + ) # List where completion matches will be stored self.matches = [] @@ -1107,35 +1896,66 @@ def __init__(self, shell=None, namespace=None, global_namespace=None, #= re.compile(r'[\s|\[]*(\w+)(?:\s*=?\s*.*)') self.magic_arg_matchers = [ - self.magic_config_matches, - self.magic_color_matches, + self.magic_config_matcher, + self.magic_color_matcher, ] # This is set externally by InteractiveShell self.custom_completers = None + # This is a list of names of unicode characters that can be completed + # into their corresponding unicode value. The list is large, so we + # lazily initialize it on first use. Consuming code should access this + # attribute through the `@unicode_names` property. + self._unicode_names = None + + self._backslash_combining_matchers = [ + self.latex_name_matcher, + self.unicode_name_matcher, + back_latex_name_matcher, + back_unicode_name_matcher, + self.fwd_unicode_matcher, + ] + + if not self.backslash_combining_completions: + for matcher in self._backslash_combining_matchers: + self.disable_matchers.append(_get_matcher_id(matcher)) + + if not self.merge_completions: + self.suppress_competing_matchers = True + @property - def matchers(self): + def matchers(self) -> List[Matcher]: """All active matcher routines for completion""" if self.dict_keys_only: - return [self.dict_key_matches] + return [self.dict_key_matcher] if self.use_jedi: return [ - self.file_matches, - self.magic_matches, - self.dict_key_matches, + *self.custom_matchers, + *self._backslash_combining_matchers, + *self.magic_arg_matchers, + self.custom_completer_matcher, + self.magic_matcher, + self._jedi_matcher, + self.dict_key_matcher, + self.file_matcher, ] else: return [ + *self.custom_matchers, + *self._backslash_combining_matchers, + *self.magic_arg_matchers, + self.custom_completer_matcher, + self.dict_key_matcher, + # TODO: convert python_matches to v2 API + self.magic_matcher, self.python_matches, - self.file_matches, - self.magic_matches, - self.python_func_kw_matches, - self.dict_key_matches, + self.file_matcher, + self.python_func_kw_matcher, ] - def all_completions(self, text) -> List[str]: + def all_completions(self, text:str) -> List[str]: """ Wrapper around the completion methods for the benefit of emacs. """ @@ -1146,14 +1966,22 @@ def all_completions(self, text) -> List[str]: return self.complete(text)[1] - def _clean_glob(self, text): + def _clean_glob(self, text:str): return self.glob("%s*" % text) - def _clean_glob_win32(self,text): + def _clean_glob_win32(self, text:str): return [f.replace("\\","/") for f in self.glob("%s*" % text)] - def file_matches(self, text): + @context_matcher() + def file_matcher(self, context: CompletionContext) -> SimpleMatcherResult: + """Same as :any:`file_matches`, but adopted to new Matcher API.""" + matches = self.file_matches(context.token) + # TODO: add a heuristic for suppressing (e.g. if it has OS-specific delimiter, + # starts with `/home/`, `C:\`, etc) + return _convert_matcher_v1_result_to_v2(matches, type="path") + + def file_matches(self, text: str) -> List[str]: """Match filenames, expanding ~USER type strings. Most of the seemingly convoluted logic in this completer is an @@ -1165,7 +1993,11 @@ def file_matches(self, text): only the parts after what's already been typed (instead of the full completions, as is normally done). I don't think with the current (as of Python 2.3) Python readline it's possible to do - better.""" + better. + + .. deprecated:: 8.6 + You can use :meth:`file_matcher` instead. + """ # chars that require escaping with backslash - i.e. chars # that readline treats incorrectly as delimiters, but we @@ -1235,8 +2067,22 @@ def file_matches(self, text): # Mark directories in input list by appending '/' to their names. return [x+'/' if os.path.isdir(x) else x for x in matches] - def magic_matches(self, text): - """Match magics""" + @context_matcher() + def magic_matcher(self, context: CompletionContext) -> SimpleMatcherResult: + """Match magics.""" + text = context.token + matches = self.magic_matches(text) + result = _convert_matcher_v1_result_to_v2(matches, type="magic") + is_magic_prefix = len(text) > 0 and text[0] == "%" + result["suppress"] = is_magic_prefix and bool(result["completions"]) + return result + + def magic_matches(self, text: str): + """Match magics. + + .. deprecated:: 8.6 + You can use :meth:`magic_matcher` instead. + """ # Get all shell magics now rather than statically, so magics loaded at # runtime show up too. lsm = self.shell.magics_manager.lsmagic() @@ -1277,8 +2123,19 @@ def matches(magic): return comp - def magic_config_matches(self, text:str) -> List[str]: - """ Match class names and attributes for %config magic """ + @context_matcher() + def magic_config_matcher(self, context: CompletionContext) -> SimpleMatcherResult: + """Match class names and attributes for %config magic.""" + # NOTE: uses `line_buffer` equivalent for compatibility + matches = self.magic_config_matches(context.line_with_cursor) + return _convert_matcher_v1_result_to_v2(matches, type="param") + + def magic_config_matches(self, text: str) -> List[str]: + """Match class names and attributes for %config magic. + + .. deprecated:: 8.6 + You can use :meth:`magic_config_matcher` instead. + """ texts = text.strip().split() if len(texts) > 0 and (texts[0] == 'config' or texts[0] == '%config'): @@ -1312,8 +2169,19 @@ def magic_config_matches(self, text:str) -> List[str]: if attr.startswith(texts[1]) ] return [] - def magic_color_matches(self, text:str) -> List[str] : - """ Match color schemes for %colors magic""" + @context_matcher() + def magic_color_matcher(self, context: CompletionContext) -> SimpleMatcherResult: + """Match color schemes for %colors magic.""" + # NOTE: uses `line_buffer` equivalent for compatibility + matches = self.magic_color_matches(context.line_with_cursor) + return _convert_matcher_v1_result_to_v2(matches, type="param") + + def magic_color_matches(self, text: str) -> List[str]: + """Match color schemes for %colors magic. + + .. deprecated:: 8.6 + You can use :meth:`magic_color_matcher` instead. + """ texts = text.split() if text.endswith(' '): # .split() strips off the trailing whitespace. Add '' back @@ -1326,10 +2194,24 @@ def magic_color_matches(self, text:str) -> List[str] : if color.startswith(prefix) ] return [] - def _jedi_matches(self, cursor_column:int, cursor_line:int, text:str): + @context_matcher(identifier="IPCompleter.jedi_matcher") + def _jedi_matcher(self, context: CompletionContext) -> _JediMatcherResult: + matches = self._jedi_matches( + cursor_column=context.cursor_position, + cursor_line=context.cursor_line, + text=context.full_text, + ) + return { + "completions": matches, + # static analysis should not suppress other matchers + "suppress": False, + } + + def _jedi_matches( + self, cursor_column: int, cursor_line: int, text: str + ) -> Iterator[_JediCompletionLike]: """ - - Return a list of :any:`jedi.api.Completions` object from a ``text`` and + Return a list of :any:`jedi.api.Completion`s object from a ``text`` and cursor position. Parameters @@ -1341,11 +2223,13 @@ def _jedi_matches(self, cursor_column:int, cursor_line:int, text:str): text : str text to complete - Debugging - --------- - + Notes + ----- If ``IPCompleter.debug`` is ``True`` may return a :any:`_FakeJediCompletion` object containing a string with the Jedi debug information attached. + + .. deprecated:: 8.6 + You can use :meth:`_jedi_matcher` instead. """ namespaces = [self.namespace] if self.global_namespace is not None: @@ -1366,23 +2250,22 @@ def _jedi_matches(self, cursor_column:int, cursor_line:int, text:str): else: raise ValueError("Don't understand self.omit__names == {}".format(self.omit__names)) - interpreter = jedi.Interpreter( - text[:offset], namespaces, column=cursor_column, line=cursor_line + 1) + interpreter = jedi.Interpreter(text[:offset], namespaces) try_jedi = True try: - # should we check the type of the node is Error ? + # find the first token in the current tree -- if it is a ' or " then we are in a string + completing_string = False try: - # jedi < 0.11 - from jedi.parser.tree import ErrorLeaf - except ImportError: - # jedi >= 0.11 - from parso.tree import ErrorLeaf + first_child = next(c for c in interpreter._get_module().tree_node.children if hasattr(c, 'value')) + except StopIteration: + pass + else: + # note the value may be ', ", or it may also be ''' or """, or + # in some cases, """what/you/typed..., but all of these are + # strings. + completing_string = len(first_child.value) > 0 and first_child.value[0] in {"'", '"'} - next_to_last_tree = interpreter._get_module().tree_node.children[-2] - completing_string = False - if isinstance(next_to_last_tree, ErrorLeaf): - completing_string = next_to_last_tree.value.lstrip()[0] in {'"', "'"} # if we are in a string jedi is likely not the right candidate for # now. Skip it. try_jedi = not completing_string @@ -1392,16 +2275,24 @@ def _jedi_matches(self, cursor_column:int, cursor_line:int, text:str): print("Error detecting if completing a non-finished string :", e, '|') if not try_jedi: - return [] + return iter([]) try: - return filter(completion_filter, interpreter.completions()) + return filter(completion_filter, interpreter.complete(column=cursor_column, line=cursor_line + 1)) except Exception as e: if self.debug: - return [_FakeJediCompletion('Oops Jedi has crashed, please report a bug with the following:\n"""\n%s\ns"""' % (e))] + return iter( + [ + _FakeJediCompletion( + 'Oops Jedi has crashed, please report a bug with the following:\n"""\n%s\ns"""' + % (e) + ) + ] + ) else: - return [] + return iter([]) - def python_matches(self, text): + @completion_matcher(api_version=1) + def python_matches(self, text: str) -> Iterable[str]: """Match attributes or global python names""" if "." in text: try: @@ -1475,7 +2366,7 @@ def _default_arguments(self, obj): inspect.Parameter.POSITIONAL_OR_KEYWORD) try: - sig = inspect.signature(call_obj) + sig = inspect.signature(obj) ret.extend(k for k, v in sig.parameters.items() if v.kind in _keeps) except ValueError: @@ -1483,8 +2374,18 @@ def _default_arguments(self, obj): return list(set(ret)) - def python_func_kw_matches(self,text): - """Match named parameters (kwargs) of the last open function""" + @context_matcher() + def python_func_kw_matcher(self, context: CompletionContext) -> SimpleMatcherResult: + """Match named parameters (kwargs) of the last open function.""" + matches = self.python_func_kw_matches(context.token) + return _convert_matcher_v1_result_to_v2(matches, type="param") + + def python_func_kw_matches(self, text): + """Match named parameters (kwargs) of the last open function. + + .. deprecated:: 8.6 + You can use :meth:`python_func_kw_matcher` instead. + """ if "." in text: # a parameter cannot be dotted return [] @@ -1553,89 +2454,95 @@ def python_func_kw_matches(self,text): # Remove used named arguments from the list, no need to show twice for namedArg in set(namedArgs) - usedNamedArgs: if namedArg.startswith(text): - argMatches.append(u"%s=" %namedArg) + argMatches.append("%s=" %namedArg) except: pass return argMatches - def dict_key_matches(self, text): - "Match string keys in a dictionary, after e.g. 'foo[' " - def get_keys(obj): - # Objects can define their own completions by defining an - # _ipy_key_completions_() method. - method = get_real_method(obj, '_ipython_key_completions_') - if method is not None: - return method() - - # Special case some common in-memory dict-like types - if isinstance(obj, dict) or\ - _safe_isinstance(obj, 'pandas', 'DataFrame'): - try: - return list(obj.keys()) - except Exception: - return [] - elif _safe_isinstance(obj, 'numpy', 'ndarray') or\ - _safe_isinstance(obj, 'numpy', 'void'): - return obj.dtype.names or [] + @staticmethod + def _get_keys(obj: Any) -> List[Any]: + # Objects can define their own completions by defining an + # _ipy_key_completions_() method. + method = get_real_method(obj, '_ipython_key_completions_') + if method is not None: + return method() + + # Special case some common in-memory dict-like types + if isinstance(obj, dict) or _safe_isinstance(obj, "pandas", "DataFrame"): + try: + return list(obj.keys()) + except Exception: + return [] + elif _safe_isinstance(obj, "pandas", "core", "indexing", "_LocIndexer"): + try: + return list(obj.obj.keys()) + except Exception: + return [] + elif _safe_isinstance(obj, 'numpy', 'ndarray') or\ + _safe_isinstance(obj, 'numpy', 'void'): + return obj.dtype.names or [] + return [] + + @context_matcher() + def dict_key_matcher(self, context: CompletionContext) -> SimpleMatcherResult: + """Match string keys in a dictionary, after e.g. ``foo[``.""" + matches = self.dict_key_matches(context.token) + return _convert_matcher_v1_result_to_v2( + matches, type="dict key", suppress_if_matches=True + ) + + def dict_key_matches(self, text: str) -> List[str]: + """Match string keys in a dictionary, after e.g. ``foo[``. + + .. deprecated:: 8.6 + You can use :meth:`dict_key_matcher` instead. + """ + + # Short-circuit on closed dictionary (regular expression would + # not match anyway, but would take quite a while). + if self.text_until_cursor.strip().endswith("]"): return [] - try: - regexps = self.__dict_key_regexps - except AttributeError: - dict_key_re_fmt = r'''(?x) - ( # match dict-referring expression wrt greedy setting - %s - ) - \[ # open bracket - \s* # and optional whitespace - ([uUbB]? # string prefix (r not handled) - (?: # unclosed string - '(?:[^']|(? text_start and closing_quote: - # quotes were opened inside text, maybe close them - if continuation.startswith(closing_quote): - continuation = continuation[len(closing_quote):] - else: - suf += closing_quote - if bracket_idx > text_start: - # brackets were opened inside text, maybe close them - if not continuation.startswith(']'): - suf += ']' + # the text given to this method, e.g. `d["""a\nt + can_close_quote = False + can_close_bracket = False - return [leading + k + suf for k in matches] + continuation = self.line_buffer[len(self.text_until_cursor) :].strip() - def unicode_name_matches(self, text): - u"""Match Latex-like syntax for unicode characters base + if continuation.startswith(closing_quote): + # do not close if already closed, e.g. `d['a'` + continuation = continuation[len(closing_quote) :] + else: + can_close_quote = True + + continuation = continuation.strip() + + # e.g. `pandas.DataFrame` has different tuple indexer behaviour, + # handling it is out of scope, so let's avoid appending suffixes. + has_known_tuple_handling = isinstance(obj, dict) + + can_close_bracket = ( + not continuation.startswith("]") and self.auto_close_dict_keys + ) + can_close_tuple_item = ( + not continuation.startswith(",") + and has_known_tuple_handling + and self.auto_close_dict_keys + ) + can_close_quote = can_close_quote and self.auto_close_dict_keys + + # fast path if closing qoute should be appended but not suffix is allowed + if not can_close_quote and not can_close_bracket and closing_quote: + return [leading + k for k in matches] + + results = [] + + end_of_tuple_or_item = _DictKeyState.END_OF_TUPLE | _DictKeyState.END_OF_ITEM + + for k, state_flag in matches.items(): + result = leading + k + if can_close_quote and closing_quote: + result += closing_quote + + if state_flag == end_of_tuple_or_item: + # We do not know which suffix to add, + # e.g. both tuple item and string + # match this item. + pass + + if state_flag in end_of_tuple_or_item and can_close_bracket: + result += "]" + if state_flag == _DictKeyState.IN_TUPLE and can_close_tuple_item: + result += ", " + results.append(result) + return results + + @context_matcher() + def unicode_name_matcher(self, context: CompletionContext): + """Same as :any:`unicode_name_matches`, but adopted to new Matcher API.""" + fragment, matches = self.unicode_name_matches(context.text_until_cursor) + return _convert_matcher_v1_result_to_v2( + matches, type="unicode", fragment=fragment, suppress_if_matches=True + ) + + @staticmethod + def unicode_name_matches(text: str) -> Tuple[str, List[str]]: + """Match Latex-like syntax for unicode characters base on the name of the character. This does ``\\GREEK SMALL LETTER ETA`` -> ``η`` Works only on valid python 3 identifier, or on combining characters that will combine to form a valid identifier. - - Used on Python 3 only. """ slashpos = text.rfind('\\') if slashpos > -1: @@ -1688,15 +2637,26 @@ def unicode_name_matches(self, text): return '\\'+s,[unic] except KeyError: pass - return u'', [] + return '', [] + @context_matcher() + def latex_name_matcher(self, context: CompletionContext): + """Match Latex syntax for unicode characters. + + This does both ``\\alp`` -> ``\\alpha`` and ``\\alpha`` -> ``α`` + """ + fragment, matches = self.latex_matches(context.text_until_cursor) + return _convert_matcher_v1_result_to_v2( + matches, type="latex", fragment=fragment, suppress_if_matches=True + ) - def latex_matches(self, text): - u"""Match Latex syntax for unicode characters. + def latex_matches(self, text: str) -> Tuple[str, Sequence[str]]: + """Match Latex syntax for unicode characters. This does both ``\\alp`` -> ``\\alpha`` and ``\\alpha`` -> ``α`` - Used on Python 3 only. + .. deprecated:: 8.6 + You can use :meth:`latex_name_matcher` instead. """ slashpos = text.rfind('\\') if slashpos > -1: @@ -1709,10 +2669,29 @@ def latex_matches(self, text): # If a user has partially typed a latex symbol, give them # a full list of options \al -> [\aleph, \alpha] matches = [k for k in latex_symbols if k.startswith(s)] - return s, matches - return u'', [] + if matches: + return s, matches + return '', () + + @context_matcher() + def custom_completer_matcher(self, context): + """Dispatch custom completer. + + If a match is found, suppresses all other matchers except for Jedi. + """ + matches = self.dispatch_custom_completer(context.token) or [] + result = _convert_matcher_v1_result_to_v2( + matches, type=_UNKNOWN_TYPE, suppress_if_matches=True + ) + result["ordered"] = True + result["do_not_suppress"] = {_get_matcher_id(self._jedi_matcher)} + return result def dispatch_custom_completer(self, text): + """ + .. deprecated:: 8.6 + You can use :meth:`custom_completer_matcher` instead. + """ if not self.custom_completers: return @@ -1764,25 +2743,27 @@ def completions(self, text: str, offset: int)->Iterator[Completion]: """ Returns an iterator over the possible completions - .. warning:: Unstable + .. warning:: + + Unstable This function is unstable, API may change without warning. It will also raise unless use in proper context manager. Parameters ---------- - - text:str + text : str Full text of the current input, multi line string. - offset:int + offset : int Integer representing the position of the cursor in ``text``. Offset is 0-based indexed. Yields ------ - :any:`Completion` object - + Completion + Notes + ----- The cursor on a text can either be seen as being "in between" characters or "On" a character depending on the interface visible to the user. For consistency the cursor being on "in between" characters X @@ -1792,7 +2773,6 @@ def completions(self, text: str, offset: int)->Iterator[Completion]: Combining characters may span more that one position in the text. - .. note:: If ``IPCompleter.debug`` is :any:`True` will yield a ``--jedi/ipython--`` @@ -1811,7 +2791,15 @@ def completions(self, text: str, offset: int)->Iterator[Completion]: category=ProvisionalCompleterWarning, stacklevel=2) seen = set() + profiler:Optional[cProfile.Profile] try: + if self.profile_completions: + import cProfile + profiler = cProfile.Profile() + profiler.enable() + else: + profiler = None + for c in self._completions(text, offset, _timeout=self.jedi_compute_type_timeout/1000): if c and (c in seen): continue @@ -1821,13 +2809,19 @@ def completions(self, text: str, offset: int)->Iterator[Completion]: """if completions take too long and users send keyboard interrupt, do not crash and return ASAP. """ pass - - def _completions(self, full_text: str, offset: int, *, _timeout)->Iterator[Completion]: + finally: + if profiler is not None: + profiler.disable() + ensure_dir_exists(self.profiler_output_dir) + output_path = os.path.join(self.profiler_output_dir, str(uuid.uuid4())) + print("Writing profiler output to", output_path) + profiler.dump_stats(output_path) + + def _completions(self, full_text: str, offset: int, *, _timeout) -> Iterator[Completion]: """ Core completion module.Same signature as :any:`completions`, with the extra `timeout` parameter (in seconds). - Computing jedi's completion ``.type`` can be quite expensive (it is a lazy property) and can require some warm-up, more warm up than just computing the ``name`` of a completion. The warm-up can be : @@ -1851,12 +2845,31 @@ def _completions(self, full_text: str, offset: int, *, _timeout)->Iterator[Compl """ deadline = time.monotonic() + _timeout - before = full_text[:offset] cursor_line, cursor_column = position_to_cursor(full_text, offset) - matched_text, matches, matches_origin, jedi_matches = self._complete( - full_text=full_text, cursor_line=cursor_line, cursor_pos=cursor_column) + jedi_matcher_id = _get_matcher_id(self._jedi_matcher) + + def is_non_jedi_result( + result: MatcherResult, identifier: str + ) -> TypeGuard[SimpleMatcherResult]: + return identifier != jedi_matcher_id + + results = self._complete( + full_text=full_text, cursor_line=cursor_line, cursor_pos=cursor_column + ) + + non_jedi_results: Dict[str, SimpleMatcherResult] = { + identifier: result + for identifier, result in results.items() + if is_non_jedi_result(result, identifier) + } + + jedi_matches = ( + cast(_JediMatcherResult, results[jedi_matcher_id])["completions"] + if jedi_matcher_id in results + else () + ) iter_jm = iter(jedi_matches) if _timeout: @@ -1884,30 +2897,59 @@ def _completions(self, full_text: str, offset: int, *, _timeout)->Iterator[Compl for jm in iter_jm: delta = len(jm.name_with_symbols) - len(jm.complete) - yield Completion(start=offset - delta, - end=offset, - text=jm.name_with_symbols, - type='', # don't compute type for speed - _origin='jedi', - signature='') - - - start_offset = before.rfind(matched_text) + yield Completion( + start=offset - delta, + end=offset, + text=jm.name_with_symbols, + type=_UNKNOWN_TYPE, # don't compute type for speed + _origin="jedi", + signature="", + ) # TODO: # Suppress this, right now just for debug. - if jedi_matches and matches and self.debug: - yield Completion(start=start_offset, end=offset, text='--jedi/ipython--', - _origin='debug', type='none', signature='') + if jedi_matches and non_jedi_results and self.debug: + some_start_offset = before.rfind( + next(iter(non_jedi_results.values()))["matched_fragment"] + ) + yield Completion( + start=some_start_offset, + end=offset, + text="--jedi/ipython--", + _origin="debug", + type="none", + signature="", + ) - # I'm unsure if this is always true, so let's assert and see if it - # crash - assert before.endswith(matched_text) - for m, t in zip(matches, matches_origin): - yield Completion(start=start_offset, end=offset, text=m, _origin=t, signature='', type='') + ordered: List[Completion] = [] + sortable: List[Completion] = [] + + for origin, result in non_jedi_results.items(): + matched_text = result["matched_fragment"] + start_offset = before.rfind(matched_text) + is_ordered = result.get("ordered", False) + container = ordered if is_ordered else sortable + + # I'm unsure if this is always true, so let's assert and see if it + # crash + assert before.endswith(matched_text) + + for simple_completion in result["completions"]: + completion = Completion( + start=start_offset, + end=offset, + text=simple_completion.text, + _origin=origin, + signature="", + type=simple_completion.type or _UNKNOWN_TYPE, + ) + container.append(completion) + yield from list(self._deduplicate(ordered + self._sort(sortable)))[ + :MATCHES_LIMIT + ] - def complete(self, text=None, line_buffer=None, cursor_pos=None): + def complete(self, text=None, line_buffer=None, cursor_pos=None) -> Tuple[str, Sequence[str]]: """Find completions for the given text and line context. Note that both the text and the line_buffer are optional, but at least @@ -1915,57 +2957,123 @@ def complete(self, text=None, line_buffer=None, cursor_pos=None): Parameters ---------- - text : string, optional + text : string, optional Text to perform the completion on. If not given, the line buffer is split using the instance's CompletionSplitter object. - - line_buffer : string, optional + line_buffer : string, optional If not given, the completer attempts to obtain the current line buffer via readline. This keyword allows clients which are requesting for text completions in non-readline contexts to inform the completer of the entire text. - - cursor_pos : int, optional + cursor_pos : int, optional Index of the cursor in the full line buffer. Should be provided by remote frontends where kernel has no access to frontend state. Returns ------- + Tuple of two items: text : str - Text that was actually used in the completion. - + Text that was actually used in the completion. matches : list - A list of completion matches. - - - .. note:: + A list of completion matches. + Notes + ----- This API is likely to be deprecated and replaced by :any:`IPCompleter.completions` in the future. - """ warnings.warn('`Completer.complete` is pending deprecation since ' 'IPython 6.0 and will be replaced by `Completer.completions`.', PendingDeprecationWarning) # potential todo, FOLD the 3rd throw away argument of _complete # into the first 2 one. - return self._complete(line_buffer=line_buffer, cursor_pos=cursor_pos, text=text, cursor_line=0)[:2] + # TODO: Q: does the above refer to jedi completions (i.e. 0-indexed?) + # TODO: should we deprecate now, or does it stay? + + results = self._complete( + line_buffer=line_buffer, cursor_pos=cursor_pos, text=text, cursor_line=0 + ) + + jedi_matcher_id = _get_matcher_id(self._jedi_matcher) + + return self._arrange_and_extract( + results, + # TODO: can we confirm that excluding Jedi here was a deliberate choice in previous version? + skip_matchers={jedi_matcher_id}, + # this API does not support different start/end positions (fragments of token). + abort_if_offset_changes=True, + ) + + def _arrange_and_extract( + self, + results: Dict[str, MatcherResult], + skip_matchers: Set[str], + abort_if_offset_changes: bool, + ): + + sortable: List[AnyMatcherCompletion] = [] + ordered: List[AnyMatcherCompletion] = [] + most_recent_fragment = None + for identifier, result in results.items(): + if identifier in skip_matchers: + continue + if not result["completions"]: + continue + if not most_recent_fragment: + most_recent_fragment = result["matched_fragment"] + if ( + abort_if_offset_changes + and result["matched_fragment"] != most_recent_fragment + ): + break + if result.get("ordered", False): + ordered.extend(result["completions"]) + else: + sortable.extend(result["completions"]) + + if not most_recent_fragment: + most_recent_fragment = "" # to satisfy typechecker (and just in case) + + return most_recent_fragment, [ + m.text for m in self._deduplicate(ordered + self._sort(sortable)) + ] def _complete(self, *, cursor_line, cursor_pos, line_buffer=None, text=None, - full_text=None) -> Tuple[str, List[str], List[str], Iterable[_FakeJediCompletion]]: + full_text=None) -> _CompleteResult: """ - Like complete but can also returns raw jedi completions as well as the origin of the completion text. This could (and should) be made much cleaner but that will be simpler once we drop the old (and stateful) :any:`complete` API. - With current provisional API, cursor_pos act both (depending on the caller) as the offset in the ``text`` or ``line_buffer``, or as the ``column`` when passing multiline strings this could/should be renamed but would add extra noise. + + Parameters + ---------- + cursor_line + Index of the line the cursor is on. 0 indexed. + cursor_pos + Position of the cursor in the current line/line_buffer/text. 0 + indexed. + line_buffer : optional, str + The current line the cursor is in, this is mostly due to legacy + reason that readline could only give a us the single current line. + Prefer `full_text`. + text : str + The current "token" the cursor is in, mostly also for historical + reasons. as the completer would trigger only after the current line + was parsed. + full_text : str + Full text of the current cell. + + Returns + ------- + An ordered dictionary where keys are identifiers of completion + matchers and values are ``MatcherResult``s. """ # if the cursor position isn't given, the only sane assumption we can @@ -1979,111 +3087,236 @@ def _complete(self, *, cursor_line, cursor_pos, line_buffer=None, text=None, # if text is either None or an empty string, rely on the line buffer if (not line_buffer) and full_text: line_buffer = full_text.split('\n')[cursor_line] - if not text: - text = self.splitter.split_line(line_buffer, cursor_pos) - - if self.backslash_combining_completions: - # allow deactivation of these on windows. - base_text = text if not line_buffer else line_buffer[:cursor_pos] - latex_text, latex_matches = self.latex_matches(base_text) - if latex_matches: - return latex_text, latex_matches, ['latex_matches']*len(latex_matches), () - name_text = '' - name_matches = [] - # need to add self.fwd_unicode_match() function here when done - for meth in (self.unicode_name_matches, back_latex_name_matches, back_unicode_name_matches, self.fwd_unicode_match): - name_text, name_matches = meth(base_text) - if name_text: - return name_text, name_matches[:MATCHES_LIMIT], \ - [meth.__qualname__]*min(len(name_matches), MATCHES_LIMIT), () - + if not text: # issue #11508: check line_buffer before calling split_line + text = ( + self.splitter.split_line(line_buffer, cursor_pos) if line_buffer else "" + ) # If no line buffer is given, assume the input text is all there was if line_buffer is None: line_buffer = text + # deprecated - do not use `line_buffer` in new code. self.line_buffer = line_buffer self.text_until_cursor = self.line_buffer[:cursor_pos] - # Do magic arg matches - for matcher in self.magic_arg_matchers: - matches = list(matcher(line_buffer))[:MATCHES_LIMIT] - if matches: - origins = [matcher.__qualname__] * len(matches) - return text, matches, origins, () + if not full_text: + full_text = line_buffer + + context = CompletionContext( + full_text=full_text, + cursor_position=cursor_pos, + cursor_line=cursor_line, + token=text, + limit=MATCHES_LIMIT, + ) # Start with a clean slate of completions - matches = [] - - # FIXME: we should extend our api to return a dict with completions for - # different types of objects. The rlcomplete() method could then - # simply collapse the dict into a list for readline, but we'd have - # richer completion semantics in other environments. - completions = () - if self.use_jedi: - if not full_text: - full_text = line_buffer - completions = self._jedi_matches( - cursor_pos, cursor_line, full_text) - - if self.merge_completions: - matches = [] - for matcher in self.matchers: - try: - matches.extend([(m, matcher.__qualname__) - for m in matcher(text)]) - except: - # Show the ugly traceback if the matcher causes an - # exception, but do NOT crash the kernel! - sys.excepthook(*sys.exc_info()) - else: - for matcher in self.matchers: - matches = [(m, matcher.__qualname__) - for m in matcher(text)] - if matches: - break - - seen = set() - filtered_matches = set() - for m in matches: - t, c = m - if t not in seen: - filtered_matches.add(m) - seen.add(t) - - _filtered_matches = sorted(filtered_matches, key=lambda x: completions_sorting_key(x[0])) - - custom_res = [(m, 'custom') for m in self.dispatch_custom_completer(text) or []] - - _filtered_matches = custom_res or _filtered_matches - - _filtered_matches = _filtered_matches[:MATCHES_LIMIT] - _matches = [m[0] for m in _filtered_matches] - origins = [m[1] for m in _filtered_matches] - - self.matches = _matches - - return text, _matches, origins, completions - - def fwd_unicode_match(self, text:str) -> Tuple[str, list]: - if self._names is None: - self._names = [] - for c in range(0,0x10FFFF + 1): - try: - self._names.append(unicodedata.name(chr(c))) - except ValueError: - pass + results: Dict[str, MatcherResult] = {} + + jedi_matcher_id = _get_matcher_id(self._jedi_matcher) + + suppressed_matchers: Set[str] = set() + + matchers = { + _get_matcher_id(matcher): matcher + for matcher in sorted( + self.matchers, key=_get_matcher_priority, reverse=True + ) + } + + for matcher_id, matcher in matchers.items(): + matcher_id = _get_matcher_id(matcher) + + if matcher_id in self.disable_matchers: + continue + + if matcher_id in results: + warnings.warn(f"Duplicate matcher ID: {matcher_id}.") + + if matcher_id in suppressed_matchers: + continue + + result: MatcherResult + try: + if _is_matcher_v1(matcher): + result = _convert_matcher_v1_result_to_v2( + matcher(text), type=_UNKNOWN_TYPE + ) + elif _is_matcher_v2(matcher): + result = matcher(context) + else: + api_version = _get_matcher_api_version(matcher) + raise ValueError(f"Unsupported API version {api_version}") + except: + # Show the ugly traceback if the matcher causes an + # exception, but do NOT crash the kernel! + sys.excepthook(*sys.exc_info()) + continue + + # set default value for matched fragment if suffix was not selected. + result["matched_fragment"] = result.get("matched_fragment", context.token) + + if not suppressed_matchers: + suppression_recommended: Union[bool, Set[str]] = result.get( + "suppress", False + ) + + suppression_config = ( + self.suppress_competing_matchers.get(matcher_id, None) + if isinstance(self.suppress_competing_matchers, dict) + else self.suppress_competing_matchers + ) + should_suppress = ( + (suppression_config is True) + or (suppression_recommended and (suppression_config is not False)) + ) and has_any_completions(result) + + if should_suppress: + suppression_exceptions: Set[str] = result.get( + "do_not_suppress", set() + ) + if isinstance(suppression_recommended, Iterable): + to_suppress = set(suppression_recommended) + else: + to_suppress = set(matchers) + suppressed_matchers = to_suppress - suppression_exceptions + + new_results = {} + for previous_matcher_id, previous_result in results.items(): + if previous_matcher_id not in suppressed_matchers: + new_results[previous_matcher_id] = previous_result + results = new_results + + results[matcher_id] = result + + _, matches = self._arrange_and_extract( + results, + # TODO Jedi completions non included in legacy stateful API; was this deliberate or omission? + # if it was omission, we can remove the filtering step, otherwise remove this comment. + skip_matchers={jedi_matcher_id}, + abort_if_offset_changes=False, + ) + + # populate legacy stateful API + self.matches = matches + + return results + + @staticmethod + def _deduplicate( + matches: Sequence[AnyCompletion], + ) -> Iterable[AnyCompletion]: + filtered_matches: Dict[str, AnyCompletion] = {} + for match in matches: + text = match.text + if ( + text not in filtered_matches + or filtered_matches[text].type == _UNKNOWN_TYPE + ): + filtered_matches[text] = match + + return filtered_matches.values() + + @staticmethod + def _sort(matches: Sequence[AnyCompletion]): + return sorted(matches, key=lambda x: completions_sorting_key(x.text)) + + @context_matcher() + def fwd_unicode_matcher(self, context: CompletionContext): + """Same as :any:`fwd_unicode_match`, but adopted to new Matcher API.""" + # TODO: use `context.limit` to terminate early once we matched the maximum + # number that will be used downstream; can be added as an optional to + # `fwd_unicode_match(text: str, limit: int = None)` or we could re-implement here. + fragment, matches = self.fwd_unicode_match(context.text_until_cursor) + return _convert_matcher_v1_result_to_v2( + matches, type="unicode", fragment=fragment, suppress_if_matches=True + ) + + def fwd_unicode_match(self, text: str) -> Tuple[str, Sequence[str]]: + """ + Forward match a string starting with a backslash with a list of + potential Unicode completions. + + Will compute list of Unicode character names on first call and cache it. + + .. deprecated:: 8.6 + You can use :meth:`fwd_unicode_matcher` instead. + + Returns + ------- + At tuple with: + - matched text (empty if no matches) + - list of potential completions, empty tuple otherwise) + """ + # TODO: self.unicode_names is here a list we traverse each time with ~100k elements. + # We could do a faster match using a Trie. + + # Using pygtrie the following seem to work: + + # s = PrefixSet() + + # for c in range(0,0x10FFFF + 1): + # try: + # s.add(unicodedata.name(chr(c))) + # except ValueError: + # pass + # [''.join(k) for k in s.iter(prefix)] + + # But need to be timed and adds an extra dependency. slashpos = text.rfind('\\') # if text starts with slash if slashpos > -1: - s = text[slashpos+1:] - candidates = [x for x in self._names if x.startswith(s)] + # PERF: It's important that we don't access self._unicode_names + # until we're inside this if-block. _unicode_names is lazily + # initialized, and it takes a user-noticeable amount of time to + # initialize it, so we don't want to initialize it unless we're + # actually going to use it. + s = text[slashpos + 1 :] + sup = s.upper() + candidates = [x for x in self.unicode_names if x.startswith(sup)] + if candidates: + return s, candidates + candidates = [x for x in self.unicode_names if sup in x] + if candidates: + return s, candidates + splitsup = sup.split(" ") + candidates = [ + x for x in self.unicode_names if all(u in x for u in splitsup) + ] if candidates: return s, candidates - else: - return '', () + + return "", () # if text does not start with slash else: - return u'', () + return '', () + + @property + def unicode_names(self) -> List[str]: + """List of names of unicode code points that can be completed. + + The list is lazily initialized on first access. + """ + if self._unicode_names is None: + names = [] + for c in range(0,0x10FFFF + 1): + try: + names.append(unicodedata.name(chr(c))) + except ValueError: + pass + self._unicode_names = _unicode_name_compute(_UNICODE_RANGES) + + return self._unicode_names + +def _unicode_name_compute(ranges:List[Tuple[int,int]]) -> List[str]: + names = [] + for start,stop in ranges: + for c in range(start, stop) : + try: + names.append(unicodedata.name(chr(c))) + except ValueError: + pass + return names diff --git a/IPython/core/completerlib.py b/IPython/core/completerlib.py index 9e592b0817e..0ca97e7b7ff 100644 --- a/IPython/core/completerlib.py +++ b/IPython/core/completerlib.py @@ -52,7 +52,7 @@ TIMEOUT_GIVEUP = 20 # Regular expression for the python import statement -import_re = re.compile(r'(?P[a-zA-Z_][a-zA-Z0-9_]*?)' +import_re = re.compile(r'(?P[^\W\d]\w*?)' r'(?P[/\\]__init__)?' r'(?P%s)$' % r'|'.join(re.escape(s) for s in _suffixes)) @@ -154,6 +154,17 @@ def is_importable(module, attr, only_modules): else: return not(attr[:2] == '__' and attr[-2:] == '__') +def is_possible_submodule(module, attr): + try: + obj = getattr(module, attr) + except AttributeError: + # Is possilby an unimported submodule + return True + except TypeError: + # https://github.com/ipython/ipython/issues/9678 + return False + return inspect.ismodule(obj) + def try_import(mod: str, only_modules=False) -> List[str]: """ @@ -172,7 +183,12 @@ def try_import(mod: str, only_modules=False) -> List[str]: completions.extend( [attr for attr in dir(m) if is_importable(m, attr, only_modules)]) - completions.extend(getattr(m, '__all__', [])) + m_all = getattr(m, "__all__", []) + if only_modules: + completions.extend(attr for attr in m_all if is_possible_submodule(m, attr)) + else: + completions.extend(m_all) + if m_is_init: completions.extend(module_list(os.path.dirname(m.__file__))) completions_set = {c for c in completions if isinstance(c, str)} diff --git a/IPython/core/crashhandler.py b/IPython/core/crashhandler.py index 1e0b429d09a..f60a75bbc5b 100644 --- a/IPython/core/crashhandler.py +++ b/IPython/core/crashhandler.py @@ -19,10 +19,10 @@ # Imports #----------------------------------------------------------------------------- -import os import sys import traceback from pprint import pformat +from pathlib import Path from IPython.core import ultratb from IPython.core.release import author_email @@ -31,6 +31,8 @@ from IPython.core.release import __version__ as version +from typing import Optional + #----------------------------------------------------------------------------- # Code #----------------------------------------------------------------------------- @@ -94,34 +96,40 @@ def __call__(self, etype, evalue, etb) message_template = _default_message_template section_sep = '\n\n'+'*'*75+'\n\n' - def __init__(self, app, contact_name=None, contact_email=None, - bug_tracker=None, show_crash_traceback=True, call_pdb=False): + def __init__( + self, + app, + contact_name: Optional[str] = None, + contact_email: Optional[str] = None, + bug_tracker: Optional[str] = None, + show_crash_traceback: bool = True, + call_pdb: bool = False, + ): """Create a new crash handler Parameters ---------- - app : Application + app : Application A running :class:`Application` instance, which will be queried at crash time for internal information. - contact_name : str A string with the name of the person to contact. - contact_email : str A string with the email address of the contact. - bug_tracker : str A string with the URL for your project's bug tracker. - show_crash_traceback : bool If false, don't print the crash traceback on stderr, only generate the on-disk report + call_pdb + Whether to call pdb on crash - Non-argument instance attributes: - + Attributes + ---------- These instances contain some non-argument attributes which allow for further customization of the crash handler's behavior. Please see the source for further details. + """ self.crash_report_fname = "Crash_report_%s.txt" % app.name self.app = app @@ -151,10 +159,10 @@ def __call__(self, etype, evalue, etb): try: rptdir = self.app.ipython_dir except: - rptdir = os.getcwd() - if rptdir is None or not os.path.isdir(rptdir): - rptdir = os.getcwd() - report_name = os.path.join(rptdir,self.crash_report_fname) + rptdir = Path.cwd() + if rptdir is None or not Path.is_dir(rptdir): + rptdir = Path.cwd() + report_name = rptdir / self.crash_report_fname # write the report filename into the instance dict so it can get # properly expanded out in the user message template self.crash_report_fname = report_name @@ -176,7 +184,7 @@ def __call__(self, etype, evalue, etb): # and generate a complete report on disk try: - report = open(report_name,'w') + report = open(report_name, "w", encoding="utf-8") except: print('Could not create crash report on disk.', file=sys.stderr) return diff --git a/IPython/core/debugger.py b/IPython/core/debugger.py index ebb8dcac0d8..73b0328743b 100644 --- a/IPython/core/debugger.py +++ b/IPython/core/debugger.py @@ -2,6 +2,76 @@ """ Pdb debugger class. + +This is an extension to PDB which adds a number of new features. +Note that there is also the `IPython.terminal.debugger` class which provides UI +improvements. + +We also strongly recommend to use this via the `ipdb` package, which provides +extra configuration options. + +Among other things, this subclass of PDB: + - supports many IPython magics like pdef/psource + - hide frames in tracebacks based on `__tracebackhide__` + - allows to skip frames based on `__debuggerskip__` + +The skipping and hiding frames are configurable via the `skip_predicates` +command. + +By default, frames from readonly files will be hidden, frames containing +``__tracebackhide__=True`` will be hidden. + +Frames containing ``__debuggerskip__`` will be stepped over, frames who's parent +frames value of ``__debuggerskip__`` is ``True`` will be skipped. + + >>> def helpers_helper(): + ... pass + ... + ... def helper_1(): + ... print("don't step in me") + ... helpers_helpers() # will be stepped over unless breakpoint set. + ... + ... + ... def helper_2(): + ... print("in me neither") + ... + +One can define a decorator that wraps a function between the two helpers: + + >>> def pdb_skipped_decorator(function): + ... + ... + ... def wrapped_fn(*args, **kwargs): + ... __debuggerskip__ = True + ... helper_1() + ... __debuggerskip__ = False + ... result = function(*args, **kwargs) + ... __debuggerskip__ = True + ... helper_2() + ... # setting __debuggerskip__ to False again is not necessary + ... return result + ... + ... return wrapped_fn + +When decorating a function, ipdb will directly step into ``bar()`` by +default: + + >>> @foo_decorator + ... def bar(x, y): + ... return x * y + + +You can toggle the behavior with + + ipdb> skip_predicates debuggerskip false + +or configure it in your ``.pdbrc`` + + + +License +------- + Modified from the standard pdb.Pdb class to avoid including readline, so that the command line completion of other programs which include this isn't damaged. @@ -9,11 +79,16 @@ In the future, this class will be expanded with improvements over the standard pdb. -The code in this file is mainly lifted out of cmd.py in Python 2.2, with minor -changes. Licensing should therefore be under the standard Python terms. For -details on the PSF (Python Software Foundation) standard license, see: +The original code in this file is mainly lifted out of cmd.py in Python 2.2, +with minor changes. Licensing should therefore be under the standard Python +terms. For details on the PSF (Python Software Foundation) standard license, +see: https://docs.python.org/2/license.html + + +All the changes since then are under the same license as IPython. + """ #***************************************************************************** @@ -26,30 +101,32 @@ # #***************************************************************************** -import bdb -import functools import inspect import linecache import sys -import warnings import re +import os from IPython import get_ipython from IPython.utils import PyColorize from IPython.utils import coloransi, py3compat from IPython.core.excolors import exception_colors -from IPython.testing.skipdoctest import skip_doctest +# skip module docstests +__skip_doctest__ = True prompt = 'ipdb> ' -#We have to check this directly from sys.argv, config struct not yet available +# We have to check this directly from sys.argv, config struct not yet available from pdb import Pdb as OldPdb # Allow the set_trace code to operate outside of an ipython instance, even if # it does so with some limitations. The rest of this support is implemented in # the Tracer constructor. +DEBUGGERSKIP = "__debuggerskip__" + + def make_arrow(pad): """generate the leading arrow in front of traceback or debugger""" if pad >= 2: @@ -65,112 +142,15 @@ def BdbQuit_excepthook(et, ev, tb, excepthook=None): All other exceptions are processed using the `excepthook` parameter. """ - warnings.warn("`BdbQuit_excepthook` is deprecated since version 5.1", - DeprecationWarning, stacklevel=2) - if et==bdb.BdbQuit: - print('Exiting Debugger.') - elif excepthook is not None: - excepthook(et, ev, tb) - else: - # Backwards compatibility. Raise deprecation warning? - BdbQuit_excepthook.excepthook_ori(et,ev,tb) - - -def BdbQuit_IPython_excepthook(self,et,ev,tb,tb_offset=None): - warnings.warn( - "`BdbQuit_IPython_excepthook` is deprecated since version 5.1", - DeprecationWarning, stacklevel=2) - print('Exiting Debugger.') - - -class Tracer(object): - """ - DEPRECATED - - Class for local debugging, similar to pdb.set_trace. - - Instances of this class, when called, behave like pdb.set_trace, but - providing IPython's enhanced capabilities. - - This is implemented as a class which must be initialized in your own code - and not as a standalone function because we need to detect at runtime - whether IPython is already active or not. That detection is done in the - constructor, ensuring that this code plays nicely with a running IPython, - while functioning acceptably (though with limitations) if outside of it. - """ - - @skip_doctest - def __init__(self, colors=None): - """ - DEPRECATED - - Create a local debugger instance. - - Parameters - ---------- - - colors : str, optional - The name of the color scheme to use, it must be one of IPython's - valid color schemes. If not given, the function will default to - the current IPython scheme when running inside IPython, and to - 'NoColor' otherwise. - - Examples - -------- - :: + raise ValueError( + "`BdbQuit_excepthook` is deprecated since version 5.1", + ) - from IPython.core.debugger import Tracer; debug_here = Tracer() - Later in your code:: - - debug_here() # -> will open up the debugger at that point. - - Once the debugger activates, you can use all of its regular commands to - step through code, set breakpoints, etc. See the pdb documentation - from the Python standard library for usage details. - """ - warnings.warn("`Tracer` is deprecated since version 5.1, directly use " - "`IPython.core.debugger.Pdb.set_trace()`", - DeprecationWarning, stacklevel=2) - - ip = get_ipython() - if ip is None: - # Outside of ipython, we set our own exception hook manually - sys.excepthook = functools.partial(BdbQuit_excepthook, - excepthook=sys.excepthook) - def_colors = 'NoColor' - else: - # In ipython, we use its custom exception handler mechanism - def_colors = ip.colors - ip.set_custom_exc((bdb.BdbQuit,), BdbQuit_IPython_excepthook) - - if colors is None: - colors = def_colors - - # The stdlib debugger internally uses a modified repr from the `repr` - # module, that limits the length of printed strings to a hardcoded - # limit of 30 characters. That much trimming is too aggressive, let's - # at least raise that limit to 80 chars, which should be enough for - # most interactive uses. - try: - from reprlib import aRepr - aRepr.maxstring = 80 - except: - # This is only a user-facing convenience, so any error we encounter - # here can be warned about but can be otherwise ignored. These - # printouts will tell us about problems if this API changes - import traceback - traceback.print_exc() - - self.debugger = Pdb(colors) - - def __call__(self): - """Starts an interactive debugger at the point where called. - - This is similar to the pdb.set_trace() function from the std lib, but - using IPython's enhanced debugger.""" - - self.debugger.set_trace(sys._getframe().f_back) +def BdbQuit_IPython_excepthook(self, et, ev, tb, tb_offset=None): + raise ValueError( + "`BdbQuit_IPython_excepthook` is deprecated since version 5.1", + DeprecationWarning, stacklevel=2) RGX_EXTRA_INDENT = re.compile(r'(?<=\n)\s+') @@ -198,21 +178,41 @@ class Pdb(OldPdb): for a standalone version that uses prompt_toolkit, see `IPython.terminal.debugger.TerminalPdb` and `IPython.terminal.debugger.set_trace()` + + + This debugger can hide and skip frames that are tagged according to some predicates. + See the `skip_predicates` commands. + """ - def __init__(self, color_scheme=None, completekey=None, - stdin=None, stdout=None, context=5, **kwargs): + default_predicates = { + "tbhide": True, + "readonly": False, + "ipython_internal": True, + "debuggerskip": True, + } + + def __init__(self, completekey=None, stdin=None, stdout=None, context=5, **kwargs): """Create a new IPython debugger. - - :param color_scheme: Deprecated, do not use. - :param completekey: Passed to pdb.Pdb. - :param stdin: Passed to pdb.Pdb. - :param stdout: Passed to pdb.Pdb. - :param context: Number of lines of source code context to show when + + Parameters + ---------- + completekey : default None + Passed to pdb.Pdb. + stdin : default None + Passed to pdb.Pdb. + stdout : default None + Passed to pdb.Pdb. + context : int + Number of lines of source code context to show when displaying stacktrace information. - :param kwargs: Passed to pdb.Pdb. - The possibilities are python version dependent, see the python - docs for more info. + **kwargs + Passed to pdb.Pdb. + + Notes + ----- + The possibilities are python version dependent, see the python + docs for more info. """ # Parent constructor: @@ -220,8 +220,8 @@ def __init__(self, color_scheme=None, completekey=None, self.context = int(context) if self.context <= 0: raise ValueError("Context must be a positive integer") - except (TypeError, ValueError): - raise ValueError("Context must be a positive integer") + except (TypeError, ValueError) as e: + raise ValueError("Context must be a positive integer") from e # `kwargs` ensures full compatibility with stdlib's `pdb.Pdb`. OldPdb.__init__(self, completekey, stdin, stdout, **kwargs) @@ -237,14 +237,10 @@ def __init__(self, color_scheme=None, completekey=None, self.shell = TerminalInteractiveShell.instance() # needed by any code which calls __import__("__main__") after # the debugger was entered. See also #9941. - sys.modules['__main__'] = save_main + sys.modules["__main__"] = save_main - if color_scheme is not None: - warnings.warn( - "The `color_scheme` argument is deprecated since version 5.1", - DeprecationWarning, stacklevel=2) - else: - color_scheme = self.shell.colors + + color_scheme = self.shell.colors self.aliases = {} @@ -272,7 +268,6 @@ def __init__(self, color_scheme=None, completekey=None, cst['Neutral'].colors.breakpoint_enabled = C.LightRed cst['Neutral'].colors.breakpoint_disabled = C.Red - # Add a python parser so we can syntax highlight source while # debugging. self.parser = PyColorize.Parser(style=color_scheme) @@ -280,26 +275,78 @@ def __init__(self, color_scheme=None, completekey=None, # Set the prompt - the default prompt is '(Pdb)' self.prompt = prompt + self.skip_hidden = True + self.report_skipped = True + + # list of predicates we use to skip frames + self._predicates = self.default_predicates + # def set_colors(self, scheme): """Shorthand access to the color table scheme selector method.""" self.color_scheme_table.set_active_scheme(scheme) self.parser.style = scheme + def set_trace(self, frame=None): + if frame is None: + frame = sys._getframe().f_back + self.initial_frame = frame + return super().set_trace(frame) + + def _hidden_predicate(self, frame): + """ + Given a frame return whether it it should be hidden or not by IPython. + """ + + if self._predicates["readonly"]: + fname = frame.f_code.co_filename + # we need to check for file existence and interactively define + # function would otherwise appear as RO. + if os.path.isfile(fname) and not os.access(fname, os.W_OK): + return True + + if self._predicates["tbhide"]: + if frame in (self.curframe, getattr(self, "initial_frame", None)): + return False + frame_locals = self._get_frame_locals(frame) + if "__tracebackhide__" not in frame_locals: + return False + return frame_locals["__tracebackhide__"] + return False + + def hidden_frames(self, stack): + """ + Given an index in the stack return whether it should be skipped. + + This is used in up/down and where to skip frames. + """ + # The f_locals dictionary is updated from the actual frame + # locals whenever the .f_locals accessor is called, so we + # avoid calling it here to preserve self.curframe_locals. + # Furthermore, there is no good reason to hide the current frame. + ip_hide = [self._hidden_predicate(s[0]) for s in stack] + ip_start = [i for i, s in enumerate(ip_hide) if s == "__ipython_bottom__"] + if ip_start and self._predicates["ipython_internal"]: + ip_hide = [h if i > ip_start[0] else True for (i, h) in enumerate(ip_hide)] + return ip_hide + def interaction(self, frame, traceback): try: OldPdb.interaction(self, frame, traceback) except KeyboardInterrupt: - self.stdout.write('\n' + self.shell.get_exception_only()) + self.stdout.write("\n" + self.shell.get_exception_only()) - def new_do_up(self, arg): - OldPdb.do_up(self, arg) - do_u = do_up = decorate_fn_with_doc(new_do_up, OldPdb.do_up) + def precmd(self, line): + """Perform useful escapes on the command before it is executed.""" - def new_do_down(self, arg): - OldPdb.do_down(self, arg) + if line.endswith("??"): + line = "pinfo2 " + line[:-2] + elif line.endswith("?"): + line = "pinfo " + line[:-1] - do_d = do_down = decorate_fn_with_doc(new_do_down, OldPdb.do_down) + line = super().precmd(line) + + return line def new_do_frame(self, arg): OldPdb.do_frame(self, arg) @@ -307,7 +354,7 @@ def new_do_frame(self, arg): def new_do_quit(self, arg): if hasattr(self, 'old_all_completions'): - self.shell.Completer.all_completions=self.old_all_completions + self.shell.Completer.all_completions = self.old_all_completions return OldPdb.do_quit(self, arg) @@ -320,17 +367,32 @@ def new_do_restart(self, arg): return self.do_quit(arg) def print_stack_trace(self, context=None): + Colors = self.color_scheme_table.active_colors + ColorsNormal = Colors.Normal if context is None: context = self.context try: - context=int(context) + context = int(context) if context <= 0: raise ValueError("Context must be a positive integer") - except (TypeError, ValueError): - raise ValueError("Context must be a positive integer") + except (TypeError, ValueError) as e: + raise ValueError("Context must be a positive integer") from e try: - for frame_lineno in self.stack: + skipped = 0 + for hidden, frame_lineno in zip(self.hidden_frames(self.stack), self.stack): + if hidden and self.skip_hidden: + skipped += 1 + continue + if skipped: + print( + f"{Colors.excName} [... skipping {skipped} hidden frame(s)]{ColorsNormal}\n" + ) + skipped = 0 self.print_stack_entry(frame_lineno, context=context) + if skipped: + print( + f"{Colors.excName} [... skipping {skipped} hidden frame(s)]{ColorsNormal}\n" + ) except KeyboardInterrupt: pass @@ -339,11 +401,11 @@ def print_stack_entry(self, frame_lineno, prompt_prefix='\n-> ', if context is None: context = self.context try: - context=int(context) + context = int(context) if context <= 0: raise ValueError("Context must be a positive integer") - except (TypeError, ValueError): - raise ValueError("Context must be a positive integer") + except (TypeError, ValueError) as e: + raise ValueError("Context must be a positive integer") from e print(self.format_stack_entry(frame_lineno, '', context), file=self.stdout) # vds: >> @@ -352,37 +414,57 @@ def print_stack_entry(self, frame_lineno, prompt_prefix='\n-> ', self.shell.hooks.synchronize_with_editor(filename, lineno, 0) # vds: << + def _get_frame_locals(self, frame): + """ " + Accessing f_local of current frame reset the namespace, so we want to avoid + that or the following can happen + + ipdb> foo + "old" + ipdb> foo = "new" + ipdb> foo + "new" + ipdb> where + ipdb> foo + "old" + + So if frame is self.current_frame we instead return self.curframe_locals + + """ + if frame is self.curframe: + return self.curframe_locals + else: + return frame.f_locals + def format_stack_entry(self, frame_lineno, lprefix=': ', context=None): if context is None: context = self.context try: - context=int(context) + context = int(context) if context <= 0: print("Context must be a positive integer", file=self.stdout) except (TypeError, ValueError): print("Context must be a positive integer", file=self.stdout) - try: - import reprlib # Py 3 - except ImportError: - import repr as reprlib # Py 2 + + import reprlib ret = [] Colors = self.color_scheme_table.active_colors ColorsNormal = Colors.Normal - tpl_link = u'%s%%s%s' % (Colors.filenameEm, ColorsNormal) - tpl_call = u'%s%%s%s%%s%s' % (Colors.vName, Colors.valEm, ColorsNormal) - tpl_line = u'%%s%s%%s %s%%s' % (Colors.lineno, ColorsNormal) - tpl_line_em = u'%%s%s%%s %s%%s%s' % (Colors.linenoEm, Colors.line, - ColorsNormal) + tpl_link = "%s%%s%s" % (Colors.filenameEm, ColorsNormal) + tpl_call = "%s%%s%s%%s%s" % (Colors.vName, Colors.valEm, ColorsNormal) + tpl_line = "%%s%s%%s %s%%s" % (Colors.lineno, ColorsNormal) + tpl_line_em = "%%s%s%%s %s%%s%s" % (Colors.linenoEm, Colors.line, ColorsNormal) frame, lineno = frame_lineno return_value = '' - if '__return__' in frame.f_locals: - rv = frame.f_locals['__return__'] - #return_value += '->' - return_value += reprlib.repr(rv) + '\n' + loc_frame = self._get_frame_locals(frame) + if "__return__" in loc_frame: + rv = loc_frame["__return__"] + # return_value += '->' + return_value += reprlib.repr(rv) + "\n" ret.append(return_value) #s = filename + '(' + `lineno` + ')' @@ -394,10 +476,10 @@ def format_stack_entry(self, frame_lineno, lprefix=': ', context=None): else: func = "" - call = '' - if func != '?': - if '__args__' in frame.f_locals: - args = reprlib.repr(frame.f_locals['__args__']) + call = "" + if func != "?": + if "__args__" in loc_frame: + args = reprlib.repr(loc_frame["__args__"]) else: args = '()' call = tpl_call % (func, args) @@ -407,8 +489,8 @@ def format_stack_entry(self, frame_lineno, lprefix=': ', context=None): if frame is self.curframe: ret.append('> ') else: - ret.append(' ') - ret.append(u'%s(%s)%s\n' % (link,lineno,call)) + ret.append(" ") + ret.append("%s(%s)%s\n" % (link, lineno, call)) start = lineno - 1 - context//2 lines = linecache.getlines(filename) @@ -416,17 +498,17 @@ def format_stack_entry(self, frame_lineno, lprefix=': ', context=None): start = max(start, 0) lines = lines[start : start + context] - for i,line in enumerate(lines): - show_arrow = (start + 1 + i == lineno) - linetpl = (frame is self.curframe or show_arrow) \ - and tpl_line_em \ - or tpl_line - ret.append(self.__format_line(linetpl, filename, - start + 1 + i, line, - arrow = show_arrow) ) - return ''.join(ret) - - def __format_line(self, tpl_line, filename, lineno, line, arrow = False): + for i, line in enumerate(lines): + show_arrow = start + 1 + i == lineno + linetpl = (frame is self.curframe or show_arrow) and tpl_line_em or tpl_line + ret.append( + self.__format_line( + linetpl, filename, start + 1 + i, line, arrow=show_arrow + ) + ) + return "".join(ret) + + def __format_line(self, tpl_line, filename, lineno, line, arrow=False): bp_mark = "" bp_mark_color = "" @@ -456,7 +538,6 @@ def __format_line(self, tpl_line, filename, lineno, line, arrow = False): return tpl_line % (bp_mark_color + bp_mark, num, line) - def print_list_lines(self, filename, first, last): """The printing (as opposed to the parsing part of a 'list' command.""" @@ -475,9 +556,13 @@ def print_list_lines(self, filename, first, last): break if lineno == self.curframe.f_lineno: - line = self.__format_line(tpl_line_em, filename, lineno, line, arrow = True) + line = self.__format_line( + tpl_line_em, filename, lineno, line, arrow=True + ) else: - line = self.__format_line(tpl_line, filename, lineno, line, arrow = False) + line = self.__format_line( + tpl_line, filename, lineno, line, arrow=False + ) src.append(line) self.lineno = lineno @@ -487,6 +572,69 @@ def print_list_lines(self, filename, first, last): except KeyboardInterrupt: pass + def do_skip_predicates(self, args): + """ + Turn on/off individual predicates as to whether a frame should be hidden/skip. + + The global option to skip (or not) hidden frames is set with skip_hidden + + To change the value of a predicate + + skip_predicates key [true|false] + + Call without arguments to see the current values. + + To permanently change the value of an option add the corresponding + command to your ``~/.pdbrc`` file. If you are programmatically using the + Pdb instance you can also change the ``default_predicates`` class + attribute. + """ + if not args.strip(): + print("current predicates:") + for (p, v) in self._predicates.items(): + print(" ", p, ":", v) + return + type_value = args.strip().split(" ") + if len(type_value) != 2: + print( + f"Usage: skip_predicates , with one of {set(self._predicates.keys())}" + ) + return + + type_, value = type_value + if type_ not in self._predicates: + print(f"{type_!r} not in {set(self._predicates.keys())}") + return + if value.lower() not in ("true", "yes", "1", "no", "false", "0"): + print( + f"{value!r} is invalid - use one of ('true', 'yes', '1', 'no', 'false', '0')" + ) + return + + self._predicates[type_] = value.lower() in ("true", "yes", "1") + if not any(self._predicates.values()): + print( + "Warning, all predicates set to False, skip_hidden may not have any effects." + ) + + def do_skip_hidden(self, arg): + """ + Change whether or not we should skip frames with the + __tracebackhide__ attribute. + """ + if not arg.strip(): + print( + f"skip_hidden = {self.skip_hidden}, use 'yes','no', 'true', or 'false' to change." + ) + elif arg.strip().lower() in ("true", "yes"): + self.skip_hidden = True + elif arg.strip().lower() in ("false", "no"): + self.skip_hidden = False + if not any(self._predicates.values()): + print( + "Warning, all predicates set to False, skip_hidden may not have any effects." + ) + def do_list(self, arg): """Print lines of code from the current stack frame """ @@ -525,7 +673,7 @@ def do_list(self, arg): def getsourcelines(self, obj): lines, lineno = inspect.findsource(obj) - if inspect.isframe(obj) and obj.f_globals is obj.f_locals: + if inspect.isframe(obj) and obj.f_globals is self._get_frame_locals(obj): # must be a module frame: do not try to cut a block out of it return lines, 1 elif inspect.ismodule(obj): @@ -553,6 +701,7 @@ def do_debug(self, arg): argument (which is an arbitrary expression or statement to be executed in the current environment). """ + trace_function = sys.gettrace() sys.settrace(None) globals = self.curframe.f_globals locals = self.curframe_locals @@ -563,55 +712,67 @@ def do_debug(self, arg): self.message("ENTERING RECURSIVE DEBUGGER") sys.call_tracing(p.run, (arg, globals, locals)) self.message("LEAVING RECURSIVE DEBUGGER") - sys.settrace(self.trace_dispatch) + sys.settrace(trace_function) self.lastcmd = p.lastcmd def do_pdef(self, arg): """Print the call signature for any callable object. The debugger interface to %pdef""" - namespaces = [('Locals', self.curframe.f_locals), - ('Globals', self.curframe.f_globals)] - self.shell.find_line_magic('pdef')(arg, namespaces=namespaces) + namespaces = [ + ("Locals", self.curframe_locals), + ("Globals", self.curframe.f_globals), + ] + self.shell.find_line_magic("pdef")(arg, namespaces=namespaces) def do_pdoc(self, arg): """Print the docstring for an object. The debugger interface to %pdoc.""" - namespaces = [('Locals', self.curframe.f_locals), - ('Globals', self.curframe.f_globals)] - self.shell.find_line_magic('pdoc')(arg, namespaces=namespaces) + namespaces = [ + ("Locals", self.curframe_locals), + ("Globals", self.curframe.f_globals), + ] + self.shell.find_line_magic("pdoc")(arg, namespaces=namespaces) def do_pfile(self, arg): """Print (or run through pager) the file where an object is defined. The debugger interface to %pfile. """ - namespaces = [('Locals', self.curframe.f_locals), - ('Globals', self.curframe.f_globals)] - self.shell.find_line_magic('pfile')(arg, namespaces=namespaces) + namespaces = [ + ("Locals", self.curframe_locals), + ("Globals", self.curframe.f_globals), + ] + self.shell.find_line_magic("pfile")(arg, namespaces=namespaces) def do_pinfo(self, arg): """Provide detailed information about an object. The debugger interface to %pinfo, i.e., obj?.""" - namespaces = [('Locals', self.curframe.f_locals), - ('Globals', self.curframe.f_globals)] - self.shell.find_line_magic('pinfo')(arg, namespaces=namespaces) + namespaces = [ + ("Locals", self.curframe_locals), + ("Globals", self.curframe.f_globals), + ] + self.shell.find_line_magic("pinfo")(arg, namespaces=namespaces) def do_pinfo2(self, arg): """Provide extra detailed information about an object. The debugger interface to %pinfo2, i.e., obj??.""" - namespaces = [('Locals', self.curframe.f_locals), - ('Globals', self.curframe.f_globals)] - self.shell.find_line_magic('pinfo2')(arg, namespaces=namespaces) + namespaces = [ + ("Locals", self.curframe_locals), + ("Globals", self.curframe.f_globals), + ] + self.shell.find_line_magic("pinfo2")(arg, namespaces=namespaces) def do_psource(self, arg): """Print (or run through pager) the source code for an object.""" - namespaces = [('Locals', self.curframe.f_locals), - ('Globals', self.curframe.f_globals)] - self.shell.find_line_magic('psource')(arg, namespaces=namespaces) + namespaces = [ + ("Locals", self.curframe_locals), + ("Globals", self.curframe.f_globals), + ] + self.shell.find_line_magic("psource")(arg, namespaces=namespaces) def do_where(self, arg): """w(here) @@ -622,13 +783,211 @@ def do_where(self, arg): Take a number as argument as an (optional) number of context line to print""" if arg: - context = int(arg) + try: + context = int(arg) + except ValueError as err: + self.error(err) + return self.print_stack_trace(context) else: self.print_stack_trace() do_w = do_where + def break_anywhere(self, frame): + """ + _stop_in_decorator_internals is overly restrictive, as we may still want + to trace function calls, so we need to also update break_anywhere so + that is we don't `stop_here`, because of debugger skip, we may still + stop at any point inside the function + + """ + + sup = super().break_anywhere(frame) + if sup: + return sup + if self._predicates["debuggerskip"]: + if DEBUGGERSKIP in frame.f_code.co_varnames: + return True + if frame.f_back and self._get_frame_locals(frame.f_back).get(DEBUGGERSKIP): + return True + return False + + def _is_in_decorator_internal_and_should_skip(self, frame): + """ + Utility to tell us whether we are in a decorator internal and should stop. + + """ + + # if we are disabled don't skip + if not self._predicates["debuggerskip"]: + return False + + # if frame is tagged, skip by default. + if DEBUGGERSKIP in frame.f_code.co_varnames: + return True + + # if one of the parent frame value set to True skip as well. + + cframe = frame + while getattr(cframe, "f_back", None): + cframe = cframe.f_back + if self._get_frame_locals(cframe).get(DEBUGGERSKIP): + return True + + return False + + def stop_here(self, frame): + + if self._is_in_decorator_internal_and_should_skip(frame) is True: + return False + + hidden = False + if self.skip_hidden: + hidden = self._hidden_predicate(frame) + if hidden: + if self.report_skipped: + Colors = self.color_scheme_table.active_colors + ColorsNormal = Colors.Normal + print( + f"{Colors.excName} [... skipped 1 hidden frame]{ColorsNormal}\n" + ) + return super().stop_here(frame) + + def do_up(self, arg): + """u(p) [count] + Move the current frame count (default one) levels up in the + stack trace (to an older frame). + + Will skip hidden frames. + """ + # modified version of upstream that skips + # frames with __tracebackhide__ + if self.curindex == 0: + self.error("Oldest frame") + return + try: + count = int(arg or 1) + except ValueError: + self.error("Invalid frame count (%s)" % arg) + return + skipped = 0 + if count < 0: + _newframe = 0 + else: + counter = 0 + hidden_frames = self.hidden_frames(self.stack) + for i in range(self.curindex - 1, -1, -1): + if hidden_frames[i] and self.skip_hidden: + skipped += 1 + continue + counter += 1 + if counter >= count: + break + else: + # if no break occurred. + self.error( + "all frames above hidden, use `skip_hidden False` to get get into those." + ) + return + + Colors = self.color_scheme_table.active_colors + ColorsNormal = Colors.Normal + _newframe = i + self._select_frame(_newframe) + if skipped: + print( + f"{Colors.excName} [... skipped {skipped} hidden frame(s)]{ColorsNormal}\n" + ) + + def do_down(self, arg): + """d(own) [count] + Move the current frame count (default one) levels down in the + stack trace (to a newer frame). + + Will skip hidden frames. + """ + if self.curindex + 1 == len(self.stack): + self.error("Newest frame") + return + try: + count = int(arg or 1) + except ValueError: + self.error("Invalid frame count (%s)" % arg) + return + if count < 0: + _newframe = len(self.stack) - 1 + else: + counter = 0 + skipped = 0 + hidden_frames = self.hidden_frames(self.stack) + for i in range(self.curindex + 1, len(self.stack)): + if hidden_frames[i] and self.skip_hidden: + skipped += 1 + continue + counter += 1 + if counter >= count: + break + else: + self.error( + "all frames below hidden, use `skip_hidden False` to get get into those." + ) + return + + Colors = self.color_scheme_table.active_colors + ColorsNormal = Colors.Normal + if skipped: + print( + f"{Colors.excName} [... skipped {skipped} hidden frame(s)]{ColorsNormal}\n" + ) + _newframe = i + + self._select_frame(_newframe) + + do_d = do_down + do_u = do_up + + def do_context(self, context): + """context number_of_lines + Set the number of lines of source code to show when displaying + stacktrace information. + """ + try: + new_context = int(context) + if new_context <= 0: + raise ValueError() + self.context = new_context + except ValueError: + self.error("The 'context' command requires a positive integer argument.") + + +class InterruptiblePdb(Pdb): + """Version of debugger where KeyboardInterrupt exits the debugger altogether.""" + + def cmdloop(self, intro=None): + """Wrap cmdloop() such that KeyboardInterrupt stops the debugger.""" + try: + return OldPdb.cmdloop(self, intro=intro) + except KeyboardInterrupt: + self.stop_here = lambda frame: False + self.do_quit("") + sys.settrace(None) + self.quitting = False + raise + + def _cmdloop(self): + while True: + try: + # keyboard interrupts allow for an easy way to cancel + # the current command, so allow them during interactive input + self.allow_kbdint = True + self.cmdloop() + self.allow_kbdint = False + break + except KeyboardInterrupt: + self.message('--KeyboardInterrupt--') + raise + def set_trace(frame=None): """ diff --git a/IPython/core/display.py b/IPython/core/display.py index 465c000c55a..23d8636b507 100644 --- a/IPython/core/display.py +++ b/IPython/core/display.py @@ -5,12 +5,12 @@ # Distributed under the terms of the Modified BSD License. -from binascii import b2a_hex, b2a_base64, hexlify +from binascii import b2a_base64, hexlify +import html import json import mimetypes import os import struct -import sys import warnings from copy import deepcopy from os.path import splitext @@ -18,14 +18,37 @@ from IPython.utils.py3compat import cast_unicode from IPython.testing.skipdoctest import skip_doctest +from . import display_functions + + +__all__ = ['display_pretty', 'display_html', 'display_markdown', + 'display_svg', 'display_png', 'display_jpeg', 'display_latex', 'display_json', + 'display_javascript', 'display_pdf', 'DisplayObject', 'TextDisplayObject', + 'Pretty', 'HTML', 'Markdown', 'Math', 'Latex', 'SVG', 'ProgressBar', 'JSON', + 'GeoJSON', 'Javascript', 'Image', 'set_matplotlib_formats', + 'set_matplotlib_close', + 'Video'] + +_deprecated_names = ["display", "clear_output", "publish_display_data", "update_display", "DisplayHandle"] + +__all__ = __all__ + _deprecated_names + + +# ----- warn to import from IPython.display ----- + +from warnings import warn + + +def __getattr__(name): + if name in _deprecated_names: + warn(f"Importing {name} from IPython.core.display is deprecated since IPython 7.14, please import from IPython display", DeprecationWarning, stacklevel=2) + return getattr(display_functions, name) + + if name in globals().keys(): + return globals()[name] + else: + raise AttributeError(f"module {__name__} has no attribute {name}") -__all__ = ['display', 'display_pretty', 'display_html', 'display_markdown', -'display_svg', 'display_png', 'display_jpeg', 'display_latex', 'display_json', -'display_javascript', 'display_pdf', 'DisplayObject', 'TextDisplayObject', -'Pretty', 'HTML', 'Markdown', 'Math', 'Latex', 'SVG', 'ProgressBar', 'JSON', -'GeoJSON', 'Javascript', 'Image', 'clear_output', 'set_matplotlib_formats', -'set_matplotlib_close', 'publish_display_data', 'update_display', 'DisplayHandle', -'Video'] #----------------------------------------------------------------------------- # utility functions @@ -38,17 +61,6 @@ def _safe_exists(path): except Exception: return False -def _merge(d1, d2): - """Like update, but merges sub-dicts instead of clobbering at the top level. - - Updates d1 in-place - """ - - if not isinstance(d2, dict) or not isinstance(d1, dict): - return d2 - for key, value in d2.items(): - d1[key] = _merge(d1.get(key), value) - return d1 def _display_mimetype(mimetype, objs, raw=False, metadata=None): """internal implementation of all display_foo methods @@ -57,7 +69,7 @@ def _display_mimetype(mimetype, objs, raw=False, metadata=None): ---------- mimetype : str The mimetype to be published (e.g. 'image/png') - objs : tuple of objects + *objs : object The Python objects to display, or if raw=True raw text data to display. raw : bool @@ -71,334 +83,19 @@ def _display_mimetype(mimetype, objs, raw=False, metadata=None): if raw: # turn list of pngdata into list of { 'image/png': pngdata } objs = [ {mimetype: obj} for obj in objs ] - display(*objs, raw=raw, metadata=metadata, include=[mimetype]) + display_functions.display(*objs, raw=raw, metadata=metadata, include=[mimetype]) #----------------------------------------------------------------------------- # Main functions #----------------------------------------------------------------------------- -# use * to indicate transient is keyword-only -def publish_display_data(data, metadata=None, source=None, *, transient=None, **kwargs): - """Publish data and metadata to all frontends. - - See the ``display_data`` message in the messaging documentation for - more details about this message type. - - Keys of data and metadata can be any mime-type. - - Parameters - ---------- - data : dict - A dictionary having keys that are valid MIME types (like - 'text/plain' or 'image/svg+xml') and values that are the data for - that MIME type. The data itself must be a JSON'able data - structure. Minimally all data should have the 'text/plain' data, - which can be displayed by all frontends. If more than the plain - text is given, it is up to the frontend to decide which - representation to use. - metadata : dict - A dictionary for metadata related to the data. This can contain - arbitrary key, value pairs that frontends can use to interpret - the data. mime-type keys matching those in data can be used - to specify metadata about particular representations. - source : str, deprecated - Unused. - transient : dict, keyword-only - A dictionary of transient data, such as display_id. - """ - from IPython.core.interactiveshell import InteractiveShell - - display_pub = InteractiveShell.instance().display_pub - - # only pass transient if supplied, - # to avoid errors with older ipykernel. - # TODO: We could check for ipykernel version and provide a detailed upgrade message. - if transient: - kwargs['transient'] = transient - - display_pub.publish( - data=data, - metadata=metadata, - **kwargs - ) - - -def _new_id(): - """Generate a new random text id with urandom""" - return b2a_hex(os.urandom(16)).decode('ascii') - - -def display(*objs, include=None, exclude=None, metadata=None, transient=None, display_id=None, **kwargs): - """Display a Python object in all frontends. - - By default all representations will be computed and sent to the frontends. - Frontends can decide which representation is used and how. - - In terminal IPython this will be similar to using :func:`print`, for use in richer - frontends see Jupyter notebook examples with rich display logic. - - Parameters - ---------- - objs : tuple of objects - The Python objects to display. - raw : bool, optional - Are the objects to be displayed already mimetype-keyed dicts of raw display data, - or Python objects that need to be formatted before display? [default: False] - include : list, tuple or set, optional - A list of format type strings (MIME types) to include in the - format data dict. If this is set *only* the format types included - in this list will be computed. - exclude : list, tuple or set, optional - A list of format type strings (MIME types) to exclude in the format - data dict. If this is set all format types will be computed, - except for those included in this argument. - metadata : dict, optional - A dictionary of metadata to associate with the output. - mime-type keys in this dictionary will be associated with the individual - representation formats, if they exist. - transient : dict, optional - A dictionary of transient data to associate with the output. - Data in this dict should not be persisted to files (e.g. notebooks). - display_id : str, bool optional - Set an id for the display. - This id can be used for updating this display area later via update_display. - If given as `True`, generate a new `display_id` - kwargs: additional keyword-args, optional - Additional keyword-arguments are passed through to the display publisher. - - Returns - ------- - - handle: DisplayHandle - Returns a handle on updatable displays for use with :func:`update_display`, - if `display_id` is given. Returns :any:`None` if no `display_id` is given - (default). - - Examples - -------- - - >>> class Json(object): - ... def __init__(self, json): - ... self.json = json - ... def _repr_pretty_(self, pp, cycle): - ... import json - ... pp.text(json.dumps(self.json, indent=2)) - ... def __repr__(self): - ... return str(self.json) - ... - - >>> d = Json({1:2, 3: {4:5}}) - - >>> print(d) - {1: 2, 3: {4: 5}} - - >>> display(d) - { - "1": 2, - "3": { - "4": 5 - } - } - - >>> def int_formatter(integer, pp, cycle): - ... pp.text('I'*integer) - - >>> plain = get_ipython().display_formatter.formatters['text/plain'] - >>> plain.for_type(int, int_formatter) - - >>> display(7-5) - II - - >>> del plain.type_printers[int] - >>> display(7-5) - 2 - - See Also - -------- - - :func:`update_display` - - Notes - ----- - - In Python, objects can declare their textual representation using the - `__repr__` method. IPython expands on this idea and allows objects to declare - other, rich representations including: - - - HTML - - JSON - - PNG - - JPEG - - SVG - - LaTeX - - A single object can declare some or all of these representations; all are - handled by IPython's display system. - - The main idea of the first approach is that you have to implement special - display methods when you define your class, one for each representation you - want to use. Here is a list of the names of the special methods and the - values they must return: - - - `_repr_html_`: return raw HTML as a string, or a tuple (see below). - - `_repr_json_`: return a JSONable dict, or a tuple (see below). - - `_repr_jpeg_`: return raw JPEG data, or a tuple (see below). - - `_repr_png_`: return raw PNG data, or a tuple (see below). - - `_repr_svg_`: return raw SVG data as a string, or a tuple (see below). - - `_repr_latex_`: return LaTeX commands in a string surrounded by "$", - or a tuple (see below). - - `_repr_mimebundle_`: return a full mimebundle containing the mapping - from all mimetypes to data. - Use this for any mime-type not listed above. - - The above functions may also return the object's metadata alonside the - data. If the metadata is available, the functions will return a tuple - containing the data and metadata, in that order. If there is no metadata - available, then the functions will return the data only. - - When you are directly writing your own classes, you can adapt them for - display in IPython by following the above approach. But in practice, you - often need to work with existing classes that you can't easily modify. - - You can refer to the documentation on integrating with the display system in - order to register custom formatters for already existing types - (:ref:`integrating_rich_display`). - - .. versionadded:: 5.4 display available without import - .. versionadded:: 6.1 display available without import - - Since IPython 5.4 and 6.1 :func:`display` is automatically made available to - the user without import. If you are using display in a document that might - be used in a pure python context or with older version of IPython, use the - following import at the top of your file:: - - from IPython.display import display - - """ - from IPython.core.interactiveshell import InteractiveShell - - if not InteractiveShell.initialized(): - # Directly print objects. - print(*objs) - return - - raw = kwargs.pop('raw', False) - if transient is None: - transient = {} - if metadata is None: - metadata={} - if display_id: - if display_id is True: - display_id = _new_id() - transient['display_id'] = display_id - if kwargs.get('update') and 'display_id' not in transient: - raise TypeError('display_id required for update_display') - if transient: - kwargs['transient'] = transient - - if not objs and display_id: - # if given no objects, but still a request for a display_id, - # we assume the user wants to insert an empty output that - # can be updated later - objs = [{}] - raw = True - - if not raw: - format = InteractiveShell.instance().display_formatter.format - - for obj in objs: - if raw: - publish_display_data(data=obj, metadata=metadata, **kwargs) - else: - format_dict, md_dict = format(obj, include=include, exclude=exclude) - if not format_dict: - # nothing to display (e.g. _ipython_display_ took over) - continue - if metadata: - # kwarg-specified metadata gets precedence - _merge(md_dict, metadata) - publish_display_data(data=format_dict, metadata=md_dict, **kwargs) - if display_id: - return DisplayHandle(display_id) - - -# use * for keyword-only display_id arg -def update_display(obj, *, display_id, **kwargs): - """Update an existing display by id - - Parameters - ---------- - - obj: - The object with which to update the display - display_id: keyword-only - The id of the display to update - - See Also - -------- - - :func:`display` - """ - kwargs['update'] = True - display(obj, display_id=display_id, **kwargs) - - -class DisplayHandle(object): - """A handle on an updatable display - - Call `.update(obj)` to display a new object. - - Call `.display(obj`) to add a new instance of this display, - and update existing instances. - - See Also - -------- - - :func:`display`, :func:`update_display` - - """ - - def __init__(self, display_id=None): - if display_id is None: - display_id = _new_id() - self.display_id = display_id - - def __repr__(self): - return "<%s display_id=%s>" % (self.__class__.__name__, self.display_id) - - def display(self, obj, **kwargs): - """Make a new display with my id, updating existing instances. - - Parameters - ---------- - - obj: - object to display - **kwargs: - additional keyword arguments passed to display - """ - display(obj, display_id=self.display_id, **kwargs) - - def update(self, obj, **kwargs): - """Update existing displays with my id - - Parameters - ---------- - - obj: - object to display - **kwargs: - additional keyword arguments passed to update_display - """ - update_display(obj, display_id=self.display_id, **kwargs) - def display_pretty(*objs, **kwargs): """Display the pretty (default) representation of an object. Parameters ---------- - objs : tuple of objects + *objs : object The Python objects to display, or if raw=True raw text data to display. raw : bool @@ -418,7 +115,7 @@ def display_html(*objs, **kwargs): Parameters ---------- - objs : tuple of objects + *objs : object The Python objects to display, or if raw=True raw HTML data to display. raw : bool @@ -435,7 +132,7 @@ def display_markdown(*objs, **kwargs): Parameters ---------- - objs : tuple of objects + *objs : object The Python objects to display, or if raw=True raw markdown data to display. raw : bool @@ -453,7 +150,7 @@ def display_svg(*objs, **kwargs): Parameters ---------- - objs : tuple of objects + *objs : object The Python objects to display, or if raw=True raw svg data to display. raw : bool @@ -470,7 +167,7 @@ def display_png(*objs, **kwargs): Parameters ---------- - objs : tuple of objects + *objs : object The Python objects to display, or if raw=True raw png data to display. raw : bool @@ -487,7 +184,7 @@ def display_jpeg(*objs, **kwargs): Parameters ---------- - objs : tuple of objects + *objs : object The Python objects to display, or if raw=True raw JPEG data to display. raw : bool @@ -504,7 +201,7 @@ def display_latex(*objs, **kwargs): Parameters ---------- - objs : tuple of objects + *objs : object The Python objects to display, or if raw=True raw latex data to display. raw : bool @@ -523,7 +220,7 @@ def display_json(*objs, **kwargs): Parameters ---------- - objs : tuple of objects + *objs : object The Python objects to display, or if raw=True raw json data to display. raw : bool @@ -540,7 +237,7 @@ def display_javascript(*objs, **kwargs): Parameters ---------- - objs : tuple of objects + *objs : object The Python objects to display, or if raw=True raw javascript data to display. raw : bool @@ -557,7 +254,7 @@ def display_pdf(*objs, **kwargs): Parameters ---------- - objs : tuple of objects + *objs : object The Python objects to display, or if raw=True raw javascript data to display. raw : bool @@ -615,9 +312,12 @@ def __init__(self, data=None, url=None, filename=None, metadata=None): filename = data data = None - self.data = data self.url = url self.filename = filename + # because of @data.setter methods in + # subclasses ensure url and filename are set + # before assigning to self.data + self.data = data if metadata is not None: self.metadata = metadata @@ -649,29 +349,59 @@ def _data_and_metadata(self): def reload(self): """Reload the raw data from file or URL.""" if self.filename is not None: - with open(self.filename, self._read_flags) as f: + encoding = None if "b" in self._read_flags else "utf-8" + with open(self.filename, self._read_flags, encoding=encoding) as f: self.data = f.read() elif self.url is not None: - try: - # Deferred import - from urllib.request import urlopen - response = urlopen(self.url) - self.data = response.read() - # extract encoding from header, if there is one: - encoding = None + # Deferred import + from urllib.request import urlopen + response = urlopen(self.url) + data = response.read() + # extract encoding from header, if there is one: + encoding = None + if 'content-type' in response.headers: for sub in response.headers['content-type'].split(';'): sub = sub.strip() if sub.startswith('charset'): encoding = sub.split('=')[-1].strip() break - # decode data, if an encoding was specified - if encoding: - self.data = self.data.decode(encoding, 'replace') - except: - self.data = None + if 'content-encoding' in response.headers: + # TODO: do deflate? + if 'gzip' in response.headers['content-encoding']: + import gzip + from io import BytesIO + + # assume utf-8 if encoding is not specified + with gzip.open( + BytesIO(data), "rt", encoding=encoding or "utf-8" + ) as fp: + encoding = None + data = fp.read() + + # decode data, if an encoding was specified + # We only touch self.data once since + # subclasses such as SVG have @data.setter methods + # that transform self.data into ... well svg. + if encoding: + self.data = data.decode(encoding, 'replace') + else: + self.data = data + class TextDisplayObject(DisplayObject): - """Validate that display data is text""" + """Create a text display object given raw data. + + Parameters + ---------- + data : str or unicode + The raw data or a URL or file to load the data from. + url : unicode + A URL to download the data from. + filename : unicode + Path to a local file to load the data from. + metadata : dict + Dict of metadata associated to be the object when displayed + """ def _check_data(self): if self.data is not None and not isinstance(self.data, str): raise TypeError("%s expects text, not %r" % (self.__class__.__name__, self.data)) @@ -736,6 +466,11 @@ def _repr_latex_(self): class SVG(DisplayObject): + """Embed an SVG into the display. + + Note if you just want to view a svg image via a URL use `:class:Image` with + a url=URL keyword argument. + """ _read_flags = 'rb' # wrap data in a property, which extracts the tag, discarding @@ -764,16 +499,16 @@ def data(self, svg): pass svg = cast_unicode(svg) self._data = svg - + def _repr_svg_(self): return self._data_and_metadata() class ProgressBar(DisplayObject): - """Progressbar supports displaying a progressbar like element + """Progressbar supports displaying a progressbar like element """ def __init__(self, total): """Creates a new progressbar - + Parameters ---------- total : int @@ -799,10 +534,10 @@ def _repr_html_(self): self.html_width, self.total, self.progress) def display(self): - display(self, display_id=self._display_id) + display_functions.display(self, display_id=self._display_id) def update(self): - display(self, display_id=self._display_id, update=True) + display_functions.display(self, display_id=self._display_id, update=True) @property def progress(self): @@ -850,10 +585,10 @@ def __init__(self, data=None, url=None, filename=None, expanded=False, metadata= Path to a local file to load the data from. expanded : boolean Metadata to control whether a JSON display component is expanded. - metadata: dict + metadata : dict Specify extra metadata to attach to the json display object. root : str - The name of the root element of the JSON tree + The name of the root element of the JSON tree """ self.metadata = { 'expanded': expanded, @@ -879,7 +614,7 @@ def data(self, data): data = str(data) if isinstance(data, str): - if getattr(self, 'filename', None) is None: + if self.filename is None and self.url is None: warnings.warn("JSON expects JSONable dict or list, not JSON strings") data = json.loads(data) self._data = data @@ -890,8 +625,9 @@ def _data_and_metadata(self): def _repr_json_(self): return self._data_and_metadata() + _css_t = """var link = document.createElement("link"); - link.ref = "stylesheet"; + link.rel = "stylesheet"; link.type = "text/css"; link.href = "%s"; document.head.appendChild(link); @@ -916,7 +652,7 @@ class GeoJSON(JSON): Scalar types (None, number, string) are not allowed, only dict containers. """ - + def __init__(self, *args, **kwargs): """Create a GeoJSON display object given raw data. @@ -934,12 +670,11 @@ def __init__(self, *args, **kwargs): A URL to download the data from. filename : unicode Path to a local file to load the data from. - metadata: dict + metadata : dict Specify extra metadata to attach to the json display object. Examples -------- - The following will display an interactive map of Mars with a point of interest on frontend that do support GeoJSON display. @@ -965,7 +700,7 @@ def __init__(self, *args, **kwargs): the GeoJSON object. """ - + super(GeoJSON, self).__init__(*args, **kwargs) @@ -977,7 +712,7 @@ def _ipython_display_(self): metadata = { 'application/geo+json': self.metadata } - display(bundle, metadata=metadata, raw=True) + display_functions.display(bundle, metadata=metadata, raw=True) class Javascript(TextDisplayObject): @@ -1006,7 +741,7 @@ def __init__(self, data=None, url=None, filename=None, lib=None, css=None): running the source code. The full URLs of the libraries should be given. A single Javascript library URL can also be given as a string. - css: : list or str + css : list or str A sequence of css files to load before running the source code. The full URLs of the css files should be given. A single css URL can also be given as a string. @@ -1084,9 +819,20 @@ class Image(DisplayObject): _FMT_GIF: 'image/gif', } - def __init__(self, data=None, url=None, filename=None, format=None, - embed=None, width=None, height=None, retina=False, - unconfined=False, metadata=None): + def __init__( + self, + data=None, + url=None, + filename=None, + format=None, + embed=None, + width=None, + height=None, + retina=False, + unconfined=False, + metadata=None, + alt=None, + ): """Create a PNG/JPEG/GIF image object given raw data. When this object is returned by an input cell or passed to the @@ -1098,15 +844,19 @@ def __init__(self, data=None, url=None, filename=None, format=None, data : unicode, str or bytes The raw image data or a URL or filename to load the data from. This always results in embedded image data. + url : unicode A URL to download the data from. If you specify `url=`, the image data will not be embedded unless you also specify `embed=True`. + filename : unicode Path to a local file to load the data from. Images from a file are always embedded. + format : unicode The format of the image data (png/jpeg/jpg/gif). If a filename or URL is given for format will be inferred from the filename extension. + embed : bool Should the image data be embedded using a data URI (True) or be loaded using an tag. Set this to True if you want the image @@ -1116,10 +866,13 @@ def __init__(self, data=None, url=None, filename=None, format=None, default value is `False`. Note that QtConsole is not able to display images if `embed` is set to `False` + width : int Width in pixels to which to constrain the image in html + height : int Height in pixels to which to constrain the image in html + retina : bool Automatically set the width and height to half of the measured width and height. @@ -1127,25 +880,38 @@ def __init__(self, data=None, url=None, filename=None, format=None, from image data. For non-embedded images, you can just set the desired display width and height directly. - unconfined: bool + + unconfined : bool Set unconfined=True to disable max-width confinement of the image. - metadata: dict + + metadata : dict Specify extra metadata to attach to the image. + alt : unicode + Alternative text for the image, for use by screen readers. + Examples -------- - # embedded image data, works in qtconsole and notebook - # when passed positionally, the first arg can be any of raw image data, - # a URL, or a filename from which to load image data. - # The result is always embedding image data for inline images. - Image('http://www.google.fr/images/srpr/logo3w.png') - Image('/path/to/image.jpg') - Image(b'RAW_PNG_DATA...') - - # Specifying Image(url=...) does not embed the image data, - # it only generates `` tag with a link to the source. - # This will not work in the qtconsole or offline. - Image(url='http://www.google.fr/images/srpr/logo3w.png') + embedded image data, works in qtconsole and notebook + when passed positionally, the first arg can be any of raw image data, + a URL, or a filename from which to load image data. + The result is always embedding image data for inline images. + + >>> Image('https://www.google.fr/images/srpr/logo3w.png') # doctest: +SKIP + + + >>> Image('/path/to/image.jpg') + + + >>> Image(b'RAW_PNG_DATA...') + + + Specifying Image(url=...) does not embed the image data, + it only generates ```` tag with a link to the source. + This will not work in the qtconsole or offline. + + >>> Image(url='https://www.google.fr/images/srpr/logo3w.png') + """ if isinstance(data, (Path, PurePath)): @@ -1200,7 +966,8 @@ def __init__(self, data=None, url=None, filename=None, format=None, self.height = height self.retina = retina self.unconfined = unconfined - super(Image, self).__init__(data=data, url=url, filename=filename, + self.alt = alt + super(Image, self).__init__(data=data, url=url, filename=filename, metadata=metadata) if self.width is None and self.metadata.get('width', {}): @@ -1209,6 +976,9 @@ def __init__(self, data=None, url=None, filename=None, format=None, if self.height is None and self.metadata.get('height', {}): self.height = metadata['height'] + if self.alt is None and self.metadata.get("alt", {}): + self.alt = metadata["alt"] + if retina: self._retina_shape() @@ -1238,18 +1008,21 @@ def reload(self): def _repr_html_(self): if not self.embed: - width = height = klass = '' + width = height = klass = alt = "" if self.width: width = ' width="%d"' % self.width if self.height: height = ' height="%d"' % self.height if self.unconfined: klass = ' class="unconfined"' - return u''.format( + if self.alt: + alt = ' alt="%s"' % html.escape(self.alt) + return ''.format( url=self.url, width=width, height=height, klass=klass, + alt=alt, ) def _repr_mimebundle_(self, include=None, exclude=None): @@ -1270,9 +1043,9 @@ def _data_and_metadata(self, always_both=False): """shortcut for returning metadata with shape information, if defined""" try: b64_data = b2a_base64(self.data).decode('ascii') - except TypeError: + except TypeError as e: raise FileNotFoundError( - "No such file or directory: '%s'" % (self.data)) + "No such file or directory: '%s'" % (self.data)) from e md = {} if self.metadata: md.update(self.metadata) @@ -1282,6 +1055,8 @@ def _data_and_metadata(self, always_both=False): md['height'] = self.height if self.unconfined: md['unconfined'] = self.unconfined + if self.alt: + md["alt"] = self.alt if md or always_both: return b64_data, md else: @@ -1308,7 +1083,7 @@ def _find_ext(self, s): class Video(DisplayObject): def __init__(self, data=None, url=None, filename=None, embed=False, - mimetype=None, width=None, height=None): + mimetype=None, width=None, height=None, html_attributes="controls"): """Create a video object given raw data or an URL. When this object is returned by an input cell or passed to the @@ -1319,41 +1094,54 @@ def __init__(self, data=None, url=None, filename=None, embed=False, ---------- data : unicode, str or bytes The raw video data or a URL or filename to load the data from. - Raw data will require passing `embed=True`. + Raw data will require passing ``embed=True``. + url : unicode - A URL for the video. If you specify `url=`, + A URL for the video. If you specify ``url=``, the image data will not be embedded. + filename : unicode Path to a local file containing the video. - Will be interpreted as a local URL unless `embed=True`. + Will be interpreted as a local URL unless ``embed=True``. + embed : bool Should the video be embedded using a data URI (True) or be loaded using a