From 0404c26b0a90c45e2994cd1da46bb8debc98b6ed Mon Sep 17 00:00:00 2001 From: Yann Richet Date: Tue, 13 Oct 2026 11:04:46 +0000 Subject: [PATCH 01/16] Benchmark and test notebooks for libKriging, with CI Notebooks (Python, R, Octave, Julia) on test functions of the Virtual Library of Simulation Experiments, covering Kriging kernels/trends/objectives, noise, update/simulate, WarpKriging, MLPKriging, Vecchia/Nystrom/NestedKriging, persistence, robustness, comparisons with scikit-learn, SMT, OpenTURNS, GPy, DiceKriging, RobustGaSP, STK, GaussianProcesses.jl, and a cross-binding consistency protocol. CI runs them on libKriging master and on the last release, and aggregates every check into one report. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01KstATfYd5VTRTVWWM8Jx8C --- .github/workflows/notebooks.yml | 379 +++++++ .gitignore | 7 + LICENSE | 201 ++++ README.md | 100 ++ baseline/metrics-master.csv | 948 ++++++++++++++++++ baseline/metrics-release.csv | 916 +++++++++++++++++ data/borehole_X.csv | 80 ++ data/borehole_Xt.csv | 200 ++++ data/borehole_y.csv | 80 ++ data/borehole_yt.csv | 200 ++++ data/branin_X.csv | 25 + data/branin_Xt.csv | 200 ++++ data/branin_y.csv | 25 + data/branin_yt.csv | 200 ++++ data/hartmann3_X.csv | 40 + data/hartmann3_Xt.csv | 200 ++++ data/hartmann3_y.csv | 40 + data/hartmann3_yt.csv | 200 ++++ known_issues.csv | 11 + lkbench/__init__.py | 4 + lkbench/checks.py | 166 +++ lkbench/compat.py | 61 ++ lkbench/design.py | 31 + lkbench/functions.py | 241 +++++ lkbench/metrics.py | 50 + notebooks/julia/01_cross_binding.ipynb | 150 +++ notebooks/julia/checks.jl | 75 ++ notebooks/julia/src/01_cross_binding.jl | 95 ++ notebooks/octave/01_cross_binding.ipynb | 102 ++ notebooks/octave/02_vs_stk.ipynb | 145 +++ notebooks/octave/lkb_check.m | 14 + notebooks/octave/lkb_close.m | 15 + notebooks/octave/lkb_finalize.m | 24 + notebooks/octave/lkb_metric.m | 3 + notebooks/octave/lkb_suite.m | 20 + notebooks/octave/read_data.m | 3 + notebooks/octave/src/01_cross_binding.m | 59 ++ notebooks/octave/src/02_vs_stk.m | 85 ++ notebooks/python/00_test_functions.ipynb | 168 ++++ .../python/01_kriging_kernels_trends.ipynb | 297 ++++++ .../python/02_objectives_gradients.ipynb | 277 +++++ notebooks/python/03_noise.ipynb | 235 +++++ notebooks/python/04_update_simulate.ipynb | 220 ++++ notebooks/python/05_warpkriging.ipynb | 286 ++++++ notebooks/python/06_high_dimension_mlp.ipynb | 212 ++++ notebooks/python/07_scalability.ipynb | 219 ++++ notebooks/python/08_api_persistence.ipynb | 262 +++++ notebooks/python/09_robustness.ipynb | 260 +++++ .../python/10_compare_python_packages.ipynb | 296 ++++++ notebooks/python/11_cross_binding.ipynb | 123 +++ notebooks/python/src/00_test_functions.py | 89 ++ .../python/src/01_kriging_kernels_trends.py | 194 ++++ .../python/src/02_objectives_gradients.py | 174 ++++ notebooks/python/src/03_noise.py | 151 +++ notebooks/python/src/04_update_simulate.py | 136 +++ notebooks/python/src/05_warpkriging.py | 183 ++++ notebooks/python/src/06_high_dimension_mlp.py | 140 +++ notebooks/python/src/07_scalability.py | 147 +++ notebooks/python/src/08_api_persistence.py | 178 ++++ notebooks/python/src/09_robustness.py | 164 +++ .../python/src/10_compare_python_packages.py | 215 ++++ notebooks/python/src/11_cross_binding.py | 80 ++ notebooks/r/01_cross_binding.ipynb | 92 ++ .../r/02_vs_dicekriging_robustgasp.ipynb | 164 +++ notebooks/r/checks.R | 71 ++ notebooks/r/src/01_cross_binding.R | 49 + .../r/src/02_vs_dicekriging_robustgasp.R | 97 ++ pyproject.toml | 17 + tools/make_data.py | 28 + tools/py2nb.py | 79 ++ tools/report.py | 204 ++++ tools/run_notebooks.py | 122 +++ 72 files changed, 11024 insertions(+) create mode 100644 .github/workflows/notebooks.yml create mode 100644 .gitignore create mode 100644 LICENSE create mode 100644 README.md create mode 100644 baseline/metrics-master.csv create mode 100644 baseline/metrics-release.csv create mode 100644 data/borehole_X.csv create mode 100644 data/borehole_Xt.csv create mode 100644 data/borehole_y.csv create mode 100644 data/borehole_yt.csv create mode 100644 data/branin_X.csv create mode 100644 data/branin_Xt.csv create mode 100644 data/branin_y.csv create mode 100644 data/branin_yt.csv create mode 100644 data/hartmann3_X.csv create mode 100644 data/hartmann3_Xt.csv create mode 100644 data/hartmann3_y.csv create mode 100644 data/hartmann3_yt.csv create mode 100644 known_issues.csv create mode 100644 lkbench/__init__.py create mode 100644 lkbench/checks.py create mode 100644 lkbench/compat.py create mode 100644 lkbench/design.py create mode 100644 lkbench/functions.py create mode 100644 lkbench/metrics.py create mode 100644 notebooks/julia/01_cross_binding.ipynb create mode 100644 notebooks/julia/checks.jl create mode 100644 notebooks/julia/src/01_cross_binding.jl create mode 100644 notebooks/octave/01_cross_binding.ipynb create mode 100644 notebooks/octave/02_vs_stk.ipynb create mode 100644 notebooks/octave/lkb_check.m create mode 100644 notebooks/octave/lkb_close.m create mode 100644 notebooks/octave/lkb_finalize.m create mode 100644 notebooks/octave/lkb_metric.m create mode 100644 notebooks/octave/lkb_suite.m create mode 100644 notebooks/octave/read_data.m create mode 100644 notebooks/octave/src/01_cross_binding.m create mode 100644 notebooks/octave/src/02_vs_stk.m create mode 100644 notebooks/python/00_test_functions.ipynb create mode 100644 notebooks/python/01_kriging_kernels_trends.ipynb create mode 100644 notebooks/python/02_objectives_gradients.ipynb create mode 100644 notebooks/python/03_noise.ipynb create mode 100644 notebooks/python/04_update_simulate.ipynb create mode 100644 notebooks/python/05_warpkriging.ipynb create mode 100644 notebooks/python/06_high_dimension_mlp.ipynb create mode 100644 notebooks/python/07_scalability.ipynb create mode 100644 notebooks/python/08_api_persistence.ipynb create mode 100644 notebooks/python/09_robustness.ipynb create mode 100644 notebooks/python/10_compare_python_packages.ipynb create mode 100644 notebooks/python/11_cross_binding.ipynb create mode 100644 notebooks/python/src/00_test_functions.py create mode 100644 notebooks/python/src/01_kriging_kernels_trends.py create mode 100644 notebooks/python/src/02_objectives_gradients.py create mode 100644 notebooks/python/src/03_noise.py create mode 100644 notebooks/python/src/04_update_simulate.py create mode 100644 notebooks/python/src/05_warpkriging.py create mode 100644 notebooks/python/src/06_high_dimension_mlp.py create mode 100644 notebooks/python/src/07_scalability.py create mode 100644 notebooks/python/src/08_api_persistence.py create mode 100644 notebooks/python/src/09_robustness.py create mode 100644 notebooks/python/src/10_compare_python_packages.py create mode 100644 notebooks/python/src/11_cross_binding.py create mode 100644 notebooks/r/01_cross_binding.ipynb create mode 100644 notebooks/r/02_vs_dicekriging_robustgasp.ipynb create mode 100644 notebooks/r/checks.R create mode 100644 notebooks/r/src/01_cross_binding.R create mode 100644 notebooks/r/src/02_vs_dicekriging_robustgasp.R create mode 100644 pyproject.toml create mode 100644 tools/make_data.py create mode 100644 tools/py2nb.py create mode 100644 tools/report.py create mode 100644 tools/run_notebooks.py diff --git a/.github/workflows/notebooks.yml b/.github/workflows/notebooks.yml new file mode 100644 index 0000000..dcbe029 --- /dev/null +++ b/.github/workflows/notebooks.yml @@ -0,0 +1,379 @@ +name: notebooks + +# Executes every notebook of the benchmark against libKriging, then aggregates the +# checks of all languages into one report (tools/report.py), published as the job +# summary and as an artifact. +# +# Two channels: +# master libKriging built from source at `libkriging_ref` (default: master), all +# languages: Python (wheel built here), R (rlibkriging repository head, +# which follows libKriging master), Octave (mLibKriging built here), +# Julia (JLibKriging.jl built against the same libKriging checkout). +# release the last published packages: pylibkriging from PyPI, rlibkriging from +# CRAN, mLibKriging built from the last release tag. A feature missing +# from the release is recorded as SKIP, not as a failure. +# +# A notebook fails (red job) when one of its error-level checks fails; the report +# job fails when any notebook failed or crashed, or when the bindings disagree +# (cross-binding protocol, notebooks/python/11_cross_binding.ipynb). + +on: + push: + branches: [main] + pull_request: + schedule: + - cron: '17 3 * * 1' # weekly, Monday 03:17 UTC + workflow_dispatch: + inputs: + libkriging_ref: + description: 'libKriging branch, tag or commit for the master channel' + default: 'master' + libkriging_repo: + description: 'libKriging repository (owner/name), e.g. a fork' + default: 'libKriging/libKriging' + channels: + description: 'channels to run: "master", "release" or "master,release"' + default: 'master,release' + repository_dispatch: + # libKriging can trigger a run after a merge: + # POST /repos//dispatches {"event_type": "libkriging-updated", "client_payload": {"ref": ""}} + types: [libkriging-updated] + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +env: + LIBKRIGING_REPO: ${{ inputs.libkriging_repo || 'libKriging/libKriging' }} + LIBKRIGING_REF: ${{ inputs.libkriging_ref || github.event.client_payload.ref || 'master' }} + # cmake >= 4 rejects dependencies declaring cmake_minimum_required < 3.5 + CMAKE_POLICY_VERSION_MINIMUM: "3.5" + # NestedKriging's OpenMP loop can hang on 2-core runners (same workaround as libKriging's CI) + OMP_NUM_THREADS: 1 + PYTHON_VERSION: '3.12' + +jobs: + # --------------------------------------------------------------------------- + setup: + runs-on: ubuntu-24.04 + outputs: + sha: ${{ steps.refs.outputs.sha }} + release_tag: ${{ steps.refs.outputs.release_tag }} + channels: ${{ steps.refs.outputs.channels }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + - name: Notebooks in sync with their sources + run: | + pip install nbformat + python tools/py2nb.py --check notebooks/*/src/* + - name: Resolve libKriging refs + id: refs + env: + GH_TOKEN: ${{ github.token }} + run: | + sha=$(git ls-remote "https://github.com/${LIBKRIGING_REPO}" "${LIBKRIGING_REF}" | head -1 | cut -f1) + [ -n "$sha" ] || sha="${LIBKRIGING_REF}" # already a commit + tag=$(gh api "repos/${LIBKRIGING_REPO}/releases/latest" --jq .tag_name) + channels='${{ inputs.channels || 'master,release' }}' + echo "sha=$sha" >> "$GITHUB_OUTPUT" + echo "release_tag=$tag" >> "$GITHUB_OUTPUT" + echo "channels=$(python -c "import json,sys; print(json.dumps(sys.argv[1].split(',')))" "$channels")" >> "$GITHUB_OUTPUT" + echo "libKriging ${LIBKRIGING_REPO}@${LIBKRIGING_REF} = $sha; last release $tag; channels $channels" >> "$GITHUB_STEP_SUMMARY" + + # --------------------------------------------------------------------------- + build-python: + needs: setup + if: contains(needs.setup.outputs.channels, 'master') + runs-on: ubuntu-24.04 + timeout-minutes: 120 + steps: + - uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + - id: cache + uses: actions/cache@v4 + with: + path: dist + key: pylibkriging-${{ needs.setup.outputs.sha }}-py${{ env.PYTHON_VERSION }} + - if: steps.cache.outputs.cache-hit != 'true' + uses: actions/checkout@v4 + with: + repository: ${{ env.LIBKRIGING_REPO }} + ref: ${{ needs.setup.outputs.sha }} + submodules: recursive + path: libKriging + - name: Build the pylibkriging wheel + if: steps.cache.outputs.cache-hit != 'true' + run: | + sudo apt-get update && sudo apt-get install -y liblapack-dev libopenblas-dev + python -m pip install --upgrade pip + python -m pip install -r libKriging/bindings/Python/pylibkriging/requirements.txt \ + -r libKriging/bindings/Python/pylibkriging/dev-requirements.txt cmake + export CMAKE_BUILD_PARALLEL_LEVEL=4 + python -m pip wheel --no-build-isolation --no-deps -w dist ./libKriging + - uses: actions/upload-artifact@v4 + with: + name: wheel-master + path: dist/*.whl + + # --------------------------------------------------------------------------- + python: + needs: [setup, build-python] + if: always() && needs.setup.result == 'success' + runs-on: ubuntu-24.04 + timeout-minutes: 90 + strategy: + fail-fast: false + matrix: + channel: [master, release] + notebook: + - 00_test_functions + - 01_kriging_kernels_trends + - 02_objectives_gradients + - 03_noise + - 04_update_simulate + - 05_warpkriging + - 06_high_dimension_mlp + - 07_scalability + - 08_api_persistence + - 09_robustness + - 10_compare_python_packages + - 11_cross_binding + name: python ${{ matrix.channel }} ${{ matrix.notebook }} + steps: + - name: Channel enabled? + id: on + run: | + echo 'enabled=${{ contains(needs.setup.outputs.channels, matrix.channel) && (matrix.channel != 'master' || needs.build-python.result == 'success') }}' >> "$GITHUB_OUTPUT" + - if: steps.on.outputs.enabled == 'true' + uses: actions/checkout@v4 + - if: steps.on.outputs.enabled == 'true' + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + - if: steps.on.outputs.enabled == 'true' && matrix.channel == 'master' + uses: actions/download-artifact@v4 + with: + name: wheel-master + path: dist + - name: Install + if: steps.on.outputs.enabled == 'true' + run: | + python -m pip install --upgrade pip + if [ "${{ matrix.channel }}" = master ]; then pip install dist/*.whl; else pip install pylibkriging; fi + pip install -e ".[notebooks,compare]" jinja2 + python -c "import pylibkriging as lk; print('pylibkriging', lk.__version__)" + - name: Execute the notebook + if: steps.on.outputs.enabled == 'true' + run: python tools/run_notebooks.py --inplace --timeout 3600 notebooks/python/${{ matrix.notebook }}.ipynb + - if: always() && steps.on.outputs.enabled == 'true' + uses: actions/upload-artifact@v4 + with: + name: results-${{ matrix.channel }}-py-${{ matrix.notebook }} + path: | + results/ + notebooks/python/${{ matrix.notebook }}.ipynb + + # --------------------------------------------------------------------------- + r: + needs: setup + runs-on: ubuntu-24.04 + timeout-minutes: 90 + strategy: + fail-fast: false + matrix: + channel: [master, release] + name: R ${{ matrix.channel }} + steps: + - name: Channel enabled? + id: on + run: echo 'enabled=${{ contains(needs.setup.outputs.channels, matrix.channel) }}' >> "$GITHUB_OUTPUT" + - if: steps.on.outputs.enabled == 'true' + uses: actions/checkout@v4 + - if: steps.on.outputs.enabled == 'true' + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + - if: steps.on.outputs.enabled == 'true' + uses: r-lib/actions/setup-r@v2 + with: + use-public-rspm: true + - name: Install + if: steps.on.outputs.enabled == 'true' + env: + GITHUB_PAT: ${{ github.token }} + run: | + sudo apt-get update && sudo apt-get install -y liblapack-dev gfortran cmake + pip install nbclient nbformat ipykernel jupyter_client + Rscript -e 'install.packages(c("IRkernel", "DiceKriging", "RobustGaSP", "remotes"))' + if [ "${{ matrix.channel }}" = master ]; then + Rscript -e 'remotes::install_github("libKriging/rlibkriging", upgrade = "never")' + else + Rscript -e 'install.packages("rlibkriging")' + fi + Rscript -e 'IRkernel::installspec(user = TRUE); cat("rlibkriging", as.character(packageVersion("rlibkriging")), "\n")' + - name: Execute the notebooks + if: steps.on.outputs.enabled == 'true' + run: python tools/run_notebooks.py --inplace --timeout 3600 notebooks/r/*.ipynb + - if: always() && steps.on.outputs.enabled == 'true' + uses: actions/upload-artifact@v4 + with: + name: results-${{ matrix.channel }}-r + path: | + results/ + notebooks/r/*.ipynb + + # --------------------------------------------------------------------------- + octave: + needs: setup + runs-on: ubuntu-24.04 + timeout-minutes: 120 + strategy: + fail-fast: false + matrix: + channel: [master, release] + name: Octave ${{ matrix.channel }} + env: + # the release binaries target another Octave version than Ubuntu's: build from source + REF: ${{ matrix.channel == 'master' && needs.setup.outputs.sha || needs.setup.outputs.release_tag }} + steps: + - name: Channel enabled? + id: on + run: echo 'enabled=${{ contains(needs.setup.outputs.channels, matrix.channel) }}' >> "$GITHUB_OUTPUT" + - if: steps.on.outputs.enabled == 'true' + uses: actions/checkout@v4 + - if: steps.on.outputs.enabled == 'true' + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + - name: Install Octave + if: steps.on.outputs.enabled == 'true' + run: | + sudo apt-get update && sudo apt-get install -y octave octave-dev liblapack-dev libopenblas-dev + pip install nbclient nbformat ipykernel octave_kernel + git clone --depth 1 https://github.com/stk-kriging/stk.git "$RUNNER_TEMP/stk" + - if: steps.on.outputs.enabled == 'true' + id: cache + uses: actions/cache@v4 + with: + path: mlibkriging + key: mlibkriging-${{ env.REF }}-${{ runner.os }}-octave + - if: steps.on.outputs.enabled == 'true' && steps.cache.outputs.cache-hit != 'true' + uses: actions/checkout@v4 + with: + repository: ${{ env.LIBKRIGING_REPO }} + ref: ${{ env.REF }} + submodules: recursive + path: libKriging + - name: Build mLibKriging + if: steps.on.outputs.enabled == 'true' && steps.cache.outputs.cache-hit != 'true' + run: | + cmake -S libKriging -B build -DCMAKE_BUILD_TYPE=Release -DENABLE_OCTAVE_BINDING=on \ + -DENABLE_PYTHON_BINDING=off -DENABLE_MATLAB_BINDING=off -DBUILD_SHARED_LIBS=off -DSTATIC_LIB=on + cmake --build build --target mLibKriging -j 4 + mkdir -p mlibkriging + cp build/bindings/Octave/mlibkriging/mLibKriging.mex libKriging/bindings/Octave/mlibkriging/*.m mlibkriging/ + - name: Execute the notebooks + if: steps.on.outputs.enabled == 'true' + run: | + export OCTAVE_PATH="$PWD/mlibkriging" STK_PATH="$RUNNER_TEMP/stk" + python tools/run_notebooks.py --inplace --timeout 3600 notebooks/octave/*.ipynb + - if: always() && steps.on.outputs.enabled == 'true' + uses: actions/upload-artifact@v4 + with: + name: results-${{ matrix.channel }}-octave + path: | + results/ + notebooks/octave/*.ipynb + + # --------------------------------------------------------------------------- + julia: + needs: setup + if: contains(needs.setup.outputs.channels, 'master') + runs-on: ubuntu-24.04 + timeout-minutes: 120 + name: Julia master + steps: + - uses: actions/checkout@v4 + - uses: actions/checkout@v4 + with: + repository: ${{ env.LIBKRIGING_REPO }} + ref: ${{ needs.setup.outputs.sha }} + submodules: recursive + path: libKriging + - uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + - uses: julia-actions/setup-julia@v2 + with: + version: '1.11' + - name: Install + run: | + sudo apt-get update && sudo apt-get install -y liblapack-dev libopenblas-dev + pip install nbclient nbformat ipykernel jupyter_client + # JLibKriging.jl compiles libKriging; LIBKRIGING_SRC_DIR makes it use this exact checkout + export LIBKRIGING_SRC_DIR="$PWD/libKriging" + julia -e 'using Pkg + Pkg.add(url="https://github.com/libKriging/JLibKriging.jl") + Pkg.add(["IJulia", "DelimitedFiles", "GaussianProcesses", "Optim"]) + using IJulia; installkernel("Julia"; specname="julia-1.11") + using JLibKriging' + - name: Execute the notebooks + run: python tools/run_notebooks.py --inplace --timeout 3600 --kernel julia-1.11 notebooks/julia/*.ipynb + - if: always() + uses: actions/upload-artifact@v4 + with: + name: results-master-julia + path: | + results/ + notebooks/julia/*.ipynb + + # --------------------------------------------------------------------------- + report: + needs: [setup, python, r, octave, julia] + if: always() && needs.setup.result == 'success' + runs-on: ubuntu-24.04 + strategy: + fail-fast: false + matrix: + channel: [master, release] + name: report ${{ matrix.channel }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + - if: contains(needs.setup.outputs.channels, matrix.channel) + uses: actions/download-artifact@v4 + with: + pattern: results-${{ matrix.channel }}-* + path: artifacts + - name: Report + if: contains(needs.setup.outputs.channels, matrix.channel) + run: | + pip install numpy pandas + mkdir -p results executed + find artifacts -path '*/results/*' -type f -exec cp {} results/ \; + for d in artifacts/*/; do if [ -d "$d/notebooks" ]; then cp -r "$d/notebooks/." executed/; fi; done + expect="py00_test_functions,py01_kernels_trends,py02_objectives_gradients,py03_noise,py04_update_simulate" + expect="$expect,py05_warpkriging,py06_high_dimension_mlp,py07_scalability,py08_api_persistence" + expect="$expect,py09_robustness,py10_compare_python_packages,py11_cross_binding" + expect="$expect,r01_cross_binding,r02_vs_dicekriging_robustgasp,oct01_cross_binding,oct02_vs_stk" + if [ "${{ matrix.channel }}" = master ]; then expect="$expect,jl01_cross_binding"; fi + echo "## Channel: ${{ matrix.channel }}" >> "$GITHUB_STEP_SUMMARY" + python tools/report.py --results results --expect "$expect" \ + --baseline baseline/metrics-${{ matrix.channel }}.csv \ + --write-baseline new-baseline/metrics-${{ matrix.channel }}.csv \ + --summary "$GITHUB_STEP_SUMMARY" > report.md + - if: always() && contains(needs.setup.outputs.channels, matrix.channel) + uses: actions/upload-artifact@v4 + with: + name: report-${{ matrix.channel }} + path: | + report.md + results/ + executed/ + new-baseline/ diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..24aa0c7 --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +results/ +__pycache__/ +*.egg-info/ +.ipynb_checkpoints/ +new-baseline/ +executed/ +report.md diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md new file mode 100644 index 0000000..7d2ee7f --- /dev/null +++ b/README.md @@ -0,0 +1,100 @@ +# libKriging benchmarks + +Test and benchmark notebooks for [libKriging](https://github.com/libKriging/libKriging), +kept outside the library so that they can follow both its development head and +its releases. Every notebook **checks** what it computes (error-level checks fail +the run, warn-level ones are reported), and CI executes all of them and +aggregates the findings of every language into one report. + +## What is covered + +| notebook | problems (test functions) | libKriging features | compared with | +|---|---|---|---| +| [python/00](notebooks/python/00_test_functions.ipynb) | the 21 test functions themselves, against published optima / ranges | — | — | +| [python/01](notebooks/python/01_kriging_kernels_trends.ipynb) | Forrester, Gramacy & Lee 2012, Branin, six-hump camel, log-Goldstein-Price, Lim, Ishigami, Hartmann 3 | 4 kernels × 4 trends, interpolation, `predict` (cov, derivatives) | finite differences | +| [python/02](notebooks/python/02_objectives_gradients.ipynb) | Branin, Hartmann 3, Ishigami, Friedman | `LL` / `LOO` / `LMP` objectives and gradients, optimality of θ̂, `leaveOneOutVec`, multistart | finite differences, explicit refits | +| [python/03](notebooks/python/03_noise.ipynb) | Forrester, Friedman, Gramacy & Lee 2012, Branin (replicates) | `noise="nugget"`, heteroscedastic noise vector | true noise, replicate averaging | +| [python/04](notebooks/python/04_update_simulate.ipynb) | Branin, Hartmann 3, Hartmann 6 | `update`, `simulate`, `update_simulate` | fits from scratch, moments | +| [python/05](notebooks/python/05_warpkriging.ipynb) | Gramacy & Lee 2012 / 2008, mixed categorical Branin, ordinal Hartmann 3 | `WarpKriging`: affine, boxcox, kumaraswamy, knots, neural_mono, categorical, ordinal | plain `Kriging` | +| [python/06](notebooks/python/06_high_dimension_mlp.ipynb) | OTL circuit, piston, borehole, robot arm, Dette-Pepelyshev, wing weight, Welch (20D), projected Branin (10D) | `normalize`, screening, `MLPKriging` | plain `Kriging` | +| [python/07](notebooks/python/07_scalability.ipynb) | Hartmann 3, OTL circuit (n up to 5000), Branin | `method="vecchia(m)"`, `"nystrom(k)"`, `NestedKriging` (5 aggregations), `subsetOfData` | exact fit | +| [python/08](notebooks/python/08_api_persistence.ipynb) | Hartmann 3 | `save`/`load`, pickle, copy, accessors, scikit-learn estimators | — | +| [python/09](notebooks/python/09_robustness.ipynb) | Branin | invalid inputs, duplicates, constant y, ill-conditioning, scale equivariance, extrapolation, cost in q | — | +| [python/10](notebooks/python/10_compare_python_packages.ipynb) | 10 functions, 2D to 10D | default fits | scikit-learn, SMT, OpenTURNS, GPy (exact equality at fixed hyperparameters + accuracy/cost) | +| [python/11](notebooks/python/11_cross_binding.ipynb), [r/01](notebooks/r/01_cross_binding.ipynb), [octave/01](notebooks/octave/01_cross_binding.ipynb), [julia/01](notebooks/julia/01_cross_binding.ipynb) | shared datasets of `data/` | the same protocol in every binding | each other (report) | +| [r/02](notebooks/r/02_vs_dicekriging_robustgasp.ipynb) | Branin, Hartmann 3, Borehole | `rlibkriging` | DiceKriging (exact equality), RobustGaSP | +| [octave/02](notebooks/octave/02_vs_stk.ipynb) | Branin, Hartmann 3, Borehole | `mLibKriging` | STK (exact equality, Gaussian kernel) | +| [julia/01](notebooks/julia/01_cross_binding.ipynb) | Branin, Hartmann 3, Borehole | `JLibKriging` | GaussianProcesses.jl | + +Test functions: [`lkbench/functions.py`](lkbench/functions.py), after the +[Virtual Library of Simulation Experiments](https://www.sfu.ca/~ssurjano/) +(Surjanovic & Bingham). + +## Checks and statuses + +Each notebook records its checks with `lkbench.Suite` (Python) or the small +equivalents in `notebooks/{r,octave,julia}/` and writes +`results/.checks.csv` and `results/.metrics.csv`. + +| status | meaning | fails CI | +|---|---|---| +| PASS | | | +| FAIL | an error-level check failed: a correctness property (gradient vs finite differences, `update` vs fit from scratch, equality with another package at fixed hyperparameters, ...) or an accuracy floor | yes | +| WARN | a warn-level check failed: performance comparison, optimizer behaviour, judgement thresholds | no | +| KNOWN | an error-level failure listed in [`known_issues.csv`](known_issues.csv), with the issue | no | +| SKIP | feature absent from the installed version (release channel) | no | + +`tools/report.py` aggregates all results: status per notebook, failures, +warnings, known issues still present, known issues that now pass (to be removed +from the list), cross-binding consistency against Python, and metric drifts +against [`baseline/`](baseline). + +## Running locally + +```sh +pip install -e ".[notebooks,compare]" jinja2 pylibkriging # or a pylibkriging built from source +python tools/run_notebooks.py notebooks/python/*.ipynb +python tools/report.py +``` + +R needs `rlibkriging`, `DiceKriging`, `RobustGaSP` and `IRkernel`; Octave needs +mLibKriging on `OCTAVE_PATH`, STK on `STK_PATH` and `octave_kernel`; Julia needs +`JLibKriging`, `IJulia`, `DelimitedFiles` (and optionally `GaussianProcesses`, +`Optim`). See the CI workflow for exact commands. + +## Editing notebooks + +Notebooks are generated from plain-text sources (`notebooks//src/`, cells +separated by `# %%` / `# %% [markdown]`, or `% %%` in Octave), so that reviews are +plain diffs; outputs are not committed (CI publishes the executed notebooks as +artifacts). + +```sh +python tools/py2nb.py notebooks/*/src/* # regenerate +python tools/py2nb.py --check notebooks/*/src/* # CI: sources and notebooks in sync +``` + +Shared datasets (`data/`, used by the cross-binding notebooks of every language) +are written once by `tools/make_data.py` and committed: the files are the +reference. + +## CI + +[`.github/workflows/notebooks.yml`](.github/workflows/notebooks.yml), on push, +pull request, weekly, on demand (`libkriging_ref`, `libkriging_repo`, +`channels` inputs) and on a `repository_dispatch` of type `libkriging-updated` +(sent by libKriging after a merge, with the commit in `client_payload.ref`): + +* **master channel**: libKriging built from source at the requested ref, in + every language (Python wheel, Octave mex, JLibKriging.jl against the same + checkout; R from the `libKriging/rlibkriging` repository, which follows + libKriging master); +* **release channel**: pylibkriging from PyPI, rlibkriging from CRAN, + mLibKriging built from the last release tag (the published binaries target + another Octave version than Ubuntu's). + +One job per Python notebook, one per language for the others, then one report +per channel (job summary + artifact with the report, all CSV results, the +executed notebooks and a refreshed baseline). The initial `baseline/` files come +from a local run (Python notebooks only, 4-core container); to refresh them, +commit the `new-baseline/` files of a trusted CI run. diff --git a/baseline/metrics-master.csv b/baseline/metrics-master.csv new file mode 100644 index 0000000..98a29f4 --- /dev/null +++ b/baseline/metrics-master.csv @@ -0,0 +1,948 @@ +suite,lang,case,package,metric,value +py01_kernels_trends,python,forrester/gauss/constant,pylibkriging,fit_s,0.00161386 +py01_kernels_trends,python,forrester/gauss/constant,pylibkriging,rmse,0.0848271 +py01_kernels_trends,python,forrester/gauss/constant,pylibkriging,q2,0.999634 +py01_kernels_trends,python,forrester/gauss/constant,pylibkriging,nlpd,-3.06486 +py01_kernels_trends,python,forrester/gauss/linear,pylibkriging,fit_s,0.00126195 +py01_kernels_trends,python,forrester/gauss/linear,pylibkriging,rmse,0.0855468 +py01_kernels_trends,python,forrester/gauss/linear,pylibkriging,q2,0.999628 +py01_kernels_trends,python,forrester/gauss/linear,pylibkriging,nlpd,-3.11029 +py01_kernels_trends,python,forrester/gauss/interactive,pylibkriging,fit_s,0.00143647 +py01_kernels_trends,python,forrester/gauss/interactive,pylibkriging,rmse,0.0855468 +py01_kernels_trends,python,forrester/gauss/interactive,pylibkriging,q2,0.999628 +py01_kernels_trends,python,forrester/gauss/interactive,pylibkriging,nlpd,-3.11029 +py01_kernels_trends,python,forrester/gauss/quadratic,pylibkriging,fit_s,0.000566483 +py01_kernels_trends,python,forrester/gauss/quadratic,pylibkriging,rmse,1.57249 +py01_kernels_trends,python,forrester/gauss/quadratic,pylibkriging,q2,0.874394 +py01_kernels_trends,python,forrester/gauss/quadratic,pylibkriging,nlpd,5.33879 +py01_kernels_trends,python,forrester/exp/constant,pylibkriging,fit_s,0.00041604 +py01_kernels_trends,python,forrester/exp/constant,pylibkriging,rmse,0.721582 +py01_kernels_trends,python,forrester/exp/constant,pylibkriging,q2,0.973551 +py01_kernels_trends,python,forrester/exp/constant,pylibkriging,nlpd,1.54551 +py01_kernels_trends,python,forrester/exp/linear,pylibkriging,fit_s,0.000438213 +py01_kernels_trends,python,forrester/exp/linear,pylibkriging,rmse,0.793413 +py01_kernels_trends,python,forrester/exp/linear,pylibkriging,q2,0.968024 +py01_kernels_trends,python,forrester/exp/linear,pylibkriging,nlpd,1.50639 +py01_kernels_trends,python,forrester/exp/interactive,pylibkriging,fit_s,0.000480413 +py01_kernels_trends,python,forrester/exp/interactive,pylibkriging,rmse,0.793413 +py01_kernels_trends,python,forrester/exp/interactive,pylibkriging,q2,0.968024 +py01_kernels_trends,python,forrester/exp/interactive,pylibkriging,nlpd,1.50639 +py01_kernels_trends,python,forrester/exp/quadratic,pylibkriging,fit_s,0.000730038 +py01_kernels_trends,python,forrester/exp/quadratic,pylibkriging,rmse,0.723267 +py01_kernels_trends,python,forrester/exp/quadratic,pylibkriging,q2,0.973428 +py01_kernels_trends,python,forrester/exp/quadratic,pylibkriging,nlpd,1.28782 +py01_kernels_trends,python,forrester/matern3_2/constant,pylibkriging,fit_s,0.00060606 +py01_kernels_trends,python,forrester/matern3_2/constant,pylibkriging,rmse,0.334007 +py01_kernels_trends,python,forrester/matern3_2/constant,pylibkriging,q2,0.994333 +py01_kernels_trends,python,forrester/matern3_2/constant,pylibkriging,nlpd,-0.227715 +py01_kernels_trends,python,forrester/matern3_2/linear,pylibkriging,fit_s,0.000588894 +py01_kernels_trends,python,forrester/matern3_2/linear,pylibkriging,rmse,0.364477 +py01_kernels_trends,python,forrester/matern3_2/linear,pylibkriging,q2,0.993252 +py01_kernels_trends,python,forrester/matern3_2/linear,pylibkriging,nlpd,-0.187262 +py01_kernels_trends,python,forrester/matern3_2/interactive,pylibkriging,fit_s,0.000905514 +py01_kernels_trends,python,forrester/matern3_2/interactive,pylibkriging,rmse,0.364477 +py01_kernels_trends,python,forrester/matern3_2/interactive,pylibkriging,q2,0.993252 +py01_kernels_trends,python,forrester/matern3_2/interactive,pylibkriging,nlpd,-0.187262 +py01_kernels_trends,python,forrester/matern3_2/quadratic,pylibkriging,fit_s,0.000975609 +py01_kernels_trends,python,forrester/matern3_2/quadratic,pylibkriging,rmse,0.435609 +py01_kernels_trends,python,forrester/matern3_2/quadratic,pylibkriging,q2,0.990361 +py01_kernels_trends,python,forrester/matern3_2/quadratic,pylibkriging,nlpd,-0.183076 +py01_kernels_trends,python,forrester/matern5_2/constant,pylibkriging,fit_s,0.000726223 +py01_kernels_trends,python,forrester/matern5_2/constant,pylibkriging,rmse,0.281945 +py01_kernels_trends,python,forrester/matern5_2/constant,pylibkriging,q2,0.995962 +py01_kernels_trends,python,forrester/matern5_2/constant,pylibkriging,nlpd,-1.01945 +py01_kernels_trends,python,forrester/matern5_2/constant,pylibkriging BFGS10,rmse,0.281945 +py01_kernels_trends,python,forrester/matern5_2/constant,pylibkriging BFGS10,q2,0.995962 +py01_kernels_trends,python,forrester/matern5_2/constant,pylibkriging BFGS10,nlpd,-1.01945 +py01_kernels_trends,python,forrester/matern5_2/linear,pylibkriging,fit_s,0.000932932 +py01_kernels_trends,python,forrester/matern5_2/linear,pylibkriging,rmse,0.298959 +py01_kernels_trends,python,forrester/matern5_2/linear,pylibkriging,q2,0.99546 +py01_kernels_trends,python,forrester/matern5_2/linear,pylibkriging,nlpd,-0.95551 +py01_kernels_trends,python,forrester/matern5_2/interactive,pylibkriging,fit_s,0.000725269 +py01_kernels_trends,python,forrester/matern5_2/interactive,pylibkriging,rmse,0.298959 +py01_kernels_trends,python,forrester/matern5_2/interactive,pylibkriging,q2,0.99546 +py01_kernels_trends,python,forrester/matern5_2/interactive,pylibkriging,nlpd,-0.95551 +py01_kernels_trends,python,forrester/matern5_2/quadratic,pylibkriging,fit_s,0.00131941 +py01_kernels_trends,python,forrester/matern5_2/quadratic,pylibkriging,rmse,0.365039 +py01_kernels_trends,python,forrester/matern5_2/quadratic,pylibkriging,q2,0.993231 +py01_kernels_trends,python,forrester/matern5_2/quadratic,pylibkriging,nlpd,-0.790032 +py01_kernels_trends,python,gramacy_lee_2012/gauss/constant,pylibkriging,fit_s,0.0016458 +py01_kernels_trends,python,gramacy_lee_2012/gauss/constant,pylibkriging,rmse,0.191274 +py01_kernels_trends,python,gramacy_lee_2012/gauss/constant,pylibkriging,q2,0.978742 +py01_kernels_trends,python,gramacy_lee_2012/gauss/constant,pylibkriging,nlpd,-0.704948 +py01_kernels_trends,python,gramacy_lee_2012/gauss/linear,pylibkriging,fit_s,0.00319505 +py01_kernels_trends,python,gramacy_lee_2012/gauss/linear,pylibkriging,rmse,0.236028 +py01_kernels_trends,python,gramacy_lee_2012/gauss/linear,pylibkriging,q2,0.96763 +py01_kernels_trends,python,gramacy_lee_2012/gauss/linear,pylibkriging,nlpd,-0.580993 +py01_kernels_trends,python,gramacy_lee_2012/gauss/interactive,pylibkriging,fit_s,0.00250673 +py01_kernels_trends,python,gramacy_lee_2012/gauss/interactive,pylibkriging,rmse,0.236028 +py01_kernels_trends,python,gramacy_lee_2012/gauss/interactive,pylibkriging,q2,0.96763 +py01_kernels_trends,python,gramacy_lee_2012/gauss/interactive,pylibkriging,nlpd,-0.580993 +py01_kernels_trends,python,gramacy_lee_2012/gauss/quadratic,pylibkriging,fit_s,0.00243235 +py01_kernels_trends,python,gramacy_lee_2012/gauss/quadratic,pylibkriging,rmse,0.138869 +py01_kernels_trends,python,gramacy_lee_2012/gauss/quadratic,pylibkriging,q2,0.988795 +py01_kernels_trends,python,gramacy_lee_2012/gauss/quadratic,pylibkriging,nlpd,-0.530437 +py01_kernels_trends,python,gramacy_lee_2012/exp/constant,pylibkriging,fit_s,0.000752449 +py01_kernels_trends,python,gramacy_lee_2012/exp/constant,pylibkriging,rmse,0.196586 +py01_kernels_trends,python,gramacy_lee_2012/exp/constant,pylibkriging,q2,0.977545 +py01_kernels_trends,python,gramacy_lee_2012/exp/constant,pylibkriging,nlpd,-0.0621192 +py01_kernels_trends,python,gramacy_lee_2012/exp/linear,pylibkriging,fit_s,0.0010047 +py01_kernels_trends,python,gramacy_lee_2012/exp/linear,pylibkriging,rmse,0.20979 +py01_kernels_trends,python,gramacy_lee_2012/exp/linear,pylibkriging,q2,0.974427 +py01_kernels_trends,python,gramacy_lee_2012/exp/linear,pylibkriging,nlpd,-0.0466568 +py01_kernels_trends,python,gramacy_lee_2012/exp/interactive,pylibkriging,fit_s,0.000931501 +py01_kernels_trends,python,gramacy_lee_2012/exp/interactive,pylibkriging,rmse,0.20979 +py01_kernels_trends,python,gramacy_lee_2012/exp/interactive,pylibkriging,q2,0.974427 +py01_kernels_trends,python,gramacy_lee_2012/exp/interactive,pylibkriging,nlpd,-0.0466568 +py01_kernels_trends,python,gramacy_lee_2012/exp/quadratic,pylibkriging,fit_s,0.0025785 +py01_kernels_trends,python,gramacy_lee_2012/exp/quadratic,pylibkriging,rmse,0.220579 +py01_kernels_trends,python,gramacy_lee_2012/exp/quadratic,pylibkriging,q2,0.971729 +py01_kernels_trends,python,gramacy_lee_2012/exp/quadratic,pylibkriging,nlpd,0.136521 +py01_kernels_trends,python,gramacy_lee_2012/matern3_2/constant,pylibkriging,fit_s,0.000761509 +py01_kernels_trends,python,gramacy_lee_2012/matern3_2/constant,pylibkriging,rmse,0.177253 +py01_kernels_trends,python,gramacy_lee_2012/matern3_2/constant,pylibkriging,q2,0.981744 +py01_kernels_trends,python,gramacy_lee_2012/matern3_2/constant,pylibkriging,nlpd,-0.482212 +py01_kernels_trends,python,gramacy_lee_2012/matern3_2/linear,pylibkriging,fit_s,0.00132823 +py01_kernels_trends,python,gramacy_lee_2012/matern3_2/linear,pylibkriging,rmse,0.211499 +py01_kernels_trends,python,gramacy_lee_2012/matern3_2/linear,pylibkriging,q2,0.974009 +py01_kernels_trends,python,gramacy_lee_2012/matern3_2/linear,pylibkriging,nlpd,-0.375844 +py01_kernels_trends,python,gramacy_lee_2012/matern3_2/interactive,pylibkriging,fit_s,0.000978708 +py01_kernels_trends,python,gramacy_lee_2012/matern3_2/interactive,pylibkriging,rmse,0.211499 +py01_kernels_trends,python,gramacy_lee_2012/matern3_2/interactive,pylibkriging,q2,0.974009 +py01_kernels_trends,python,gramacy_lee_2012/matern3_2/interactive,pylibkriging,nlpd,-0.375844 +py01_kernels_trends,python,gramacy_lee_2012/matern3_2/quadratic,pylibkriging,fit_s,0.00228167 +py01_kernels_trends,python,gramacy_lee_2012/matern3_2/quadratic,pylibkriging,rmse,0.173012 +py01_kernels_trends,python,gramacy_lee_2012/matern3_2/quadratic,pylibkriging,q2,0.982607 +py01_kernels_trends,python,gramacy_lee_2012/matern3_2/quadratic,pylibkriging,nlpd,-0.223308 +py01_kernels_trends,python,gramacy_lee_2012/matern5_2/constant,pylibkriging,fit_s,0.00131416 +py01_kernels_trends,python,gramacy_lee_2012/matern5_2/constant,pylibkriging,rmse,0.177638 +py01_kernels_trends,python,gramacy_lee_2012/matern5_2/constant,pylibkriging,q2,0.981665 +py01_kernels_trends,python,gramacy_lee_2012/matern5_2/constant,pylibkriging,nlpd,-0.575421 +py01_kernels_trends,python,gramacy_lee_2012/matern5_2/constant,pylibkriging BFGS10,rmse,0.177638 +py01_kernels_trends,python,gramacy_lee_2012/matern5_2/constant,pylibkriging BFGS10,q2,0.981665 +py01_kernels_trends,python,gramacy_lee_2012/matern5_2/constant,pylibkriging BFGS10,nlpd,-0.575422 +py01_kernels_trends,python,gramacy_lee_2012/matern5_2/linear,pylibkriging,fit_s,0.00130415 +py01_kernels_trends,python,gramacy_lee_2012/matern5_2/linear,pylibkriging,rmse,0.219759 +py01_kernels_trends,python,gramacy_lee_2012/matern5_2/linear,pylibkriging,q2,0.971939 +py01_kernels_trends,python,gramacy_lee_2012/matern5_2/linear,pylibkriging,nlpd,-0.458933 +py01_kernels_trends,python,gramacy_lee_2012/matern5_2/interactive,pylibkriging,fit_s,0.00119758 +py01_kernels_trends,python,gramacy_lee_2012/matern5_2/interactive,pylibkriging,rmse,0.219759 +py01_kernels_trends,python,gramacy_lee_2012/matern5_2/interactive,pylibkriging,q2,0.971939 +py01_kernels_trends,python,gramacy_lee_2012/matern5_2/interactive,pylibkriging,nlpd,-0.458933 +py01_kernels_trends,python,gramacy_lee_2012/matern5_2/quadratic,pylibkriging,fit_s,0.00169826 +py01_kernels_trends,python,gramacy_lee_2012/matern5_2/quadratic,pylibkriging,rmse,0.160506 +py01_kernels_trends,python,gramacy_lee_2012/matern5_2/quadratic,pylibkriging,q2,0.985031 +py01_kernels_trends,python,gramacy_lee_2012/matern5_2/quadratic,pylibkriging,nlpd,-0.337276 +py01_kernels_trends,python,branin/gauss/constant,pylibkriging,fit_s,0.00104284 +py01_kernels_trends,python,branin/gauss/constant,pylibkriging,rmse,7.2629 +py01_kernels_trends,python,branin/gauss/constant,pylibkriging,q2,0.980655 +py01_kernels_trends,python,branin/gauss/constant,pylibkriging,nlpd,4.3152 +py01_kernels_trends,python,branin/gauss/linear,pylibkriging,fit_s,0.00129008 +py01_kernels_trends,python,branin/gauss/linear,pylibkriging,rmse,7.25733 +py01_kernels_trends,python,branin/gauss/linear,pylibkriging,q2,0.980685 +py01_kernels_trends,python,branin/gauss/linear,pylibkriging,nlpd,4.30813 +py01_kernels_trends,python,branin/gauss/interactive,pylibkriging,fit_s,0.00280094 +py01_kernels_trends,python,branin/gauss/interactive,pylibkriging,rmse,7.2576 +py01_kernels_trends,python,branin/gauss/interactive,pylibkriging,q2,0.980683 +py01_kernels_trends,python,branin/gauss/interactive,pylibkriging,nlpd,4.30176 +py01_kernels_trends,python,branin/gauss/quadratic,pylibkriging,fit_s,0.00112319 +py01_kernels_trends,python,branin/gauss/quadratic,pylibkriging,rmse,7.27808 +py01_kernels_trends,python,branin/gauss/quadratic,pylibkriging,q2,0.980574 +py01_kernels_trends,python,branin/gauss/quadratic,pylibkriging,nlpd,4.29399 +py01_kernels_trends,python,branin/exp/constant,pylibkriging,fit_s,0.00222707 +py01_kernels_trends,python,branin/exp/constant,pylibkriging,rmse,14.5887 +py01_kernels_trends,python,branin/exp/constant,pylibkriging,q2,0.921947 +py01_kernels_trends,python,branin/exp/constant,pylibkriging,nlpd,4.09457 +py01_kernels_trends,python,branin/exp/linear,pylibkriging,fit_s,0.00213099 +py01_kernels_trends,python,branin/exp/linear,pylibkriging,rmse,14.9622 +py01_kernels_trends,python,branin/exp/linear,pylibkriging,q2,0.9179 +py01_kernels_trends,python,branin/exp/linear,pylibkriging,nlpd,4.0909 +py01_kernels_trends,python,branin/exp/interactive,pylibkriging,fit_s,0.00196409 +py01_kernels_trends,python,branin/exp/interactive,pylibkriging,rmse,11.6963 +py01_kernels_trends,python,branin/exp/interactive,pylibkriging,q2,0.94983 +py01_kernels_trends,python,branin/exp/interactive,pylibkriging,nlpd,3.79671 +py01_kernels_trends,python,branin/exp/quadratic,pylibkriging,fit_s,0.00227737 +py01_kernels_trends,python,branin/exp/quadratic,pylibkriging,rmse,9.33909 +py01_kernels_trends,python,branin/exp/quadratic,pylibkriging,q2,0.968014 +py01_kernels_trends,python,branin/exp/quadratic,pylibkriging,nlpd,3.62362 +py01_kernels_trends,python,branin/matern3_2/constant,pylibkriging,fit_s,0.00244212 +py01_kernels_trends,python,branin/matern3_2/constant,pylibkriging,rmse,3.24796 +py01_kernels_trends,python,branin/matern3_2/constant,pylibkriging,q2,0.996131 +py01_kernels_trends,python,branin/matern3_2/constant,pylibkriging,nlpd,2.30444 +py01_kernels_trends,python,branin/matern3_2/linear,pylibkriging,fit_s,0.00244641 +py01_kernels_trends,python,branin/matern3_2/linear,pylibkriging,rmse,3.14402 +py01_kernels_trends,python,branin/matern3_2/linear,pylibkriging,q2,0.996375 +py01_kernels_trends,python,branin/matern3_2/linear,pylibkriging,nlpd,2.28067 +py01_kernels_trends,python,branin/matern3_2/interactive,pylibkriging,fit_s,0.00703263 +py01_kernels_trends,python,branin/matern3_2/interactive,pylibkriging,rmse,3.19226 +py01_kernels_trends,python,branin/matern3_2/interactive,pylibkriging,q2,0.996263 +py01_kernels_trends,python,branin/matern3_2/interactive,pylibkriging,nlpd,2.20359 +py01_kernels_trends,python,branin/matern3_2/quadratic,pylibkriging,fit_s,0.00404716 +py01_kernels_trends,python,branin/matern3_2/quadratic,pylibkriging,rmse,3.01232 +py01_kernels_trends,python,branin/matern3_2/quadratic,pylibkriging,q2,0.996672 +py01_kernels_trends,python,branin/matern3_2/quadratic,pylibkriging,nlpd,2.11627 +py01_kernels_trends,python,branin/matern5_2/constant,pylibkriging,fit_s,0.0029068 +py01_kernels_trends,python,branin/matern5_2/constant,pylibkriging,rmse,0.52012 +py01_kernels_trends,python,branin/matern5_2/constant,pylibkriging,q2,0.999901 +py01_kernels_trends,python,branin/matern5_2/constant,pylibkriging,nlpd,0.283399 +py01_kernels_trends,python,branin/matern5_2/constant,pylibkriging BFGS10,rmse,0.52012 +py01_kernels_trends,python,branin/matern5_2/constant,pylibkriging BFGS10,q2,0.999901 +py01_kernels_trends,python,branin/matern5_2/constant,pylibkriging BFGS10,nlpd,0.283395 +py01_kernels_trends,python,branin/matern5_2/linear,pylibkriging,fit_s,0.00212407 +py01_kernels_trends,python,branin/matern5_2/linear,pylibkriging,rmse,0.460868 +py01_kernels_trends,python,branin/matern5_2/linear,pylibkriging,q2,0.999922 +py01_kernels_trends,python,branin/matern5_2/linear,pylibkriging,nlpd,0.209741 +py01_kernels_trends,python,branin/matern5_2/interactive,pylibkriging,fit_s,0.0018971 +py01_kernels_trends,python,branin/matern5_2/interactive,pylibkriging,rmse,0.498484 +py01_kernels_trends,python,branin/matern5_2/interactive,pylibkriging,q2,0.999909 +py01_kernels_trends,python,branin/matern5_2/interactive,pylibkriging,nlpd,0.239079 +py01_kernels_trends,python,branin/matern5_2/quadratic,pylibkriging,fit_s,0.00570393 +py01_kernels_trends,python,branin/matern5_2/quadratic,pylibkriging,rmse,0.67472 +py01_kernels_trends,python,branin/matern5_2/quadratic,pylibkriging,q2,0.999833 +py01_kernels_trends,python,branin/matern5_2/quadratic,pylibkriging,nlpd,0.419266 +py01_kernels_trends,python,six_hump_camel/gauss/constant,pylibkriging,fit_s,0.00197244 +py01_kernels_trends,python,six_hump_camel/gauss/constant,pylibkriging,rmse,18.9501 +py01_kernels_trends,python,six_hump_camel/gauss/constant,pylibkriging,q2,0.536354 +py01_kernels_trends,python,six_hump_camel/gauss/constant,pylibkriging,nlpd,84.151 +py01_kernels_trends,python,six_hump_camel/gauss/linear,pylibkriging,fit_s,0.00170588 +py01_kernels_trends,python,six_hump_camel/gauss/linear,pylibkriging,rmse,18.9579 +py01_kernels_trends,python,six_hump_camel/gauss/linear,pylibkriging,q2,0.535976 +py01_kernels_trends,python,six_hump_camel/gauss/linear,pylibkriging,nlpd,44.8696 +py01_kernels_trends,python,six_hump_camel/gauss/interactive,pylibkriging,fit_s,0.0020082 +py01_kernels_trends,python,six_hump_camel/gauss/interactive,pylibkriging,rmse,18.9819 +py01_kernels_trends,python,six_hump_camel/gauss/interactive,pylibkriging,q2,0.5348 +py01_kernels_trends,python,six_hump_camel/gauss/interactive,pylibkriging,nlpd,35.2765 +py01_kernels_trends,python,six_hump_camel/gauss/quadratic,pylibkriging,fit_s,0.00248289 +py01_kernels_trends,python,six_hump_camel/gauss/quadratic,pylibkriging,rmse,10.9058 +py01_kernels_trends,python,six_hump_camel/gauss/quadratic,pylibkriging,q2,0.846441 +py01_kernels_trends,python,six_hump_camel/gauss/quadratic,pylibkriging,nlpd,4.92017 +py01_kernels_trends,python,six_hump_camel/exp/constant,pylibkriging,fit_s,0.00100708 +py01_kernels_trends,python,six_hump_camel/exp/constant,pylibkriging,rmse,16.9653 +py01_kernels_trends,python,six_hump_camel/exp/constant,pylibkriging,q2,0.628393 +py01_kernels_trends,python,six_hump_camel/exp/constant,pylibkriging,nlpd,4.48595 +py01_kernels_trends,python,six_hump_camel/exp/linear,pylibkriging,fit_s,0.00118208 +py01_kernels_trends,python,six_hump_camel/exp/linear,pylibkriging,rmse,17.1781 +py01_kernels_trends,python,six_hump_camel/exp/linear,pylibkriging,q2,0.619012 +py01_kernels_trends,python,six_hump_camel/exp/linear,pylibkriging,nlpd,4.47061 +py01_kernels_trends,python,six_hump_camel/exp/interactive,pylibkriging,fit_s,0.00123763 +py01_kernels_trends,python,six_hump_camel/exp/interactive,pylibkriging,rmse,17.1856 +py01_kernels_trends,python,six_hump_camel/exp/interactive,pylibkriging,q2,0.618679 +py01_kernels_trends,python,six_hump_camel/exp/interactive,pylibkriging,nlpd,4.43133 +py01_kernels_trends,python,six_hump_camel/exp/quadratic,pylibkriging,fit_s,0.0013876 +py01_kernels_trends,python,six_hump_camel/exp/quadratic,pylibkriging,rmse,7.99484 +py01_kernels_trends,python,six_hump_camel/exp/quadratic,pylibkriging,q2,0.917476 +py01_kernels_trends,python,six_hump_camel/exp/quadratic,pylibkriging,nlpd,3.35612 +py01_kernels_trends,python,six_hump_camel/matern3_2/constant,pylibkriging,fit_s,0.00175476 +py01_kernels_trends,python,six_hump_camel/matern3_2/constant,pylibkriging,rmse,18.4958 +py01_kernels_trends,python,six_hump_camel/matern3_2/constant,pylibkriging,q2,0.558321 +py01_kernels_trends,python,six_hump_camel/matern3_2/constant,pylibkriging,nlpd,18.8349 +py01_kernels_trends,python,six_hump_camel/matern3_2/linear,pylibkriging,fit_s,0.00206542 +py01_kernels_trends,python,six_hump_camel/matern3_2/linear,pylibkriging,rmse,18.5562 +py01_kernels_trends,python,six_hump_camel/matern3_2/linear,pylibkriging,q2,0.555433 +py01_kernels_trends,python,six_hump_camel/matern3_2/linear,pylibkriging,nlpd,14.9494 +py01_kernels_trends,python,six_hump_camel/matern3_2/interactive,pylibkriging,fit_s,0.00178266 +py01_kernels_trends,python,six_hump_camel/matern3_2/interactive,pylibkriging,rmse,18.561 +py01_kernels_trends,python,six_hump_camel/matern3_2/interactive,pylibkriging,q2,0.555202 +py01_kernels_trends,python,six_hump_camel/matern3_2/interactive,pylibkriging,nlpd,12.9923 +py01_kernels_trends,python,six_hump_camel/matern3_2/quadratic,pylibkriging,fit_s,0.00338221 +py01_kernels_trends,python,six_hump_camel/matern3_2/quadratic,pylibkriging,rmse,8.91645 +py01_kernels_trends,python,six_hump_camel/matern3_2/quadratic,pylibkriging,q2,0.897353 +py01_kernels_trends,python,six_hump_camel/matern3_2/quadratic,pylibkriging,nlpd,3.94939 +py01_kernels_trends,python,six_hump_camel/matern5_2/constant,pylibkriging,fit_s,0.00228882 +py01_kernels_trends,python,six_hump_camel/matern5_2/constant,pylibkriging,rmse,18.3236 +py01_kernels_trends,python,six_hump_camel/matern5_2/constant,pylibkriging,q2,0.566507 +py01_kernels_trends,python,six_hump_camel/matern5_2/constant,pylibkriging,nlpd,18.4322 +py01_kernels_trends,python,six_hump_camel/matern5_2/constant,pylibkriging BFGS10,rmse,1.36208 +py01_kernels_trends,python,six_hump_camel/matern5_2/constant,pylibkriging BFGS10,q2,0.997605 +py01_kernels_trends,python,six_hump_camel/matern5_2/constant,pylibkriging BFGS10,nlpd,1.03215 +py01_kernels_trends,python,six_hump_camel/matern5_2/linear,pylibkriging,fit_s,0.00242186 +py01_kernels_trends,python,six_hump_camel/matern5_2/linear,pylibkriging,rmse,18.7127 +py01_kernels_trends,python,six_hump_camel/matern5_2/linear,pylibkriging,q2,0.547901 +py01_kernels_trends,python,six_hump_camel/matern5_2/linear,pylibkriging,nlpd,24.5771 +py01_kernels_trends,python,six_hump_camel/matern5_2/interactive,pylibkriging,fit_s,0.00234485 +py01_kernels_trends,python,six_hump_camel/matern5_2/interactive,pylibkriging,rmse,18.7194 +py01_kernels_trends,python,six_hump_camel/matern5_2/interactive,pylibkriging,q2,0.547577 +py01_kernels_trends,python,six_hump_camel/matern5_2/interactive,pylibkriging,nlpd,20.4255 +py01_kernels_trends,python,six_hump_camel/matern5_2/quadratic,pylibkriging,fit_s,0.00251341 +py01_kernels_trends,python,six_hump_camel/matern5_2/quadratic,pylibkriging,rmse,10.2878 +py01_kernels_trends,python,six_hump_camel/matern5_2/quadratic,pylibkriging,q2,0.863351 +py01_kernels_trends,python,six_hump_camel/matern5_2/quadratic,pylibkriging,nlpd,4.46344 +py01_kernels_trends,python,log_goldstein_price/gauss/constant,pylibkriging,fit_s,0.00588727 +py01_kernels_trends,python,log_goldstein_price/gauss/constant,pylibkriging,rmse,0.295934 +py01_kernels_trends,python,log_goldstein_price/gauss/constant,pylibkriging,q2,0.920537 +py01_kernels_trends,python,log_goldstein_price/gauss/constant,pylibkriging,nlpd,1.58666 +py01_kernels_trends,python,log_goldstein_price/gauss/linear,pylibkriging,fit_s,0.00581551 +py01_kernels_trends,python,log_goldstein_price/gauss/linear,pylibkriging,rmse,0.296355 +py01_kernels_trends,python,log_goldstein_price/gauss/linear,pylibkriging,q2,0.92031 +py01_kernels_trends,python,log_goldstein_price/gauss/linear,pylibkriging,nlpd,1.2962 +py01_kernels_trends,python,log_goldstein_price/gauss/interactive,pylibkriging,fit_s,0.00644922 +py01_kernels_trends,python,log_goldstein_price/gauss/interactive,pylibkriging,rmse,0.285136 +py01_kernels_trends,python,log_goldstein_price/gauss/interactive,pylibkriging,q2,0.926229 +py01_kernels_trends,python,log_goldstein_price/gauss/interactive,pylibkriging,nlpd,1.04122 +py01_kernels_trends,python,log_goldstein_price/gauss/quadratic,pylibkriging,fit_s,0.00578475 +py01_kernels_trends,python,log_goldstein_price/gauss/quadratic,pylibkriging,rmse,0.237308 +py01_kernels_trends,python,log_goldstein_price/gauss/quadratic,pylibkriging,q2,0.948902 +py01_kernels_trends,python,log_goldstein_price/gauss/quadratic,pylibkriging,nlpd,0.146484 +py01_kernels_trends,python,log_goldstein_price/exp/constant,pylibkriging,fit_s,0.00258231 +py01_kernels_trends,python,log_goldstein_price/exp/constant,pylibkriging,rmse,0.334876 +py01_kernels_trends,python,log_goldstein_price/exp/constant,pylibkriging,q2,0.898247 +py01_kernels_trends,python,log_goldstein_price/exp/constant,pylibkriging,nlpd,0.339235 +py01_kernels_trends,python,log_goldstein_price/exp/linear,pylibkriging,fit_s,0.00211 +py01_kernels_trends,python,log_goldstein_price/exp/linear,pylibkriging,rmse,0.333447 +py01_kernels_trends,python,log_goldstein_price/exp/linear,pylibkriging,q2,0.899114 +py01_kernels_trends,python,log_goldstein_price/exp/linear,pylibkriging,nlpd,0.336515 +py01_kernels_trends,python,log_goldstein_price/exp/interactive,pylibkriging,fit_s,0.0025239 +py01_kernels_trends,python,log_goldstein_price/exp/interactive,pylibkriging,rmse,0.336819 +py01_kernels_trends,python,log_goldstein_price/exp/interactive,pylibkriging,q2,0.897063 +py01_kernels_trends,python,log_goldstein_price/exp/interactive,pylibkriging,nlpd,0.329566 +py01_kernels_trends,python,log_goldstein_price/exp/quadratic,pylibkriging,fit_s,0.00309515 +py01_kernels_trends,python,log_goldstein_price/exp/quadratic,pylibkriging,rmse,0.327816 +py01_kernels_trends,python,log_goldstein_price/exp/quadratic,pylibkriging,q2,0.902492 +py01_kernels_trends,python,log_goldstein_price/exp/quadratic,pylibkriging,nlpd,0.322056 +py01_kernels_trends,python,log_goldstein_price/matern3_2/constant,pylibkriging,fit_s,0.00425816 +py01_kernels_trends,python,log_goldstein_price/matern3_2/constant,pylibkriging,rmse,0.684362 +py01_kernels_trends,python,log_goldstein_price/matern3_2/constant,pylibkriging,q2,0.575038 +py01_kernels_trends,python,log_goldstein_price/matern3_2/constant,pylibkriging,nlpd,0.972803 +py01_kernels_trends,python,log_goldstein_price/matern3_2/linear,pylibkriging,fit_s,0.00413442 +py01_kernels_trends,python,log_goldstein_price/matern3_2/linear,pylibkriging,rmse,0.650201 +py01_kernels_trends,python,log_goldstein_price/matern3_2/linear,pylibkriging,q2,0.616404 +py01_kernels_trends,python,log_goldstein_price/matern3_2/linear,pylibkriging,nlpd,0.921455 +py01_kernels_trends,python,log_goldstein_price/matern3_2/interactive,pylibkriging,fit_s,0.00704145 +py01_kernels_trends,python,log_goldstein_price/matern3_2/interactive,pylibkriging,rmse,0.274326 +py01_kernels_trends,python,log_goldstein_price/matern3_2/interactive,pylibkriging,q2,0.931717 +py01_kernels_trends,python,log_goldstein_price/matern3_2/interactive,pylibkriging,nlpd,0.0549679 +py01_kernels_trends,python,log_goldstein_price/matern3_2/quadratic,pylibkriging,fit_s,0.00996184 +py01_kernels_trends,python,log_goldstein_price/matern3_2/quadratic,pylibkriging,rmse,0.263745 +py01_kernels_trends,python,log_goldstein_price/matern3_2/quadratic,pylibkriging,q2,0.936883 +py01_kernels_trends,python,log_goldstein_price/matern3_2/quadratic,pylibkriging,nlpd,0.0446267 +py01_kernels_trends,python,log_goldstein_price/matern5_2/constant,pylibkriging,fit_s,0.00410438 +py01_kernels_trends,python,log_goldstein_price/matern5_2/constant,pylibkriging,rmse,0.674391 +py01_kernels_trends,python,log_goldstein_price/matern5_2/constant,pylibkriging,q2,0.58733 +py01_kernels_trends,python,log_goldstein_price/matern5_2/constant,pylibkriging,nlpd,0.944765 +py01_kernels_trends,python,log_goldstein_price/matern5_2/constant,pylibkriging BFGS10,rmse,0.268216 +py01_kernels_trends,python,log_goldstein_price/matern5_2/constant,pylibkriging BFGS10,q2,0.934725 +py01_kernels_trends,python,log_goldstein_price/matern5_2/constant,pylibkriging BFGS10,nlpd,0.169585 +py01_kernels_trends,python,log_goldstein_price/matern5_2/linear,pylibkriging,fit_s,0.00728607 +py01_kernels_trends,python,log_goldstein_price/matern5_2/linear,pylibkriging,rmse,0.268808 +py01_kernels_trends,python,log_goldstein_price/matern5_2/linear,pylibkriging,q2,0.934436 +py01_kernels_trends,python,log_goldstein_price/matern5_2/linear,pylibkriging,nlpd,0.142382 +py01_kernels_trends,python,log_goldstein_price/matern5_2/interactive,pylibkriging,fit_s,0.00524259 +py01_kernels_trends,python,log_goldstein_price/matern5_2/interactive,pylibkriging,rmse,0.271088 +py01_kernels_trends,python,log_goldstein_price/matern5_2/interactive,pylibkriging,q2,0.933319 +py01_kernels_trends,python,log_goldstein_price/matern5_2/interactive,pylibkriging,nlpd,0.128788 +py01_kernels_trends,python,log_goldstein_price/matern5_2/quadratic,pylibkriging,fit_s,0.011641 +py01_kernels_trends,python,log_goldstein_price/matern5_2/quadratic,pylibkriging,rmse,0.253093 +py01_kernels_trends,python,log_goldstein_price/matern5_2/quadratic,pylibkriging,q2,0.941878 +py01_kernels_trends,python,log_goldstein_price/matern5_2/quadratic,pylibkriging,nlpd,0.025469 +py01_kernels_trends,python,lim_polynomial/gauss/constant,pylibkriging,fit_s,0.00113583 +py01_kernels_trends,python,lim_polynomial/gauss/constant,pylibkriging,rmse,0.026127 +py01_kernels_trends,python,lim_polynomial/gauss/constant,pylibkriging,q2,0.999737 +py01_kernels_trends,python,lim_polynomial/gauss/constant,pylibkriging,nlpd,-2.69817 +py01_kernels_trends,python,lim_polynomial/gauss/linear,pylibkriging,fit_s,0.00152946 +py01_kernels_trends,python,lim_polynomial/gauss/linear,pylibkriging,rmse,1.06243 +py01_kernels_trends,python,lim_polynomial/gauss/linear,pylibkriging,q2,0.564691 +py01_kernels_trends,python,lim_polynomial/gauss/linear,pylibkriging,nlpd,2.7394 +py01_kernels_trends,python,lim_polynomial/gauss/interactive,pylibkriging,fit_s,0.00153446 +py01_kernels_trends,python,lim_polynomial/gauss/interactive,pylibkriging,rmse,1.06504 +py01_kernels_trends,python,lim_polynomial/gauss/interactive,pylibkriging,q2,0.562551 +py01_kernels_trends,python,lim_polynomial/gauss/interactive,pylibkriging,nlpd,2.33575 +py01_kernels_trends,python,lim_polynomial/gauss/quadratic,pylibkriging,fit_s,0.00196862 +py01_kernels_trends,python,lim_polynomial/gauss/quadratic,pylibkriging,rmse,0.281816 +py01_kernels_trends,python,lim_polynomial/gauss/quadratic,pylibkriging,q2,0.969371 +py01_kernels_trends,python,lim_polynomial/gauss/quadratic,pylibkriging,nlpd,0.71682 +py01_kernels_trends,python,lim_polynomial/exp/constant,pylibkriging,fit_s,0.000828028 +py01_kernels_trends,python,lim_polynomial/exp/constant,pylibkriging,rmse,0.249023 +py01_kernels_trends,python,lim_polynomial/exp/constant,pylibkriging,q2,0.976085 +py01_kernels_trends,python,lim_polynomial/exp/constant,pylibkriging,nlpd,0.394159 +py01_kernels_trends,python,lim_polynomial/exp/linear,pylibkriging,fit_s,0.000962973 +py01_kernels_trends,python,lim_polynomial/exp/linear,pylibkriging,rmse,0.245558 +py01_kernels_trends,python,lim_polynomial/exp/linear,pylibkriging,q2,0.976746 +py01_kernels_trends,python,lim_polynomial/exp/linear,pylibkriging,nlpd,0.224593 +py01_kernels_trends,python,lim_polynomial/exp/interactive,pylibkriging,fit_s,0.00143671 +py01_kernels_trends,python,lim_polynomial/exp/interactive,pylibkriging,rmse,0.239032 +py01_kernels_trends,python,lim_polynomial/exp/interactive,pylibkriging,q2,0.977965 +py01_kernels_trends,python,lim_polynomial/exp/interactive,pylibkriging,nlpd,0.226133 +py01_kernels_trends,python,lim_polynomial/exp/quadratic,pylibkriging,fit_s,0.00274634 +py01_kernels_trends,python,lim_polynomial/exp/quadratic,pylibkriging,rmse,0.237713 +py01_kernels_trends,python,lim_polynomial/exp/quadratic,pylibkriging,q2,0.978208 +py01_kernels_trends,python,lim_polynomial/exp/quadratic,pylibkriging,nlpd,-0.139582 +py01_kernels_trends,python,lim_polynomial/matern3_2/constant,pylibkriging,fit_s,0.0011251 +py01_kernels_trends,python,lim_polynomial/matern3_2/constant,pylibkriging,rmse,0.0600911 +py01_kernels_trends,python,lim_polynomial/matern3_2/constant,pylibkriging,q2,0.998607 +py01_kernels_trends,python,lim_polynomial/matern3_2/constant,pylibkriging,nlpd,-1.34658 +py01_kernels_trends,python,lim_polynomial/matern3_2/linear,pylibkriging,fit_s,0.000877857 +py01_kernels_trends,python,lim_polynomial/matern3_2/linear,pylibkriging,rmse,0.0584376 +py01_kernels_trends,python,lim_polynomial/matern3_2/linear,pylibkriging,q2,0.998683 +py01_kernels_trends,python,lim_polynomial/matern3_2/linear,pylibkriging,nlpd,-1.34912 +py01_kernels_trends,python,lim_polynomial/matern3_2/interactive,pylibkriging,fit_s,0.00140738 +py01_kernels_trends,python,lim_polynomial/matern3_2/interactive,pylibkriging,rmse,0.0576096 +py01_kernels_trends,python,lim_polynomial/matern3_2/interactive,pylibkriging,q2,0.99872 +py01_kernels_trends,python,lim_polynomial/matern3_2/interactive,pylibkriging,nlpd,-1.35376 +py01_kernels_trends,python,lim_polynomial/matern3_2/quadratic,pylibkriging,fit_s,0.00383472 +py01_kernels_trends,python,lim_polynomial/matern3_2/quadratic,pylibkriging,rmse,0.215336 +py01_kernels_trends,python,lim_polynomial/matern3_2/quadratic,pylibkriging,q2,0.982117 +py01_kernels_trends,python,lim_polynomial/matern3_2/quadratic,pylibkriging,nlpd,-0.350837 +py01_kernels_trends,python,lim_polynomial/matern5_2/constant,pylibkriging,fit_s,0.000718832 +py01_kernels_trends,python,lim_polynomial/matern5_2/constant,pylibkriging,rmse,0.0246348 +py01_kernels_trends,python,lim_polynomial/matern5_2/constant,pylibkriging,q2,0.999766 +py01_kernels_trends,python,lim_polynomial/matern5_2/constant,pylibkriging,nlpd,-2.59598 +py01_kernels_trends,python,lim_polynomial/matern5_2/constant,pylibkriging BFGS10,rmse,0.0246338 +py01_kernels_trends,python,lim_polynomial/matern5_2/constant,pylibkriging BFGS10,q2,0.999766 +py01_kernels_trends,python,lim_polynomial/matern5_2/constant,pylibkriging BFGS10,nlpd,-2.59598 +py01_kernels_trends,python,lim_polynomial/matern5_2/linear,pylibkriging,fit_s,0.0013113 +py01_kernels_trends,python,lim_polynomial/matern5_2/linear,pylibkriging,rmse,0.0232483 +py01_kernels_trends,python,lim_polynomial/matern5_2/linear,pylibkriging,q2,0.999792 +py01_kernels_trends,python,lim_polynomial/matern5_2/linear,pylibkriging,nlpd,-2.58843 +py01_kernels_trends,python,lim_polynomial/matern5_2/interactive,pylibkriging,fit_s,0.00150442 +py01_kernels_trends,python,lim_polynomial/matern5_2/interactive,pylibkriging,rmse,0.0230248 +py01_kernels_trends,python,lim_polynomial/matern5_2/interactive,pylibkriging,q2,0.999796 +py01_kernels_trends,python,lim_polynomial/matern5_2/interactive,pylibkriging,nlpd,-2.58839 +py01_kernels_trends,python,lim_polynomial/matern5_2/quadratic,pylibkriging,fit_s,0.00300908 +py01_kernels_trends,python,lim_polynomial/matern5_2/quadratic,pylibkriging,rmse,0.209802 +py01_kernels_trends,python,lim_polynomial/matern5_2/quadratic,pylibkriging,q2,0.983025 +py01_kernels_trends,python,lim_polynomial/matern5_2/quadratic,pylibkriging,nlpd,-0.403998 +py01_kernels_trends,python,ishigami/gauss/constant,pylibkriging,fit_s,0.0101857 +py01_kernels_trends,python,ishigami/gauss/constant,pylibkriging,rmse,2.07486 +py01_kernels_trends,python,ishigami/gauss/constant,pylibkriging,q2,0.668191 +py01_kernels_trends,python,ishigami/gauss/constant,pylibkriging,nlpd,2.80326 +py01_kernels_trends,python,ishigami/gauss/linear,pylibkriging,fit_s,0.0193923 +py01_kernels_trends,python,ishigami/gauss/linear,pylibkriging,rmse,2.8459 +py01_kernels_trends,python,ishigami/gauss/linear,pylibkriging,q2,0.375762 +py01_kernels_trends,python,ishigami/gauss/linear,pylibkriging,nlpd,3.41919 +py01_kernels_trends,python,ishigami/gauss/interactive,pylibkriging,fit_s,0.0174267 +py01_kernels_trends,python,ishigami/gauss/interactive,pylibkriging,rmse,2.85366 +py01_kernels_trends,python,ishigami/gauss/interactive,pylibkriging,q2,0.372353 +py01_kernels_trends,python,ishigami/gauss/interactive,pylibkriging,nlpd,3.28643 +py01_kernels_trends,python,ishigami/gauss/quadratic,pylibkriging,fit_s,0.00865173 +py01_kernels_trends,python,ishigami/gauss/quadratic,pylibkriging,rmse,2.08981 +py01_kernels_trends,python,ishigami/gauss/quadratic,pylibkriging,q2,0.663392 +py01_kernels_trends,python,ishigami/gauss/quadratic,pylibkriging,nlpd,2.80097 +py01_kernels_trends,python,ishigami/exp/constant,pylibkriging,fit_s,0.00899458 +py01_kernels_trends,python,ishigami/exp/constant,pylibkriging,rmse,1.41632 +py01_kernels_trends,python,ishigami/exp/constant,pylibkriging,q2,0.845391 +py01_kernels_trends,python,ishigami/exp/constant,pylibkriging,nlpd,1.75239 +py01_kernels_trends,python,ishigami/exp/linear,pylibkriging,fit_s,0.0115979 +py01_kernels_trends,python,ishigami/exp/linear,pylibkriging,rmse,1.4095 +py01_kernels_trends,python,ishigami/exp/linear,pylibkriging,q2,0.846877 +py01_kernels_trends,python,ishigami/exp/linear,pylibkriging,nlpd,1.74779 +py01_kernels_trends,python,ishigami/exp/interactive,pylibkriging,fit_s,0.00745058 +py01_kernels_trends,python,ishigami/exp/interactive,pylibkriging,rmse,1.41511 +py01_kernels_trends,python,ishigami/exp/interactive,pylibkriging,q2,0.845657 +py01_kernels_trends,python,ishigami/exp/interactive,pylibkriging,nlpd,1.75024 +py01_kernels_trends,python,ishigami/exp/quadratic,pylibkriging,fit_s,0.00843048 +py01_kernels_trends,python,ishigami/exp/quadratic,pylibkriging,rmse,1.40329 +py01_kernels_trends,python,ishigami/exp/quadratic,pylibkriging,q2,0.848223 +py01_kernels_trends,python,ishigami/exp/quadratic,pylibkriging,nlpd,1.73254 +py01_kernels_trends,python,ishigami/matern3_2/constant,pylibkriging,fit_s,0.0480943 +py01_kernels_trends,python,ishigami/matern3_2/constant,pylibkriging,rmse,1.32971 +py01_kernels_trends,python,ishigami/matern3_2/constant,pylibkriging,q2,0.863723 +py01_kernels_trends,python,ishigami/matern3_2/constant,pylibkriging,nlpd,1.50265 +py01_kernels_trends,python,ishigami/matern3_2/linear,pylibkriging,fit_s,0.047653 +py01_kernels_trends,python,ishigami/matern3_2/linear,pylibkriging,rmse,1.30244 +py01_kernels_trends,python,ishigami/matern3_2/linear,pylibkriging,q2,0.869255 +py01_kernels_trends,python,ishigami/matern3_2/linear,pylibkriging,nlpd,1.47906 +py01_kernels_trends,python,ishigami/matern3_2/interactive,pylibkriging,fit_s,0.060349 +py01_kernels_trends,python,ishigami/matern3_2/interactive,pylibkriging,rmse,1.30591 +py01_kernels_trends,python,ishigami/matern3_2/interactive,pylibkriging,q2,0.868558 +py01_kernels_trends,python,ishigami/matern3_2/interactive,pylibkriging,nlpd,1.47384 +py01_kernels_trends,python,ishigami/matern3_2/quadratic,pylibkriging,fit_s,0.0471144 +py01_kernels_trends,python,ishigami/matern3_2/quadratic,pylibkriging,rmse,1.32197 +py01_kernels_trends,python,ishigami/matern3_2/quadratic,pylibkriging,q2,0.865305 +py01_kernels_trends,python,ishigami/matern3_2/quadratic,pylibkriging,nlpd,1.46827 +py01_kernels_trends,python,ishigami/matern5_2/constant,pylibkriging,fit_s,0.0145204 +py01_kernels_trends,python,ishigami/matern5_2/constant,pylibkriging,rmse,3.00566 +py01_kernels_trends,python,ishigami/matern5_2/constant,pylibkriging,q2,0.303709 +py01_kernels_trends,python,ishigami/matern5_2/constant,pylibkriging,nlpd,3.42085 +py01_kernels_trends,python,ishigami/matern5_2/constant,pylibkriging BFGS10,rmse,1.33903 +py01_kernels_trends,python,ishigami/matern5_2/constant,pylibkriging BFGS10,q2,0.861806 +py01_kernels_trends,python,ishigami/matern5_2/constant,pylibkriging BFGS10,nlpd,1.66132 +py01_kernels_trends,python,ishigami/matern5_2/linear,pylibkriging,fit_s,0.0518982 +py01_kernels_trends,python,ishigami/matern5_2/linear,pylibkriging,rmse,1.3434 +py01_kernels_trends,python,ishigami/matern5_2/linear,pylibkriging,q2,0.860901 +py01_kernels_trends,python,ishigami/matern5_2/linear,pylibkriging,nlpd,1.65103 +py01_kernels_trends,python,ishigami/matern5_2/interactive,pylibkriging,fit_s,0.0539804 +py01_kernels_trends,python,ishigami/matern5_2/interactive,pylibkriging,rmse,1.34479 +py01_kernels_trends,python,ishigami/matern5_2/interactive,pylibkriging,q2,0.860615 +py01_kernels_trends,python,ishigami/matern5_2/interactive,pylibkriging,nlpd,1.64153 +py01_kernels_trends,python,ishigami/matern5_2/quadratic,pylibkriging,fit_s,0.052948 +py01_kernels_trends,python,ishigami/matern5_2/quadratic,pylibkriging,rmse,1.37082 +py01_kernels_trends,python,ishigami/matern5_2/quadratic,pylibkriging,q2,0.855166 +py01_kernels_trends,python,ishigami/matern5_2/quadratic,pylibkriging,nlpd,1.62228 +py01_kernels_trends,python,hartmann3/gauss/constant,pylibkriging,fit_s,0.00805306 +py01_kernels_trends,python,hartmann3/gauss/constant,pylibkriging,rmse,0.156226 +py01_kernels_trends,python,hartmann3/gauss/constant,pylibkriging,q2,0.971116 +py01_kernels_trends,python,hartmann3/gauss/constant,pylibkriging,nlpd,-0.555623 +py01_kernels_trends,python,hartmann3/gauss/linear,pylibkriging,fit_s,0.0129044 +py01_kernels_trends,python,hartmann3/gauss/linear,pylibkriging,rmse,0.153297 +py01_kernels_trends,python,hartmann3/gauss/linear,pylibkriging,q2,0.972189 +py01_kernels_trends,python,hartmann3/gauss/linear,pylibkriging,nlpd,-0.568404 +py01_kernels_trends,python,hartmann3/gauss/interactive,pylibkriging,fit_s,0.00773382 +py01_kernels_trends,python,hartmann3/gauss/interactive,pylibkriging,rmse,0.712948 +py01_kernels_trends,python,hartmann3/gauss/interactive,pylibkriging,q2,0.398455 +py01_kernels_trends,python,hartmann3/gauss/interactive,pylibkriging,nlpd,1.74554 +py01_kernels_trends,python,hartmann3/gauss/quadratic,pylibkriging,fit_s,0.00593877 +py01_kernels_trends,python,hartmann3/gauss/quadratic,pylibkriging,rmse,0.73238 +py01_kernels_trends,python,hartmann3/gauss/quadratic,pylibkriging,q2,0.365217 +py01_kernels_trends,python,hartmann3/gauss/quadratic,pylibkriging,nlpd,2.14587 +py01_kernels_trends,python,hartmann3/exp/constant,pylibkriging,fit_s,0.00374722 +py01_kernels_trends,python,hartmann3/exp/constant,pylibkriging,rmse,0.347348 +py01_kernels_trends,python,hartmann3/exp/constant,pylibkriging,q2,0.857215 +py01_kernels_trends,python,hartmann3/exp/constant,pylibkriging,nlpd,0.352683 +py01_kernels_trends,python,hartmann3/exp/linear,pylibkriging,fit_s,0.00216174 +py01_kernels_trends,python,hartmann3/exp/linear,pylibkriging,rmse,0.347273 +py01_kernels_trends,python,hartmann3/exp/linear,pylibkriging,q2,0.857276 +py01_kernels_trends,python,hartmann3/exp/linear,pylibkriging,nlpd,0.343631 +py01_kernels_trends,python,hartmann3/exp/interactive,pylibkriging,fit_s,0.0024302 +py01_kernels_trends,python,hartmann3/exp/interactive,pylibkriging,rmse,0.378349 +py01_kernels_trends,python,hartmann3/exp/interactive,pylibkriging,q2,0.83059 +py01_kernels_trends,python,hartmann3/exp/interactive,pylibkriging,nlpd,0.385842 +py01_kernels_trends,python,hartmann3/exp/quadratic,pylibkriging,fit_s,0.0039556 +py01_kernels_trends,python,hartmann3/exp/quadratic,pylibkriging,rmse,0.355119 +py01_kernels_trends,python,hartmann3/exp/quadratic,pylibkriging,q2,0.850755 +py01_kernels_trends,python,hartmann3/exp/quadratic,pylibkriging,nlpd,0.334371 +py01_kernels_trends,python,hartmann3/matern3_2/constant,pylibkriging,fit_s,0.0110524 +py01_kernels_trends,python,hartmann3/matern3_2/constant,pylibkriging,rmse,0.227521 +py01_kernels_trends,python,hartmann3/matern3_2/constant,pylibkriging,q2,0.938737 +py01_kernels_trends,python,hartmann3/matern3_2/constant,pylibkriging,nlpd,-0.181207 +py01_kernels_trends,python,hartmann3/matern3_2/linear,pylibkriging,fit_s,0.00763154 +py01_kernels_trends,python,hartmann3/matern3_2/linear,pylibkriging,rmse,0.23038 +py01_kernels_trends,python,hartmann3/matern3_2/linear,pylibkriging,q2,0.937188 +py01_kernels_trends,python,hartmann3/matern3_2/linear,pylibkriging,nlpd,-0.162203 +py01_kernels_trends,python,hartmann3/matern3_2/interactive,pylibkriging,fit_s,0.00412679 +py01_kernels_trends,python,hartmann3/matern3_2/interactive,pylibkriging,rmse,0.315752 +py01_kernels_trends,python,hartmann3/matern3_2/interactive,pylibkriging,q2,0.88201 +py01_kernels_trends,python,hartmann3/matern3_2/interactive,pylibkriging,nlpd,0.898253 +py01_kernels_trends,python,hartmann3/matern3_2/quadratic,pylibkriging,fit_s,0.00620675 +py01_kernels_trends,python,hartmann3/matern3_2/quadratic,pylibkriging,rmse,0.291393 +py01_kernels_trends,python,hartmann3/matern3_2/quadratic,pylibkriging,q2,0.899513 +py01_kernels_trends,python,hartmann3/matern3_2/quadratic,pylibkriging,nlpd,0.53743 +py01_kernels_trends,python,hartmann3/matern5_2/constant,pylibkriging,fit_s,0.00997186 +py01_kernels_trends,python,hartmann3/matern5_2/constant,pylibkriging,rmse,0.190287 +py01_kernels_trends,python,hartmann3/matern5_2/constant,pylibkriging,q2,0.957148 +py01_kernels_trends,python,hartmann3/matern5_2/constant,pylibkriging,nlpd,-0.403054 +py01_kernels_trends,python,hartmann3/matern5_2/constant,pylibkriging BFGS10,rmse,0.190286 +py01_kernels_trends,python,hartmann3/matern5_2/constant,pylibkriging BFGS10,q2,0.957148 +py01_kernels_trends,python,hartmann3/matern5_2/constant,pylibkriging BFGS10,nlpd,-0.40306 +py01_kernels_trends,python,hartmann3/matern5_2/linear,pylibkriging,fit_s,0.0127335 +py01_kernels_trends,python,hartmann3/matern5_2/linear,pylibkriging,rmse,0.189633 +py01_kernels_trends,python,hartmann3/matern5_2/linear,pylibkriging,q2,0.957442 +py01_kernels_trends,python,hartmann3/matern5_2/linear,pylibkriging,nlpd,-0.410075 +py01_kernels_trends,python,hartmann3/matern5_2/interactive,pylibkriging,fit_s,0.0101018 +py01_kernels_trends,python,hartmann3/matern5_2/interactive,pylibkriging,rmse,0.307114 +py01_kernels_trends,python,hartmann3/matern5_2/interactive,pylibkriging,q2,0.888377 +py01_kernels_trends,python,hartmann3/matern5_2/interactive,pylibkriging,nlpd,1.50063 +py01_kernels_trends,python,hartmann3/matern5_2/quadratic,pylibkriging,fit_s,0.0101326 +py01_kernels_trends,python,hartmann3/matern5_2/quadratic,pylibkriging,rmse,0.285355 +py01_kernels_trends,python,hartmann3/matern5_2/quadratic,pylibkriging,q2,0.903634 +py01_kernels_trends,python,hartmann3/matern5_2/quadratic,pylibkriging,nlpd,0.898128 +py02_objectives_gradients,python,branin/LL,pylibkriging,rmse,0.833715 +py02_objectives_gradients,python,branin/LL,pylibkriging,q2,0.999729 +py02_objectives_gradients,python,branin/LL,pylibkriging,nlpd,0.697034 +py02_objectives_gradients,python,branin/LOO,pylibkriging,rmse,0.550995 +py02_objectives_gradients,python,branin/LOO,pylibkriging,q2,0.999882 +py02_objectives_gradients,python,branin/LOO,pylibkriging,nlpd,0.471613 +py02_objectives_gradients,python,branin/LMP,pylibkriging,rmse,0.66967 +py02_objectives_gradients,python,branin/LMP,pylibkriging,q2,0.999825 +py02_objectives_gradients,python,branin/LMP,pylibkriging,nlpd,0.617527 +py02_objectives_gradients,python,hartmann3/LL,pylibkriging,rmse,0.157276 +py02_objectives_gradients,python,hartmann3/LL,pylibkriging,q2,0.970945 +py02_objectives_gradients,python,hartmann3/LL,pylibkriging,nlpd,-0.497548 +py02_objectives_gradients,python,hartmann3/LOO,pylibkriging,rmse,0.144982 +py02_objectives_gradients,python,hartmann3/LOO,pylibkriging,q2,0.97531 +py02_objectives_gradients,python,hartmann3/LOO,pylibkriging,nlpd,-0.584885 +py02_objectives_gradients,python,hartmann3/LMP,pylibkriging,rmse,0.150075 +py02_objectives_gradients,python,hartmann3/LMP,pylibkriging,q2,0.973544 +py02_objectives_gradients,python,hartmann3/LMP,pylibkriging,nlpd,-0.528379 +py02_objectives_gradients,python,ishigami/LL,pylibkriging,rmse,3.41501 +py02_objectives_gradients,python,ishigami/LL,pylibkriging,q2,0.203357 +py02_objectives_gradients,python,ishigami/LL,pylibkriging,nlpd,4.86258 +py02_objectives_gradients,python,ishigami/LOO,pylibkriging,rmse,2.45853 +py02_objectives_gradients,python,ishigami/LOO,pylibkriging,q2,0.587112 +py02_objectives_gradients,python,ishigami/LOO,pylibkriging,nlpd,2.72077 +py02_objectives_gradients,python,ishigami/LMP,pylibkriging,rmse,3.47385 +py02_objectives_gradients,python,ishigami/LMP,pylibkriging,q2,0.175667 +py02_objectives_gradients,python,ishigami/LMP,pylibkriging,nlpd,5.27033 +py02_objectives_gradients,python,friedman/LL,pylibkriging,rmse,0.273164 +py02_objectives_gradients,python,friedman/LL,pylibkriging,q2,0.996855 +py02_objectives_gradients,python,friedman/LL,pylibkriging,nlpd,-0.118408 +py02_objectives_gradients,python,friedman/LOO,pylibkriging,rmse,0.233921 +py02_objectives_gradients,python,friedman/LOO,pylibkriging,q2,0.997694 +py02_objectives_gradients,python,friedman/LOO,pylibkriging,nlpd,-0.169369 +py02_objectives_gradients,python,friedman/LMP,pylibkriging,rmse,0.269913 +py02_objectives_gradients,python,friedman/LMP,pylibkriging,q2,0.996929 +py02_objectives_gradients,python,friedman/LMP,pylibkriging,nlpd,-0.122709 +py03_noise,python,forrester,nugget,rmse,0.198091 +py03_noise,python,forrester,nugget,q2,0.998087 +py03_noise,python,forrester,nugget,nlpd,0.286631 +py03_noise,python,forrester,noise-free fit,rmse,0.487916 +py03_noise,python,forrester,noise-free fit,q2,0.988392 +py03_noise,python,forrester,noise-free fit,nlpd,862.67 +py03_noise,python,friedman,nugget,rmse,0.467335 +py03_noise,python,friedman,nugget,q2,0.99089 +py03_noise,python,friedman,nugget,nlpd,1.07073 +py03_noise,python,friedman,noise-free fit,rmse,1.27157 +py03_noise,python,friedman,noise-free fit,q2,0.932558 +py03_noise,python,friedman,noise-free fit,nlpd,1.64497 +py03_noise,python,gramacy_lee_2012/hetero,known variances,rmse,0.168741 +py03_noise,python,gramacy_lee_2012/hetero,known variances,q2,0.982515 +py03_noise,python,gramacy_lee_2012/hetero,known variances,nlpd,-0.570574 +py03_noise,python,gramacy_lee_2012/hetero,nugget,rmse,0.170945 +py03_noise,python,gramacy_lee_2012/hetero,nugget,q2,0.982055 +py03_noise,python,gramacy_lee_2012/hetero,nugget,nlpd,-0.221568 +py05_warpkriging,python,gl2012,Kriging,rmse,0.167263 +py05_warpkriging,python,gl2012,Kriging,q2,0.981176 +py05_warpkriging,python,gl2012,Kriging,nlpd,-1.08917 +py05_warpkriging,python,gl2012,affine,rmse,0.167263 +py05_warpkriging,python,gl2012,affine,q2,0.981176 +py05_warpkriging,python,gl2012,affine,nlpd,-1.08917 +py05_warpkriging,python,gl2012,boxcox,rmse,0.16759 +py05_warpkriging,python,gl2012,boxcox,q2,0.981103 +py05_warpkriging,python,gl2012,boxcox,nlpd,282.422 +py05_warpkriging,python,gl2012,kumaraswamy,rmse,0.168286 +py05_warpkriging,python,gl2012,kumaraswamy,q2,0.980945 +py05_warpkriging,python,gl2012,kumaraswamy,nlpd,2.04577e+06 +py05_warpkriging,python,gl2012,knots(5),rmse,0.16791 +py05_warpkriging,python,gl2012,knots(5),q2,0.98103 +py05_warpkriging,python,gl2012,knots(5),nlpd,2.04376e+06 +py05_warpkriging,python,gl2012,neural_mono(8),rmse,0.170824 +py05_warpkriging,python,gl2012,neural_mono(8),q2,0.980366 +py05_warpkriging,python,gl2012,neural_mono(8),nlpd,-0.265141 +py05_warpkriging,python,gl2008,Kriging,rmse,0.0534649 +py05_warpkriging,python,gl2008,Kriging,q2,0.585636 +py05_warpkriging,python,gl2008,Kriging,nlpd,-1.74569 +py05_warpkriging,python,gl2008,kumaraswamy,rmse,0.0535298 +py05_warpkriging,python,gl2008,kumaraswamy,q2,0.58463 +py05_warpkriging,python,gl2008,kumaraswamy,nlpd,-1.74552 +py05_warpkriging,python,mixed,Kriging (level as number),rmse,16.7693 +py05_warpkriging,python,mixed,Kriging (level as number),q2,0.838452 +py05_warpkriging,python,mixed,Kriging (level as number),nlpd,3.81063 +py05_warpkriging,python,mixed,"categorical(5,2)",rmse,16.1438 +py05_warpkriging,python,mixed,"categorical(5,2)",q2,0.85028 +py05_warpkriging,python,mixed,"categorical(5,2)",nlpd,3.3206 +py05_warpkriging,python,ordinal,Kriging (rank as number),rmse,0.213845 +py05_warpkriging,python,ordinal,Kriging (rank as number),q2,0.953177 +py05_warpkriging,python,ordinal,Kriging (rank as number),nlpd,0.810969 +py05_warpkriging,python,ordinal,ordinal(6),rmse,0.213043 +py05_warpkriging,python,ordinal,ordinal(6),q2,0.953528 +py05_warpkriging,python,ordinal,ordinal(6),nlpd,0.78888 +py06_high_dimension_mlp,python,otl_circuit,unit cube,fit_s,0.00789523 +py06_high_dimension_mlp,python,otl_circuit,unit cube,rmse,0.0150514 +py06_high_dimension_mlp,python,otl_circuit,unit cube,q2,0.999836 +py06_high_dimension_mlp,python,otl_circuit,unit cube,nlpd,-2.90452 +py06_high_dimension_mlp,python,otl_circuit,native + normalize,rmse,0.0150514 +py06_high_dimension_mlp,python,otl_circuit,native + normalize,q2,0.999836 +py06_high_dimension_mlp,python,otl_circuit,native + normalize,nlpd,-2.90452 +py06_high_dimension_mlp,python,piston,unit cube,fit_s,0.0197392 +py06_high_dimension_mlp,python,piston,unit cube,rmse,0.0104258 +py06_high_dimension_mlp,python,piston,unit cube,q2,0.994462 +py06_high_dimension_mlp,python,piston,unit cube,nlpd,-3.49038 +py06_high_dimension_mlp,python,piston,native + normalize,rmse,0.0104258 +py06_high_dimension_mlp,python,piston,native + normalize,q2,0.994462 +py06_high_dimension_mlp,python,piston,native + normalize,nlpd,-3.49038 +py06_high_dimension_mlp,python,borehole,unit cube,fit_s,0.0140877 +py06_high_dimension_mlp,python,borehole,unit cube,rmse,0.893341 +py06_high_dimension_mlp,python,borehole,unit cube,q2,0.999599 +py06_high_dimension_mlp,python,borehole,unit cube,nlpd,1.08433 +py06_high_dimension_mlp,python,borehole,native + normalize,rmse,0.893341 +py06_high_dimension_mlp,python,borehole,native + normalize,q2,0.999599 +py06_high_dimension_mlp,python,borehole,native + normalize,nlpd,1.08433 +py06_high_dimension_mlp,python,robot_arm,unit cube,fit_s,0.167482 +py06_high_dimension_mlp,python,robot_arm,unit cube,rmse,0.358507 +py06_high_dimension_mlp,python,robot_arm,unit cube,q2,0.571154 +py06_high_dimension_mlp,python,robot_arm,unit cube,nlpd,0.355106 +py06_high_dimension_mlp,python,robot_arm,native + normalize,rmse,0.358507 +py06_high_dimension_mlp,python,robot_arm,native + normalize,q2,0.571154 +py06_high_dimension_mlp,python,robot_arm,native + normalize,nlpd,0.355106 +py06_high_dimension_mlp,python,dette_pepelyshev,unit cube,fit_s,0.0246282 +py06_high_dimension_mlp,python,dette_pepelyshev,unit cube,rmse,0.424908 +py06_high_dimension_mlp,python,dette_pepelyshev,unit cube,q2,0.997757 +py06_high_dimension_mlp,python,dette_pepelyshev,unit cube,nlpd,0.510276 +py06_high_dimension_mlp,python,dette_pepelyshev,native + normalize,rmse,0.424908 +py06_high_dimension_mlp,python,dette_pepelyshev,native + normalize,q2,0.997757 +py06_high_dimension_mlp,python,dette_pepelyshev,native + normalize,nlpd,0.510276 +py06_high_dimension_mlp,python,wing_weight,unit cube,fit_s,0.0245481 +py06_high_dimension_mlp,python,wing_weight,unit cube,rmse,0.809743 +py06_high_dimension_mlp,python,wing_weight,unit cube,q2,0.999716 +py06_high_dimension_mlp,python,wing_weight,unit cube,nlpd,1.07161 +py06_high_dimension_mlp,python,wing_weight,native + normalize,rmse,0.809743 +py06_high_dimension_mlp,python,wing_weight,native + normalize,q2,0.999716 +py06_high_dimension_mlp,python,wing_weight,native + normalize,nlpd,1.07161 +py06_high_dimension_mlp,python,welch,unit cube,fit_s,0.564758 +py06_high_dimension_mlp,python,welch,unit cube,rmse,0.167729 +py06_high_dimension_mlp,python,welch,unit cube,q2,0.993452 +py06_high_dimension_mlp,python,welch,unit cube,nlpd,-0.429138 +py06_high_dimension_mlp,python,welch,native + normalize,rmse,0.167692 +py06_high_dimension_mlp,python,welch,native + normalize,q2,0.993454 +py06_high_dimension_mlp,python,welch,native + normalize,nlpd,-0.429291 +py06_high_dimension_mlp,python,projected_branin_10d,Kriging,rmse,4.42284 +py06_high_dimension_mlp,python,projected_branin_10d,Kriging,q2,0.959122 +py06_high_dimension_mlp,python,projected_branin_10d,Kriging,nlpd,2.89997 +py06_high_dimension_mlp,python,projected_branin_10d,MLPKriging tanh BFGS,fit_s,0.130867 +py06_high_dimension_mlp,python,projected_branin_10d,MLPKriging tanh BFGS,rmse,57.402 +py06_high_dimension_mlp,python,projected_branin_10d,MLPKriging tanh BFGS,q2,-5.88553 +py06_high_dimension_mlp,python,projected_branin_10d,MLPKriging tanh BFGS,nlpd,136.771 +py06_high_dimension_mlp,python,projected_branin_10d,MLPKriging tanh BFGS10,fit_s,0.442791 +py06_high_dimension_mlp,python,projected_branin_10d,MLPKriging tanh BFGS10,rmse,4.28606 +py06_high_dimension_mlp,python,projected_branin_10d,MLPKriging tanh BFGS10,q2,0.961612 +py06_high_dimension_mlp,python,projected_branin_10d,MLPKriging tanh BFGS10,nlpd,29.0854 +py06_high_dimension_mlp,python,projected_branin_10d,MLPKriging selu BFGS,fit_s,0.139126 +py06_high_dimension_mlp,python,projected_branin_10d,MLPKriging selu BFGS,rmse,58.9913 +py06_high_dimension_mlp,python,projected_branin_10d,MLPKriging selu BFGS,q2,-6.2721 +py06_high_dimension_mlp,python,projected_branin_10d,MLPKriging selu BFGS,nlpd,46.9788 +py06_high_dimension_mlp,python,projected_branin_10d,MLPKriging selu BFGS10,fit_s,0.440179 +py06_high_dimension_mlp,python,projected_branin_10d,MLPKriging selu BFGS10,rmse,4.39195 +py06_high_dimension_mlp,python,projected_branin_10d,MLPKriging selu BFGS10,q2,0.959691 +py06_high_dimension_mlp,python,projected_branin_10d,MLPKriging selu BFGS10,nlpd,10.6053 +py07_scalability,python,hartmann3/n=1500,exact,fit_s,11.1133 +py07_scalability,python,hartmann3/n=1500,exact,rmse,0.000259233 +py07_scalability,python,hartmann3/n=1500,exact,q2,1 +py07_scalability,python,hartmann3/n=1500,exact,nlpd,-6.92184 +py07_scalability,python,hartmann3/n=1500,vecchia(30),fit_s,5.43768 +py07_scalability,python,hartmann3/n=1500,vecchia(30),rmse,0.00353225 +py07_scalability,python,hartmann3/n=1500,vecchia(30),q2,0.999987 +py07_scalability,python,hartmann3/n=1500,vecchia(30),nlpd,-4.6266 +py07_scalability,python,hartmann3/n=5000,vecchia(30),fit_s,16.2077 +py07_scalability,python,hartmann3/n=5000,vecchia(30),rmse,0.000851738 +py07_scalability,python,hartmann3/n=5000,vecchia(30),q2,0.999999 +py07_scalability,python,hartmann3/n=5000,vecchia(30),nlpd,-5.95078 +py07_scalability,python,hartmann3/n=1500,nystrom(100),fit_s,2.04779 +py07_scalability,python,hartmann3/n=1500,nystrom(100),rmse,1.51127 +py07_scalability,python,hartmann3/n=1500,nystrom(100),q2,-1.39237 +py07_scalability,python,hartmann3/n=1500,nystrom(100),nlpd,1.14197e+24 +py07_scalability,python,hartmann3/n=5000,nystrom(100),fit_s,3.25622 +py07_scalability,python,hartmann3/n=5000,nystrom(100),rmse,9.95398 +py07_scalability,python,hartmann3/n=5000,nystrom(100),q2,-102.786 +py07_scalability,python,hartmann3/n=5000,nystrom(100),nlpd,4.95409e+25 +py07_scalability,python,hartmann3/n=1500,nested NK (20 groups),fit_s,0.269932 +py07_scalability,python,hartmann3/n=1500,nested NK (20 groups),rmse,0.0029908 +py07_scalability,python,hartmann3/n=1500,nested NK (20 groups),q2,0.999991 +py07_scalability,python,hartmann3/n=1500,nested NK (20 groups),nlpd,-4.85946 +py07_scalability,python,hartmann3/n=5000,nested NK (20 groups),fit_s,2.87469 +py07_scalability,python,hartmann3/n=5000,nested NK (20 groups),rmse,0.000222655 +py07_scalability,python,hartmann3/n=5000,nested NK (20 groups),q2,1 +py07_scalability,python,hartmann3/n=5000,nested NK (20 groups),nlpd,-7.44453 +py07_scalability,python,hartmann3/n=1500,nested rBCM (20 groups),fit_s,0.25767 +py07_scalability,python,hartmann3/n=1500,nested rBCM (20 groups),rmse,0.00304431 +py07_scalability,python,hartmann3/n=1500,nested rBCM (20 groups),q2,0.99999 +py07_scalability,python,hartmann3/n=1500,nested rBCM (20 groups),nlpd,-4.50201 +py07_scalability,python,hartmann3/n=5000,nested rBCM (20 groups),fit_s,2.5369 +py07_scalability,python,hartmann3/n=5000,nested rBCM (20 groups),rmse,0.000224517 +py07_scalability,python,hartmann3/n=5000,nested rBCM (20 groups),q2,1 +py07_scalability,python,hartmann3/n=5000,nested rBCM (20 groups),nlpd,-7.18182 +py07_scalability,python,hartmann3/n=1500,subset of data (500),fit_s,0.712767 +py07_scalability,python,hartmann3/n=1500,subset of data (500),rmse,0.00269153 +py07_scalability,python,hartmann3/n=1500,subset of data (500),q2,0.999992 +py07_scalability,python,hartmann3/n=1500,subset of data (500),nlpd,-4.68641 +py07_scalability,python,hartmann3/n=5000,subset of data (500),fit_s,0.950298 +py07_scalability,python,hartmann3/n=5000,subset of data (500),rmse,0.00247552 +py07_scalability,python,hartmann3/n=5000,subset of data (500),q2,0.999994 +py07_scalability,python,hartmann3/n=5000,subset of data (500),nlpd,-4.71359 +py07_scalability,python,otl_circuit/n=1500,exact,fit_s,10.6103 +py07_scalability,python,otl_circuit/n=1500,exact,rmse,9.81598e-05 +py07_scalability,python,otl_circuit/n=1500,exact,q2,1 +py07_scalability,python,otl_circuit/n=1500,exact,nlpd,-8.21962 +py07_scalability,python,otl_circuit/n=1500,vecchia(30),fit_s,4.73217 +py07_scalability,python,otl_circuit/n=1500,vecchia(30),rmse,0.00629979 +py07_scalability,python,otl_circuit/n=1500,vecchia(30),q2,0.999969 +py07_scalability,python,otl_circuit/n=1500,vecchia(30),nlpd,-3.95617 +py07_scalability,python,otl_circuit/n=5000,vecchia(30),fit_s,14.5549 +py07_scalability,python,otl_circuit/n=5000,vecchia(30),rmse,0.00312888 +py07_scalability,python,otl_circuit/n=5000,vecchia(30),q2,0.999992 +py07_scalability,python,otl_circuit/n=5000,vecchia(30),nlpd,-4.57612 +py07_scalability,python,otl_circuit/n=1500,nystrom(100),fit_s,1.47345 +py07_scalability,python,otl_circuit/n=1500,nystrom(100),rmse,0.392811 +py07_scalability,python,otl_circuit/n=1500,nystrom(100),q2,0.880908 +py07_scalability,python,otl_circuit/n=1500,nystrom(100),nlpd,7.06156e+22 +py07_scalability,python,otl_circuit/n=5000,nystrom(100),fit_s,4.69579 +py07_scalability,python,otl_circuit/n=5000,nystrom(100),rmse,1.11526 +py07_scalability,python,otl_circuit/n=5000,nystrom(100),q2,0.0400057 +py07_scalability,python,otl_circuit/n=5000,nystrom(100),nlpd,5.31511e+23 +py07_scalability,python,otl_circuit/n=1500,nested NK (20 groups),fit_s,0.278129 +py07_scalability,python,otl_circuit/n=1500,nested NK (20 groups),rmse,0.00226873 +py07_scalability,python,otl_circuit/n=1500,nested NK (20 groups),q2,0.999996 +py07_scalability,python,otl_circuit/n=1500,nested NK (20 groups),nlpd,-4.90542 +py07_scalability,python,otl_circuit/n=5000,nested NK (20 groups),fit_s,3.62576 +py07_scalability,python,otl_circuit/n=5000,nested NK (20 groups),rmse,0.000261686 +py07_scalability,python,otl_circuit/n=5000,nested NK (20 groups),q2,1 +py07_scalability,python,otl_circuit/n=5000,nested NK (20 groups),nlpd,-7.20623 +py07_scalability,python,otl_circuit/n=1500,nested rBCM (20 groups),fit_s,0.315619 +py07_scalability,python,otl_circuit/n=1500,nested rBCM (20 groups),rmse,0.00291869 +py07_scalability,python,otl_circuit/n=1500,nested rBCM (20 groups),q2,0.999993 +py07_scalability,python,otl_circuit/n=1500,nested rBCM (20 groups),nlpd,-2.55512 +py07_scalability,python,otl_circuit/n=5000,nested rBCM (20 groups),fit_s,3.39037 +py07_scalability,python,otl_circuit/n=5000,nested rBCM (20 groups),rmse,0.000278149 +py07_scalability,python,otl_circuit/n=5000,nested rBCM (20 groups),q2,1 +py07_scalability,python,otl_circuit/n=5000,nested rBCM (20 groups),nlpd,-4.34927 +py07_scalability,python,otl_circuit/n=1500,subset of data (500),fit_s,0.672431 +py07_scalability,python,otl_circuit/n=1500,subset of data (500),rmse,0.0004286 +py07_scalability,python,otl_circuit/n=1500,subset of data (500),q2,1 +py07_scalability,python,otl_circuit/n=1500,subset of data (500),nlpd,-6.72678 +py07_scalability,python,otl_circuit/n=5000,subset of data (500),fit_s,0.692672 +py07_scalability,python,otl_circuit/n=5000,subset of data (500),rmse,0.000446879 +py07_scalability,python,otl_circuit/n=5000,subset of data (500),q2,1 +py07_scalability,python,otl_circuit/n=5000,subset of data (500),nlpd,-6.78844 +py07_scalability,python,branin/nested,NK,rmse,0.0343984 +py07_scalability,python,branin/nested,NK,q2,1 +py07_scalability,python,branin/nested,NK,nlpd,2.00082e+14 +py07_scalability,python,branin/nested,PoE,rmse,0.0354265 +py07_scalability,python,branin/nested,PoE,q2,1 +py07_scalability,python,branin/nested,PoE,nlpd,-1.64533 +py07_scalability,python,branin/nested,gPoE,rmse,0.0354265 +py07_scalability,python,branin/nested,gPoE,q2,1 +py07_scalability,python,branin/nested,gPoE,nlpd,-0.621836 +py07_scalability,python,branin/nested,BCM,rmse,0.0354196 +py07_scalability,python,branin/nested,BCM,q2,1 +py07_scalability,python,branin/nested,BCM,nlpd,-1.64533 +py07_scalability,python,branin/nested,rBCM,rmse,0.035287 +py07_scalability,python,branin/nested,rBCM,q2,1 +py07_scalability,python,branin/nested,rBCM,nlpd,-2.61369 +py10_compare_python_packages,python,branin,pylibkriging,q2,0.999944 +py10_compare_python_packages,python,branin,pylibkriging,nlpd,0.137225 +py10_compare_python_packages,python,branin,pylibkriging,fit_s,0.00261593 +py10_compare_python_packages,python,branin,pylibkriging BFGS10,q2,0.999945 +py10_compare_python_packages,python,branin,pylibkriging BFGS10,nlpd,0.137254 +py10_compare_python_packages,python,branin,pylibkriging BFGS10,fit_s,0.02301 +py10_compare_python_packages,python,branin,scikit-learn,q2,0.999597 +py10_compare_python_packages,python,branin,scikit-learn,nlpd,0.928508 +py10_compare_python_packages,python,branin,scikit-learn,fit_s,0.169693 +py10_compare_python_packages,python,branin,SMT,q2,0.999944 +py10_compare_python_packages,python,branin,SMT,nlpd,0.137259 +py10_compare_python_packages,python,branin,SMT,fit_s,0.833295 +py10_compare_python_packages,python,branin,OpenTURNS,q2,0.998221 +py10_compare_python_packages,python,branin,OpenTURNS,nlpd,1.55599 +py10_compare_python_packages,python,branin,OpenTURNS,fit_s,0.00490117 +py10_compare_python_packages,python,six_hump_camel,pylibkriging,q2,0.999418 +py10_compare_python_packages,python,six_hump_camel,pylibkriging,nlpd,0.0429342 +py10_compare_python_packages,python,six_hump_camel,pylibkriging,fit_s,0.00324678 +py10_compare_python_packages,python,six_hump_camel,pylibkriging BFGS10,q2,0.999418 +py10_compare_python_packages,python,six_hump_camel,pylibkriging BFGS10,nlpd,0.0429386 +py10_compare_python_packages,python,six_hump_camel,pylibkriging BFGS10,fit_s,0.060586 +py10_compare_python_packages,python,six_hump_camel,scikit-learn,q2,0.0163311 +py10_compare_python_packages,python,six_hump_camel,scikit-learn,nlpd,4.63763 +py10_compare_python_packages,python,six_hump_camel,scikit-learn,fit_s,0.206083 +py10_compare_python_packages,python,six_hump_camel,SMT,q2,0.999594 +py10_compare_python_packages,python,six_hump_camel,SMT,nlpd,-0.121007 +py10_compare_python_packages,python,six_hump_camel,SMT,fit_s,1.03758 +py10_compare_python_packages,python,six_hump_camel,OpenTURNS,q2,0.956713 +py10_compare_python_packages,python,six_hump_camel,OpenTURNS,nlpd,2.55446 +py10_compare_python_packages,python,six_hump_camel,OpenTURNS,fit_s,0.00741315 +py10_compare_python_packages,python,hartmann3,pylibkriging,q2,0.986148 +py10_compare_python_packages,python,hartmann3,pylibkriging,nlpd,-0.847247 +py10_compare_python_packages,python,hartmann3,pylibkriging,fit_s,0.0169713 +py10_compare_python_packages,python,hartmann3,pylibkriging BFGS10,q2,0.986148 +py10_compare_python_packages,python,hartmann3,pylibkriging BFGS10,nlpd,-0.847249 +py10_compare_python_packages,python,hartmann3,pylibkriging BFGS10,fit_s,0.139588 +py10_compare_python_packages,python,hartmann3,scikit-learn,q2,0.983889 +py10_compare_python_packages,python,hartmann3,scikit-learn,nlpd,-0.782697 +py10_compare_python_packages,python,hartmann3,scikit-learn,fit_s,0.188796 +py10_compare_python_packages,python,hartmann3,SMT,q2,0.986148 +py10_compare_python_packages,python,hartmann3,SMT,nlpd,-0.847249 +py10_compare_python_packages,python,hartmann3,SMT,fit_s,1.02907 +py10_compare_python_packages,python,hartmann3,OpenTURNS,q2,0.984712 +py10_compare_python_packages,python,hartmann3,OpenTURNS,nlpd,-0.809851 +py10_compare_python_packages,python,hartmann3,OpenTURNS,fit_s,0.0153861 +py10_compare_python_packages,python,ishigami,pylibkriging,q2,0.259556 +py10_compare_python_packages,python,ishigami,pylibkriging,nlpd,4.38991 +py10_compare_python_packages,python,ishigami,pylibkriging,fit_s,0.0256422 +py10_compare_python_packages,python,ishigami,pylibkriging BFGS10,q2,0.935722 +py10_compare_python_packages,python,ishigami,pylibkriging BFGS10,nlpd,0.857443 +py10_compare_python_packages,python,ishigami,pylibkriging BFGS10,fit_s,0.308839 +py10_compare_python_packages,python,ishigami,scikit-learn,q2,0.831142 +py10_compare_python_packages,python,ishigami,scikit-learn,nlpd,1.5455 +py10_compare_python_packages,python,ishigami,scikit-learn,fit_s,0.51733 +py10_compare_python_packages,python,ishigami,SMT,q2,0.51196 +py10_compare_python_packages,python,ishigami,SMT,nlpd,3.31349 +py10_compare_python_packages,python,ishigami,SMT,fit_s,2.77225 +py10_compare_python_packages,python,ishigami,OpenTURNS,q2,0.887251 +py10_compare_python_packages,python,ishigami,OpenTURNS,nlpd,1.39348 +py10_compare_python_packages,python,ishigami,OpenTURNS,fit_s,0.0499659 +py10_compare_python_packages,python,friedman,pylibkriging,q2,0.999771 +py10_compare_python_packages,python,friedman,pylibkriging,nlpd,-1.29726 +py10_compare_python_packages,python,friedman,pylibkriging,fit_s,0.0181558 +py10_compare_python_packages,python,friedman,pylibkriging BFGS10,q2,0.999771 +py10_compare_python_packages,python,friedman,pylibkriging BFGS10,nlpd,-1.29727 +py10_compare_python_packages,python,friedman,pylibkriging BFGS10,fit_s,0.215106 +py10_compare_python_packages,python,friedman,scikit-learn,q2,0.999754 +py10_compare_python_packages,python,friedman,scikit-learn,nlpd,-1.37702 +py10_compare_python_packages,python,friedman,scikit-learn,fit_s,0.283978 +py10_compare_python_packages,python,friedman,SMT,q2,0.999899 +py10_compare_python_packages,python,friedman,SMT,nlpd,-1.871 +py10_compare_python_packages,python,friedman,SMT,fit_s,4.3249 +py10_compare_python_packages,python,friedman,OpenTURNS,q2,0.990862 +py10_compare_python_packages,python,friedman,OpenTURNS,nlpd,0.469828 +py10_compare_python_packages,python,friedman,OpenTURNS,fit_s,0.0293689 +py10_compare_python_packages,python,otl_circuit,pylibkriging,q2,0.999922 +py10_compare_python_packages,python,otl_circuit,pylibkriging,nlpd,-3.46228 +py10_compare_python_packages,python,otl_circuit,pylibkriging,fit_s,0.013788 +py10_compare_python_packages,python,otl_circuit,pylibkriging BFGS10,q2,0.999922 +py10_compare_python_packages,python,otl_circuit,pylibkriging BFGS10,nlpd,-3.46227 +py10_compare_python_packages,python,otl_circuit,pylibkriging BFGS10,fit_s,0.141071 +py10_compare_python_packages,python,otl_circuit,scikit-learn,q2,0.999906 +py10_compare_python_packages,python,otl_circuit,scikit-learn,nlpd,-3.43062 +py10_compare_python_packages,python,otl_circuit,scikit-learn,fit_s,0.51218 +py10_compare_python_packages,python,otl_circuit,SMT,q2,0.999954 +py10_compare_python_packages,python,otl_circuit,SMT,nlpd,-3.35016 +py10_compare_python_packages,python,otl_circuit,SMT,fit_s,7.1224 +py10_compare_python_packages,python,otl_circuit,OpenTURNS,q2,0.99777 +py10_compare_python_packages,python,otl_circuit,OpenTURNS,nlpd,-1.61723 +py10_compare_python_packages,python,otl_circuit,OpenTURNS,fit_s,0.0603108 +py10_compare_python_packages,python,piston,pylibkriging,q2,0.997498 +py10_compare_python_packages,python,piston,pylibkriging,nlpd,-3.80667 +py10_compare_python_packages,python,piston,pylibkriging,fit_s,0.0395648 +py10_compare_python_packages,python,piston,pylibkriging BFGS10,q2,0.997498 +py10_compare_python_packages,python,piston,pylibkriging BFGS10,nlpd,-3.8067 +py10_compare_python_packages,python,piston,pylibkriging BFGS10,fit_s,0.327989 +py10_compare_python_packages,python,piston,scikit-learn,q2,0.998684 +py10_compare_python_packages,python,piston,scikit-learn,nlpd,-4.10497 +py10_compare_python_packages,python,piston,scikit-learn,fit_s,1.11386 +py10_compare_python_packages,python,piston,SMT,q2,0.997511 +py10_compare_python_packages,python,piston,SMT,nlpd,-3.86618 +py10_compare_python_packages,python,piston,SMT,fit_s,12.649 +py10_compare_python_packages,python,piston,OpenTURNS,q2,0.990631 +py10_compare_python_packages,python,piston,OpenTURNS,nlpd,-2.9906 +py10_compare_python_packages,python,piston,OpenTURNS,fit_s,0.11631 +py10_compare_python_packages,python,borehole,pylibkriging,q2,0.999694 +py10_compare_python_packages,python,borehole,pylibkriging,nlpd,1.00118 +py10_compare_python_packages,python,borehole,pylibkriging,fit_s,0.0149446 +py10_compare_python_packages,python,borehole,pylibkriging BFGS10,q2,0.999694 +py10_compare_python_packages,python,borehole,pylibkriging BFGS10,nlpd,1.00121 +py10_compare_python_packages,python,borehole,pylibkriging BFGS10,fit_s,0.164608 +py10_compare_python_packages,python,borehole,scikit-learn,q2,0.999929 +py10_compare_python_packages,python,borehole,scikit-learn,nlpd,0.194455 +py10_compare_python_packages,python,borehole,scikit-learn,fit_s,1.04563 +py10_compare_python_packages,python,borehole,SMT,q2,0.999925 +py10_compare_python_packages,python,borehole,SMT,nlpd,0.279122 +py10_compare_python_packages,python,borehole,SMT,fit_s,11.8531 +py10_compare_python_packages,python,borehole,OpenTURNS,q2,0.996206 +py10_compare_python_packages,python,borehole,OpenTURNS,nlpd,2.55431 +py10_compare_python_packages,python,borehole,OpenTURNS,fit_s,0.12395 +py10_compare_python_packages,python,wing_weight,pylibkriging,q2,0.99972 +py10_compare_python_packages,python,wing_weight,pylibkriging,nlpd,1.11517 +py10_compare_python_packages,python,wing_weight,pylibkriging,fit_s,0.0290489 +py10_compare_python_packages,python,wing_weight,pylibkriging BFGS10,q2,0.99972 +py10_compare_python_packages,python,wing_weight,pylibkriging BFGS10,nlpd,1.11512 +py10_compare_python_packages,python,wing_weight,pylibkriging BFGS10,fit_s,0.314707 +py10_compare_python_packages,python,wing_weight,scikit-learn,q2,0.998048 +py10_compare_python_packages,python,wing_weight,scikit-learn,nlpd,2.57304 +py10_compare_python_packages,python,wing_weight,scikit-learn,fit_s,0.729278 +py10_compare_python_packages,python,wing_weight,SMT,q2,0.999898 +py10_compare_python_packages,python,wing_weight,SMT,nlpd,0.604023 +py10_compare_python_packages,python,wing_weight,SMT,fit_s,25.8262 +py10_compare_python_packages,python,wing_weight,OpenTURNS,q2,0.993145 +py10_compare_python_packages,python,wing_weight,OpenTURNS,nlpd,2.83627 +py10_compare_python_packages,python,wing_weight,OpenTURNS,fit_s,0.280434 +py10_compare_python_packages,python,hartmann6,pylibkriging,q2,0.422938 +py10_compare_python_packages,python,hartmann6,pylibkriging,nlpd,0.0965167 +py10_compare_python_packages,python,hartmann6,pylibkriging,fit_s,0.0562823 +py10_compare_python_packages,python,hartmann6,pylibkriging BFGS10,q2,0.72352 +py10_compare_python_packages,python,hartmann6,pylibkriging BFGS10,nlpd,-0.156898 +py10_compare_python_packages,python,hartmann6,pylibkriging BFGS10,fit_s,0.822745 +py10_compare_python_packages,python,hartmann6,scikit-learn,q2,0.69049 +py10_compare_python_packages,python,hartmann6,scikit-learn,nlpd,-0.137845 +py10_compare_python_packages,python,hartmann6,scikit-learn,fit_s,1.3155 +py10_compare_python_packages,python,hartmann6,SMT,q2,0.708463 +py10_compare_python_packages,python,hartmann6,SMT,nlpd,-0.0177081 +py10_compare_python_packages,python,hartmann6,SMT,fit_s,12.0578 +py10_compare_python_packages,python,hartmann6,OpenTURNS,q2,0.696177 +py10_compare_python_packages,python,hartmann6,OpenTURNS,nlpd,-0.132838 +py10_compare_python_packages,python,hartmann6,OpenTURNS,fit_s,0.133332 diff --git a/baseline/metrics-release.csv b/baseline/metrics-release.csv new file mode 100644 index 0000000..0a25355 --- /dev/null +++ b/baseline/metrics-release.csv @@ -0,0 +1,916 @@ +suite,lang,case,package,metric,value +py01_kernels_trends,python,forrester/gauss/constant,pylibkriging,fit_s,0.0039537 +py01_kernels_trends,python,forrester/gauss/constant,pylibkriging,rmse,0.0848271 +py01_kernels_trends,python,forrester/gauss/constant,pylibkriging,q2,0.999634 +py01_kernels_trends,python,forrester/gauss/constant,pylibkriging,nlpd,-3.06486 +py01_kernels_trends,python,forrester/gauss/linear,pylibkriging,fit_s,0.0008564 +py01_kernels_trends,python,forrester/gauss/linear,pylibkriging,rmse,0.0855468 +py01_kernels_trends,python,forrester/gauss/linear,pylibkriging,q2,0.999628 +py01_kernels_trends,python,forrester/gauss/linear,pylibkriging,nlpd,-3.11029 +py01_kernels_trends,python,forrester/gauss/interactive,pylibkriging,fit_s,0.000818729 +py01_kernels_trends,python,forrester/gauss/interactive,pylibkriging,rmse,0.0855468 +py01_kernels_trends,python,forrester/gauss/interactive,pylibkriging,q2,0.999628 +py01_kernels_trends,python,forrester/gauss/interactive,pylibkriging,nlpd,-3.11029 +py01_kernels_trends,python,forrester/gauss/quadratic,pylibkriging,fit_s,0.000473261 +py01_kernels_trends,python,forrester/gauss/quadratic,pylibkriging,rmse,1.57249 +py01_kernels_trends,python,forrester/gauss/quadratic,pylibkriging,q2,0.874394 +py01_kernels_trends,python,forrester/gauss/quadratic,pylibkriging,nlpd,5.33879 +py01_kernels_trends,python,forrester/exp/constant,pylibkriging,fit_s,0.00042963 +py01_kernels_trends,python,forrester/exp/constant,pylibkriging,rmse,0.721582 +py01_kernels_trends,python,forrester/exp/constant,pylibkriging,q2,0.973551 +py01_kernels_trends,python,forrester/exp/constant,pylibkriging,nlpd,1.54551 +py01_kernels_trends,python,forrester/exp/linear,pylibkriging,fit_s,0.000402451 +py01_kernels_trends,python,forrester/exp/linear,pylibkriging,rmse,0.793413 +py01_kernels_trends,python,forrester/exp/linear,pylibkriging,q2,0.968024 +py01_kernels_trends,python,forrester/exp/linear,pylibkriging,nlpd,1.50639 +py01_kernels_trends,python,forrester/exp/interactive,pylibkriging,fit_s,0.000428438 +py01_kernels_trends,python,forrester/exp/interactive,pylibkriging,rmse,0.793413 +py01_kernels_trends,python,forrester/exp/interactive,pylibkriging,q2,0.968024 +py01_kernels_trends,python,forrester/exp/interactive,pylibkriging,nlpd,1.50639 +py01_kernels_trends,python,forrester/exp/quadratic,pylibkriging,fit_s,0.000418425 +py01_kernels_trends,python,forrester/exp/quadratic,pylibkriging,rmse,0.723267 +py01_kernels_trends,python,forrester/exp/quadratic,pylibkriging,q2,0.973428 +py01_kernels_trends,python,forrester/exp/quadratic,pylibkriging,nlpd,1.28782 +py01_kernels_trends,python,forrester/matern3_2/constant,pylibkriging,fit_s,0.000519753 +py01_kernels_trends,python,forrester/matern3_2/constant,pylibkriging,rmse,0.334007 +py01_kernels_trends,python,forrester/matern3_2/constant,pylibkriging,q2,0.994333 +py01_kernels_trends,python,forrester/matern3_2/constant,pylibkriging,nlpd,-0.227715 +py01_kernels_trends,python,forrester/matern3_2/linear,pylibkriging,fit_s,0.000559092 +py01_kernels_trends,python,forrester/matern3_2/linear,pylibkriging,rmse,0.364477 +py01_kernels_trends,python,forrester/matern3_2/linear,pylibkriging,q2,0.993252 +py01_kernels_trends,python,forrester/matern3_2/linear,pylibkriging,nlpd,-0.187262 +py01_kernels_trends,python,forrester/matern3_2/interactive,pylibkriging,fit_s,0.00060463 +py01_kernels_trends,python,forrester/matern3_2/interactive,pylibkriging,rmse,0.364477 +py01_kernels_trends,python,forrester/matern3_2/interactive,pylibkriging,q2,0.993252 +py01_kernels_trends,python,forrester/matern3_2/interactive,pylibkriging,nlpd,-0.187262 +py01_kernels_trends,python,forrester/matern3_2/quadratic,pylibkriging,fit_s,0.000493765 +py01_kernels_trends,python,forrester/matern3_2/quadratic,pylibkriging,rmse,0.435609 +py01_kernels_trends,python,forrester/matern3_2/quadratic,pylibkriging,q2,0.990361 +py01_kernels_trends,python,forrester/matern3_2/quadratic,pylibkriging,nlpd,-0.183076 +py01_kernels_trends,python,forrester/matern5_2/constant,pylibkriging,fit_s,0.000758886 +py01_kernels_trends,python,forrester/matern5_2/constant,pylibkriging,rmse,0.281945 +py01_kernels_trends,python,forrester/matern5_2/constant,pylibkriging,q2,0.995962 +py01_kernels_trends,python,forrester/matern5_2/constant,pylibkriging,nlpd,-1.01945 +py01_kernels_trends,python,forrester/matern5_2/constant,pylibkriging BFGS10,rmse,0.281945 +py01_kernels_trends,python,forrester/matern5_2/constant,pylibkriging BFGS10,q2,0.995962 +py01_kernels_trends,python,forrester/matern5_2/constant,pylibkriging BFGS10,nlpd,-1.01945 +py01_kernels_trends,python,forrester/matern5_2/linear,pylibkriging,fit_s,0.000576973 +py01_kernels_trends,python,forrester/matern5_2/linear,pylibkriging,rmse,0.298959 +py01_kernels_trends,python,forrester/matern5_2/linear,pylibkriging,q2,0.99546 +py01_kernels_trends,python,forrester/matern5_2/linear,pylibkriging,nlpd,-0.95551 +py01_kernels_trends,python,forrester/matern5_2/interactive,pylibkriging,fit_s,0.00054884 +py01_kernels_trends,python,forrester/matern5_2/interactive,pylibkriging,rmse,0.298959 +py01_kernels_trends,python,forrester/matern5_2/interactive,pylibkriging,q2,0.99546 +py01_kernels_trends,python,forrester/matern5_2/interactive,pylibkriging,nlpd,-0.95551 +py01_kernels_trends,python,forrester/matern5_2/quadratic,pylibkriging,fit_s,0.000477076 +py01_kernels_trends,python,forrester/matern5_2/quadratic,pylibkriging,rmse,0.365039 +py01_kernels_trends,python,forrester/matern5_2/quadratic,pylibkriging,q2,0.993231 +py01_kernels_trends,python,forrester/matern5_2/quadratic,pylibkriging,nlpd,-0.790032 +py01_kernels_trends,python,gramacy_lee_2012/gauss/constant,pylibkriging,fit_s,0.0016222 +py01_kernels_trends,python,gramacy_lee_2012/gauss/constant,pylibkriging,rmse,0.191274 +py01_kernels_trends,python,gramacy_lee_2012/gauss/constant,pylibkriging,q2,0.978742 +py01_kernels_trends,python,gramacy_lee_2012/gauss/constant,pylibkriging,nlpd,-0.704948 +py01_kernels_trends,python,gramacy_lee_2012/gauss/linear,pylibkriging,fit_s,0.00240469 +py01_kernels_trends,python,gramacy_lee_2012/gauss/linear,pylibkriging,rmse,0.236028 +py01_kernels_trends,python,gramacy_lee_2012/gauss/linear,pylibkriging,q2,0.96763 +py01_kernels_trends,python,gramacy_lee_2012/gauss/linear,pylibkriging,nlpd,-0.580993 +py01_kernels_trends,python,gramacy_lee_2012/gauss/interactive,pylibkriging,fit_s,0.00205922 +py01_kernels_trends,python,gramacy_lee_2012/gauss/interactive,pylibkriging,rmse,0.236028 +py01_kernels_trends,python,gramacy_lee_2012/gauss/interactive,pylibkriging,q2,0.96763 +py01_kernels_trends,python,gramacy_lee_2012/gauss/interactive,pylibkriging,nlpd,-0.580993 +py01_kernels_trends,python,gramacy_lee_2012/gauss/quadratic,pylibkriging,fit_s,0.00200701 +py01_kernels_trends,python,gramacy_lee_2012/gauss/quadratic,pylibkriging,rmse,0.150914 +py01_kernels_trends,python,gramacy_lee_2012/gauss/quadratic,pylibkriging,q2,0.986767 +py01_kernels_trends,python,gramacy_lee_2012/gauss/quadratic,pylibkriging,nlpd,-0.458431 +py01_kernels_trends,python,gramacy_lee_2012/exp/constant,pylibkriging,fit_s,0.00123 +py01_kernels_trends,python,gramacy_lee_2012/exp/constant,pylibkriging,rmse,0.196586 +py01_kernels_trends,python,gramacy_lee_2012/exp/constant,pylibkriging,q2,0.977545 +py01_kernels_trends,python,gramacy_lee_2012/exp/constant,pylibkriging,nlpd,-0.0621192 +py01_kernels_trends,python,gramacy_lee_2012/exp/linear,pylibkriging,fit_s,0.00172782 +py01_kernels_trends,python,gramacy_lee_2012/exp/linear,pylibkriging,rmse,0.20979 +py01_kernels_trends,python,gramacy_lee_2012/exp/linear,pylibkriging,q2,0.974427 +py01_kernels_trends,python,gramacy_lee_2012/exp/linear,pylibkriging,nlpd,-0.0466568 +py01_kernels_trends,python,gramacy_lee_2012/exp/interactive,pylibkriging,fit_s,0.0017736 +py01_kernels_trends,python,gramacy_lee_2012/exp/interactive,pylibkriging,rmse,0.20979 +py01_kernels_trends,python,gramacy_lee_2012/exp/interactive,pylibkriging,q2,0.974427 +py01_kernels_trends,python,gramacy_lee_2012/exp/interactive,pylibkriging,nlpd,-0.0466568 +py01_kernels_trends,python,gramacy_lee_2012/exp/quadratic,pylibkriging,fit_s,0.00394821 +py01_kernels_trends,python,gramacy_lee_2012/exp/quadratic,pylibkriging,rmse,0.252565 +py01_kernels_trends,python,gramacy_lee_2012/exp/quadratic,pylibkriging,q2,0.962935 +py01_kernels_trends,python,gramacy_lee_2012/exp/quadratic,pylibkriging,nlpd,0.220609 +py01_kernels_trends,python,gramacy_lee_2012/matern3_2/constant,pylibkriging,fit_s,0.000825167 +py01_kernels_trends,python,gramacy_lee_2012/matern3_2/constant,pylibkriging,rmse,0.177253 +py01_kernels_trends,python,gramacy_lee_2012/matern3_2/constant,pylibkriging,q2,0.981744 +py01_kernels_trends,python,gramacy_lee_2012/matern3_2/constant,pylibkriging,nlpd,-0.482212 +py01_kernels_trends,python,gramacy_lee_2012/matern3_2/linear,pylibkriging,fit_s,0.000857115 +py01_kernels_trends,python,gramacy_lee_2012/matern3_2/linear,pylibkriging,rmse,0.211499 +py01_kernels_trends,python,gramacy_lee_2012/matern3_2/linear,pylibkriging,q2,0.974009 +py01_kernels_trends,python,gramacy_lee_2012/matern3_2/linear,pylibkriging,nlpd,-0.375844 +py01_kernels_trends,python,gramacy_lee_2012/matern3_2/interactive,pylibkriging,fit_s,0.00114012 +py01_kernels_trends,python,gramacy_lee_2012/matern3_2/interactive,pylibkriging,rmse,0.211499 +py01_kernels_trends,python,gramacy_lee_2012/matern3_2/interactive,pylibkriging,q2,0.974009 +py01_kernels_trends,python,gramacy_lee_2012/matern3_2/interactive,pylibkriging,nlpd,-0.375844 +py01_kernels_trends,python,gramacy_lee_2012/matern3_2/quadratic,pylibkriging,fit_s,0.00237894 +py01_kernels_trends,python,gramacy_lee_2012/matern3_2/quadratic,pylibkriging,rmse,0.184528 +py01_kernels_trends,python,gramacy_lee_2012/matern3_2/quadratic,pylibkriging,q2,0.980215 +py01_kernels_trends,python,gramacy_lee_2012/matern3_2/quadratic,pylibkriging,nlpd,-0.171795 +py01_kernels_trends,python,gramacy_lee_2012/matern5_2/constant,pylibkriging,fit_s,0.000836849 +py01_kernels_trends,python,gramacy_lee_2012/matern5_2/constant,pylibkriging,rmse,0.177638 +py01_kernels_trends,python,gramacy_lee_2012/matern5_2/constant,pylibkriging,q2,0.981665 +py01_kernels_trends,python,gramacy_lee_2012/matern5_2/constant,pylibkriging,nlpd,-0.575421 +py01_kernels_trends,python,gramacy_lee_2012/matern5_2/constant,pylibkriging BFGS10,rmse,0.177638 +py01_kernels_trends,python,gramacy_lee_2012/matern5_2/constant,pylibkriging BFGS10,q2,0.981665 +py01_kernels_trends,python,gramacy_lee_2012/matern5_2/constant,pylibkriging BFGS10,nlpd,-0.575422 +py01_kernels_trends,python,gramacy_lee_2012/matern5_2/linear,pylibkriging,fit_s,0.000859499 +py01_kernels_trends,python,gramacy_lee_2012/matern5_2/linear,pylibkriging,rmse,0.219759 +py01_kernels_trends,python,gramacy_lee_2012/matern5_2/linear,pylibkriging,q2,0.971939 +py01_kernels_trends,python,gramacy_lee_2012/matern5_2/linear,pylibkriging,nlpd,-0.458933 +py01_kernels_trends,python,gramacy_lee_2012/matern5_2/interactive,pylibkriging,fit_s,0.00080061 +py01_kernels_trends,python,gramacy_lee_2012/matern5_2/interactive,pylibkriging,rmse,0.219759 +py01_kernels_trends,python,gramacy_lee_2012/matern5_2/interactive,pylibkriging,q2,0.971939 +py01_kernels_trends,python,gramacy_lee_2012/matern5_2/interactive,pylibkriging,nlpd,-0.458933 +py01_kernels_trends,python,gramacy_lee_2012/matern5_2/quadratic,pylibkriging,fit_s,0.00249147 +py01_kernels_trends,python,gramacy_lee_2012/matern5_2/quadratic,pylibkriging,rmse,0.169941 +py01_kernels_trends,python,gramacy_lee_2012/matern5_2/quadratic,pylibkriging,q2,0.983219 +py01_kernels_trends,python,gramacy_lee_2012/matern5_2/quadratic,pylibkriging,nlpd,-0.28938 +py01_kernels_trends,python,branin/gauss/constant,pylibkriging,fit_s,0.00172019 +py01_kernels_trends,python,branin/gauss/constant,pylibkriging,rmse,7.2629 +py01_kernels_trends,python,branin/gauss/constant,pylibkriging,q2,0.980655 +py01_kernels_trends,python,branin/gauss/constant,pylibkriging,nlpd,4.3152 +py01_kernels_trends,python,branin/gauss/linear,pylibkriging,fit_s,0.00188136 +py01_kernels_trends,python,branin/gauss/linear,pylibkriging,rmse,7.25733 +py01_kernels_trends,python,branin/gauss/linear,pylibkriging,q2,0.980685 +py01_kernels_trends,python,branin/gauss/linear,pylibkriging,nlpd,4.30813 +py01_kernels_trends,python,branin/gauss/interactive,pylibkriging,fit_s,0.00456238 +py01_kernels_trends,python,branin/gauss/interactive,pylibkriging,rmse,7.2576 +py01_kernels_trends,python,branin/gauss/interactive,pylibkriging,q2,0.980683 +py01_kernels_trends,python,branin/gauss/interactive,pylibkriging,nlpd,4.30176 +py01_kernels_trends,python,branin/gauss/quadratic,pylibkriging,fit_s,0.00182772 +py01_kernels_trends,python,branin/gauss/quadratic,pylibkriging,rmse,7.27808 +py01_kernels_trends,python,branin/gauss/quadratic,pylibkriging,q2,0.980574 +py01_kernels_trends,python,branin/gauss/quadratic,pylibkriging,nlpd,4.29399 +py01_kernels_trends,python,branin/exp/constant,pylibkriging,fit_s,0.00215006 +py01_kernels_trends,python,branin/exp/constant,pylibkriging,rmse,14.5887 +py01_kernels_trends,python,branin/exp/constant,pylibkriging,q2,0.921947 +py01_kernels_trends,python,branin/exp/constant,pylibkriging,nlpd,4.09457 +py01_kernels_trends,python,branin/exp/linear,pylibkriging,fit_s,0.00210071 +py01_kernels_trends,python,branin/exp/linear,pylibkriging,rmse,14.9622 +py01_kernels_trends,python,branin/exp/linear,pylibkriging,q2,0.9179 +py01_kernels_trends,python,branin/exp/linear,pylibkriging,nlpd,4.0909 +py01_kernels_trends,python,branin/exp/interactive,pylibkriging,fit_s,0.00220799 +py01_kernels_trends,python,branin/exp/interactive,pylibkriging,rmse,11.6963 +py01_kernels_trends,python,branin/exp/interactive,pylibkriging,q2,0.94983 +py01_kernels_trends,python,branin/exp/interactive,pylibkriging,nlpd,3.79671 +py01_kernels_trends,python,branin/exp/quadratic,pylibkriging,fit_s,0.00237751 +py01_kernels_trends,python,branin/exp/quadratic,pylibkriging,rmse,9.33909 +py01_kernels_trends,python,branin/exp/quadratic,pylibkriging,q2,0.968014 +py01_kernels_trends,python,branin/exp/quadratic,pylibkriging,nlpd,3.62362 +py01_kernels_trends,python,branin/matern3_2/constant,pylibkriging,fit_s,0.00252843 +py01_kernels_trends,python,branin/matern3_2/constant,pylibkriging,rmse,3.24796 +py01_kernels_trends,python,branin/matern3_2/constant,pylibkriging,q2,0.996131 +py01_kernels_trends,python,branin/matern3_2/constant,pylibkriging,nlpd,2.30444 +py01_kernels_trends,python,branin/matern3_2/linear,pylibkriging,fit_s,0.00200725 +py01_kernels_trends,python,branin/matern3_2/linear,pylibkriging,rmse,3.14402 +py01_kernels_trends,python,branin/matern3_2/linear,pylibkriging,q2,0.996375 +py01_kernels_trends,python,branin/matern3_2/linear,pylibkriging,nlpd,2.28067 +py01_kernels_trends,python,branin/matern3_2/interactive,pylibkriging,fit_s,0.00414944 +py01_kernels_trends,python,branin/matern3_2/interactive,pylibkriging,rmse,3.19226 +py01_kernels_trends,python,branin/matern3_2/interactive,pylibkriging,q2,0.996263 +py01_kernels_trends,python,branin/matern3_2/interactive,pylibkriging,nlpd,2.20359 +py01_kernels_trends,python,branin/matern3_2/quadratic,pylibkriging,fit_s,0.00253224 +py01_kernels_trends,python,branin/matern3_2/quadratic,pylibkriging,rmse,3.01232 +py01_kernels_trends,python,branin/matern3_2/quadratic,pylibkriging,q2,0.996672 +py01_kernels_trends,python,branin/matern3_2/quadratic,pylibkriging,nlpd,2.11627 +py01_kernels_trends,python,branin/matern5_2/constant,pylibkriging,fit_s,0.00171685 +py01_kernels_trends,python,branin/matern5_2/constant,pylibkriging,rmse,0.52012 +py01_kernels_trends,python,branin/matern5_2/constant,pylibkriging,q2,0.999901 +py01_kernels_trends,python,branin/matern5_2/constant,pylibkriging,nlpd,0.283399 +py01_kernels_trends,python,branin/matern5_2/constant,pylibkriging BFGS10,rmse,0.52012 +py01_kernels_trends,python,branin/matern5_2/constant,pylibkriging BFGS10,q2,0.999901 +py01_kernels_trends,python,branin/matern5_2/constant,pylibkriging BFGS10,nlpd,0.283395 +py01_kernels_trends,python,branin/matern5_2/linear,pylibkriging,fit_s,0.0021255 +py01_kernels_trends,python,branin/matern5_2/linear,pylibkriging,rmse,0.460868 +py01_kernels_trends,python,branin/matern5_2/linear,pylibkriging,q2,0.999922 +py01_kernels_trends,python,branin/matern5_2/linear,pylibkriging,nlpd,0.209741 +py01_kernels_trends,python,branin/matern5_2/interactive,pylibkriging,fit_s,0.00307012 +py01_kernels_trends,python,branin/matern5_2/interactive,pylibkriging,rmse,0.498484 +py01_kernels_trends,python,branin/matern5_2/interactive,pylibkriging,q2,0.999909 +py01_kernels_trends,python,branin/matern5_2/interactive,pylibkriging,nlpd,0.239079 +py01_kernels_trends,python,branin/matern5_2/quadratic,pylibkriging,fit_s,0.0092895 +py01_kernels_trends,python,branin/matern5_2/quadratic,pylibkriging,rmse,0.67472 +py01_kernels_trends,python,branin/matern5_2/quadratic,pylibkriging,q2,0.999833 +py01_kernels_trends,python,branin/matern5_2/quadratic,pylibkriging,nlpd,0.419266 +py01_kernels_trends,python,six_hump_camel/gauss/constant,pylibkriging,fit_s,0.00194049 +py01_kernels_trends,python,six_hump_camel/gauss/constant,pylibkriging,rmse,18.9501 +py01_kernels_trends,python,six_hump_camel/gauss/constant,pylibkriging,q2,0.536354 +py01_kernels_trends,python,six_hump_camel/gauss/constant,pylibkriging,nlpd,84.151 +py01_kernels_trends,python,six_hump_camel/gauss/linear,pylibkriging,fit_s,0.00170755 +py01_kernels_trends,python,six_hump_camel/gauss/linear,pylibkriging,rmse,18.9579 +py01_kernels_trends,python,six_hump_camel/gauss/linear,pylibkriging,q2,0.535976 +py01_kernels_trends,python,six_hump_camel/gauss/linear,pylibkriging,nlpd,44.8696 +py01_kernels_trends,python,six_hump_camel/gauss/interactive,pylibkriging,fit_s,0.00180864 +py01_kernels_trends,python,six_hump_camel/gauss/interactive,pylibkriging,rmse,18.9819 +py01_kernels_trends,python,six_hump_camel/gauss/interactive,pylibkriging,q2,0.5348 +py01_kernels_trends,python,six_hump_camel/gauss/interactive,pylibkriging,nlpd,35.2765 +py01_kernels_trends,python,six_hump_camel/gauss/quadratic,pylibkriging,fit_s,0.00178218 +py01_kernels_trends,python,six_hump_camel/gauss/quadratic,pylibkriging,rmse,10.9058 +py01_kernels_trends,python,six_hump_camel/gauss/quadratic,pylibkriging,q2,0.846441 +py01_kernels_trends,python,six_hump_camel/gauss/quadratic,pylibkriging,nlpd,4.92017 +py01_kernels_trends,python,six_hump_camel/exp/constant,pylibkriging,fit_s,0.000977039 +py01_kernels_trends,python,six_hump_camel/exp/constant,pylibkriging,rmse,16.9653 +py01_kernels_trends,python,six_hump_camel/exp/constant,pylibkriging,q2,0.628393 +py01_kernels_trends,python,six_hump_camel/exp/constant,pylibkriging,nlpd,4.48595 +py01_kernels_trends,python,six_hump_camel/exp/linear,pylibkriging,fit_s,0.00107837 +py01_kernels_trends,python,six_hump_camel/exp/linear,pylibkriging,rmse,17.1781 +py01_kernels_trends,python,six_hump_camel/exp/linear,pylibkriging,q2,0.619012 +py01_kernels_trends,python,six_hump_camel/exp/linear,pylibkriging,nlpd,4.47061 +py01_kernels_trends,python,six_hump_camel/exp/interactive,pylibkriging,fit_s,0.00123405 +py01_kernels_trends,python,six_hump_camel/exp/interactive,pylibkriging,rmse,17.1856 +py01_kernels_trends,python,six_hump_camel/exp/interactive,pylibkriging,q2,0.618679 +py01_kernels_trends,python,six_hump_camel/exp/interactive,pylibkriging,nlpd,4.43133 +py01_kernels_trends,python,six_hump_camel/exp/quadratic,pylibkriging,fit_s,0.00120115 +py01_kernels_trends,python,six_hump_camel/exp/quadratic,pylibkriging,rmse,7.99484 +py01_kernels_trends,python,six_hump_camel/exp/quadratic,pylibkriging,q2,0.917476 +py01_kernels_trends,python,six_hump_camel/exp/quadratic,pylibkriging,nlpd,3.35612 +py01_kernels_trends,python,six_hump_camel/matern3_2/constant,pylibkriging,fit_s,0.00264788 +py01_kernels_trends,python,six_hump_camel/matern3_2/constant,pylibkriging,rmse,18.4958 +py01_kernels_trends,python,six_hump_camel/matern3_2/constant,pylibkriging,q2,0.558321 +py01_kernels_trends,python,six_hump_camel/matern3_2/constant,pylibkriging,nlpd,18.8349 +py01_kernels_trends,python,six_hump_camel/matern3_2/linear,pylibkriging,fit_s,0.00251079 +py01_kernels_trends,python,six_hump_camel/matern3_2/linear,pylibkriging,rmse,18.5562 +py01_kernels_trends,python,six_hump_camel/matern3_2/linear,pylibkriging,q2,0.555433 +py01_kernels_trends,python,six_hump_camel/matern3_2/linear,pylibkriging,nlpd,14.9494 +py01_kernels_trends,python,six_hump_camel/matern3_2/interactive,pylibkriging,fit_s,0.00148797 +py01_kernels_trends,python,six_hump_camel/matern3_2/interactive,pylibkriging,rmse,18.561 +py01_kernels_trends,python,six_hump_camel/matern3_2/interactive,pylibkriging,q2,0.555202 +py01_kernels_trends,python,six_hump_camel/matern3_2/interactive,pylibkriging,nlpd,12.9923 +py01_kernels_trends,python,six_hump_camel/matern3_2/quadratic,pylibkriging,fit_s,0.00289869 +py01_kernels_trends,python,six_hump_camel/matern3_2/quadratic,pylibkriging,rmse,8.91645 +py01_kernels_trends,python,six_hump_camel/matern3_2/quadratic,pylibkriging,q2,0.897353 +py01_kernels_trends,python,six_hump_camel/matern3_2/quadratic,pylibkriging,nlpd,3.94939 +py01_kernels_trends,python,six_hump_camel/matern5_2/constant,pylibkriging,fit_s,0.00238967 +py01_kernels_trends,python,six_hump_camel/matern5_2/constant,pylibkriging,rmse,18.3236 +py01_kernels_trends,python,six_hump_camel/matern5_2/constant,pylibkriging,q2,0.566507 +py01_kernels_trends,python,six_hump_camel/matern5_2/constant,pylibkriging,nlpd,18.4322 +py01_kernels_trends,python,six_hump_camel/matern5_2/constant,pylibkriging BFGS10,rmse,1.36208 +py01_kernels_trends,python,six_hump_camel/matern5_2/constant,pylibkriging BFGS10,q2,0.997605 +py01_kernels_trends,python,six_hump_camel/matern5_2/constant,pylibkriging BFGS10,nlpd,1.03215 +py01_kernels_trends,python,six_hump_camel/matern5_2/linear,pylibkriging,fit_s,0.00207353 +py01_kernels_trends,python,six_hump_camel/matern5_2/linear,pylibkriging,rmse,18.7127 +py01_kernels_trends,python,six_hump_camel/matern5_2/linear,pylibkriging,q2,0.547901 +py01_kernels_trends,python,six_hump_camel/matern5_2/linear,pylibkriging,nlpd,24.5771 +py01_kernels_trends,python,six_hump_camel/matern5_2/interactive,pylibkriging,fit_s,0.0020268 +py01_kernels_trends,python,six_hump_camel/matern5_2/interactive,pylibkriging,rmse,18.7194 +py01_kernels_trends,python,six_hump_camel/matern5_2/interactive,pylibkriging,q2,0.547577 +py01_kernels_trends,python,six_hump_camel/matern5_2/interactive,pylibkriging,nlpd,20.4255 +py01_kernels_trends,python,six_hump_camel/matern5_2/quadratic,pylibkriging,fit_s,0.00144219 +py01_kernels_trends,python,six_hump_camel/matern5_2/quadratic,pylibkriging,rmse,10.2878 +py01_kernels_trends,python,six_hump_camel/matern5_2/quadratic,pylibkriging,q2,0.863351 +py01_kernels_trends,python,six_hump_camel/matern5_2/quadratic,pylibkriging,nlpd,4.46344 +py01_kernels_trends,python,log_goldstein_price/gauss/constant,pylibkriging,fit_s,0.00526643 +py01_kernels_trends,python,log_goldstein_price/gauss/constant,pylibkriging,rmse,0.295934 +py01_kernels_trends,python,log_goldstein_price/gauss/constant,pylibkriging,q2,0.920537 +py01_kernels_trends,python,log_goldstein_price/gauss/constant,pylibkriging,nlpd,1.58666 +py01_kernels_trends,python,log_goldstein_price/gauss/linear,pylibkriging,fit_s,0.00606179 +py01_kernels_trends,python,log_goldstein_price/gauss/linear,pylibkriging,rmse,0.296355 +py01_kernels_trends,python,log_goldstein_price/gauss/linear,pylibkriging,q2,0.92031 +py01_kernels_trends,python,log_goldstein_price/gauss/linear,pylibkriging,nlpd,1.2962 +py01_kernels_trends,python,log_goldstein_price/gauss/interactive,pylibkriging,fit_s,0.00633311 +py01_kernels_trends,python,log_goldstein_price/gauss/interactive,pylibkriging,rmse,0.285136 +py01_kernels_trends,python,log_goldstein_price/gauss/interactive,pylibkriging,q2,0.926229 +py01_kernels_trends,python,log_goldstein_price/gauss/interactive,pylibkriging,nlpd,1.04122 +py01_kernels_trends,python,log_goldstein_price/gauss/quadratic,pylibkriging,fit_s,0.00615239 +py01_kernels_trends,python,log_goldstein_price/gauss/quadratic,pylibkriging,rmse,0.237308 +py01_kernels_trends,python,log_goldstein_price/gauss/quadratic,pylibkriging,q2,0.948902 +py01_kernels_trends,python,log_goldstein_price/gauss/quadratic,pylibkriging,nlpd,0.146484 +py01_kernels_trends,python,log_goldstein_price/exp/constant,pylibkriging,fit_s,0.00271893 +py01_kernels_trends,python,log_goldstein_price/exp/constant,pylibkriging,rmse,0.334876 +py01_kernels_trends,python,log_goldstein_price/exp/constant,pylibkriging,q2,0.898247 +py01_kernels_trends,python,log_goldstein_price/exp/constant,pylibkriging,nlpd,0.339235 +py01_kernels_trends,python,log_goldstein_price/exp/linear,pylibkriging,fit_s,0.00223827 +py01_kernels_trends,python,log_goldstein_price/exp/linear,pylibkriging,rmse,0.333447 +py01_kernels_trends,python,log_goldstein_price/exp/linear,pylibkriging,q2,0.899114 +py01_kernels_trends,python,log_goldstein_price/exp/linear,pylibkriging,nlpd,0.336515 +py01_kernels_trends,python,log_goldstein_price/exp/interactive,pylibkriging,fit_s,0.00192547 +py01_kernels_trends,python,log_goldstein_price/exp/interactive,pylibkriging,rmse,0.336819 +py01_kernels_trends,python,log_goldstein_price/exp/interactive,pylibkriging,q2,0.897063 +py01_kernels_trends,python,log_goldstein_price/exp/interactive,pylibkriging,nlpd,0.329566 +py01_kernels_trends,python,log_goldstein_price/exp/quadratic,pylibkriging,fit_s,0.00333118 +py01_kernels_trends,python,log_goldstein_price/exp/quadratic,pylibkriging,rmse,0.327816 +py01_kernels_trends,python,log_goldstein_price/exp/quadratic,pylibkriging,q2,0.902492 +py01_kernels_trends,python,log_goldstein_price/exp/quadratic,pylibkriging,nlpd,0.322056 +py01_kernels_trends,python,log_goldstein_price/matern3_2/constant,pylibkriging,fit_s,0.00606227 +py01_kernels_trends,python,log_goldstein_price/matern3_2/constant,pylibkriging,rmse,0.684362 +py01_kernels_trends,python,log_goldstein_price/matern3_2/constant,pylibkriging,q2,0.575038 +py01_kernels_trends,python,log_goldstein_price/matern3_2/constant,pylibkriging,nlpd,0.972803 +py01_kernels_trends,python,log_goldstein_price/matern3_2/linear,pylibkriging,fit_s,0.00382805 +py01_kernels_trends,python,log_goldstein_price/matern3_2/linear,pylibkriging,rmse,0.650201 +py01_kernels_trends,python,log_goldstein_price/matern3_2/linear,pylibkriging,q2,0.616404 +py01_kernels_trends,python,log_goldstein_price/matern3_2/linear,pylibkriging,nlpd,0.921455 +py01_kernels_trends,python,log_goldstein_price/matern3_2/interactive,pylibkriging,fit_s,0.00707507 +py01_kernels_trends,python,log_goldstein_price/matern3_2/interactive,pylibkriging,rmse,0.274326 +py01_kernels_trends,python,log_goldstein_price/matern3_2/interactive,pylibkriging,q2,0.931717 +py01_kernels_trends,python,log_goldstein_price/matern3_2/interactive,pylibkriging,nlpd,0.0549679 +py01_kernels_trends,python,log_goldstein_price/matern3_2/quadratic,pylibkriging,fit_s,0.00617123 +py01_kernels_trends,python,log_goldstein_price/matern3_2/quadratic,pylibkriging,rmse,0.263745 +py01_kernels_trends,python,log_goldstein_price/matern3_2/quadratic,pylibkriging,q2,0.936883 +py01_kernels_trends,python,log_goldstein_price/matern3_2/quadratic,pylibkriging,nlpd,0.0446267 +py01_kernels_trends,python,log_goldstein_price/matern5_2/constant,pylibkriging,fit_s,0.00426936 +py01_kernels_trends,python,log_goldstein_price/matern5_2/constant,pylibkriging,rmse,0.674391 +py01_kernels_trends,python,log_goldstein_price/matern5_2/constant,pylibkriging,q2,0.58733 +py01_kernels_trends,python,log_goldstein_price/matern5_2/constant,pylibkriging,nlpd,0.944765 +py01_kernels_trends,python,log_goldstein_price/matern5_2/constant,pylibkriging BFGS10,rmse,0.268216 +py01_kernels_trends,python,log_goldstein_price/matern5_2/constant,pylibkriging BFGS10,q2,0.934725 +py01_kernels_trends,python,log_goldstein_price/matern5_2/constant,pylibkriging BFGS10,nlpd,0.169585 +py01_kernels_trends,python,log_goldstein_price/matern5_2/linear,pylibkriging,fit_s,0.00730562 +py01_kernels_trends,python,log_goldstein_price/matern5_2/linear,pylibkriging,rmse,0.268808 +py01_kernels_trends,python,log_goldstein_price/matern5_2/linear,pylibkriging,q2,0.934436 +py01_kernels_trends,python,log_goldstein_price/matern5_2/linear,pylibkriging,nlpd,0.142382 +py01_kernels_trends,python,log_goldstein_price/matern5_2/interactive,pylibkriging,fit_s,0.00520349 +py01_kernels_trends,python,log_goldstein_price/matern5_2/interactive,pylibkriging,rmse,0.271088 +py01_kernels_trends,python,log_goldstein_price/matern5_2/interactive,pylibkriging,q2,0.933319 +py01_kernels_trends,python,log_goldstein_price/matern5_2/interactive,pylibkriging,nlpd,0.128788 +py01_kernels_trends,python,log_goldstein_price/matern5_2/quadratic,pylibkriging,fit_s,0.00757623 +py01_kernels_trends,python,log_goldstein_price/matern5_2/quadratic,pylibkriging,rmse,0.253093 +py01_kernels_trends,python,log_goldstein_price/matern5_2/quadratic,pylibkriging,q2,0.941878 +py01_kernels_trends,python,log_goldstein_price/matern5_2/quadratic,pylibkriging,nlpd,0.025469 +py01_kernels_trends,python,lim_polynomial/gauss/constant,pylibkriging,fit_s,0.000713825 +py01_kernels_trends,python,lim_polynomial/gauss/constant,pylibkriging,rmse,0.026127 +py01_kernels_trends,python,lim_polynomial/gauss/constant,pylibkriging,q2,0.999737 +py01_kernels_trends,python,lim_polynomial/gauss/constant,pylibkriging,nlpd,-2.69817 +py01_kernels_trends,python,lim_polynomial/gauss/linear,pylibkriging,fit_s,0.000786066 +py01_kernels_trends,python,lim_polynomial/gauss/linear,pylibkriging,rmse,1.06243 +py01_kernels_trends,python,lim_polynomial/gauss/linear,pylibkriging,q2,0.564691 +py01_kernels_trends,python,lim_polynomial/gauss/linear,pylibkriging,nlpd,2.7394 +py01_kernels_trends,python,lim_polynomial/gauss/interactive,pylibkriging,fit_s,0.000851393 +py01_kernels_trends,python,lim_polynomial/gauss/interactive,pylibkriging,rmse,1.06504 +py01_kernels_trends,python,lim_polynomial/gauss/interactive,pylibkriging,q2,0.562551 +py01_kernels_trends,python,lim_polynomial/gauss/interactive,pylibkriging,nlpd,2.33575 +py01_kernels_trends,python,lim_polynomial/gauss/quadratic,pylibkriging,fit_s,0.0012238 +py01_kernels_trends,python,lim_polynomial/gauss/quadratic,pylibkriging,rmse,0.281816 +py01_kernels_trends,python,lim_polynomial/gauss/quadratic,pylibkriging,q2,0.969371 +py01_kernels_trends,python,lim_polynomial/gauss/quadratic,pylibkriging,nlpd,0.71682 +py01_kernels_trends,python,lim_polynomial/exp/constant,pylibkriging,fit_s,0.000771999 +py01_kernels_trends,python,lim_polynomial/exp/constant,pylibkriging,rmse,0.249023 +py01_kernels_trends,python,lim_polynomial/exp/constant,pylibkriging,q2,0.976085 +py01_kernels_trends,python,lim_polynomial/exp/constant,pylibkriging,nlpd,0.394159 +py01_kernels_trends,python,lim_polynomial/exp/linear,pylibkriging,fit_s,0.0006814 +py01_kernels_trends,python,lim_polynomial/exp/linear,pylibkriging,rmse,0.245558 +py01_kernels_trends,python,lim_polynomial/exp/linear,pylibkriging,q2,0.976746 +py01_kernels_trends,python,lim_polynomial/exp/linear,pylibkriging,nlpd,0.224593 +py01_kernels_trends,python,lim_polynomial/exp/interactive,pylibkriging,fit_s,0.000779867 +py01_kernels_trends,python,lim_polynomial/exp/interactive,pylibkriging,rmse,0.239032 +py01_kernels_trends,python,lim_polynomial/exp/interactive,pylibkriging,q2,0.977965 +py01_kernels_trends,python,lim_polynomial/exp/interactive,pylibkriging,nlpd,0.226133 +py01_kernels_trends,python,lim_polynomial/exp/quadratic,pylibkriging,fit_s,0.00164485 +py01_kernels_trends,python,lim_polynomial/exp/quadratic,pylibkriging,rmse,0.237713 +py01_kernels_trends,python,lim_polynomial/exp/quadratic,pylibkriging,q2,0.978208 +py01_kernels_trends,python,lim_polynomial/exp/quadratic,pylibkriging,nlpd,-0.139582 +py01_kernels_trends,python,lim_polynomial/matern3_2/constant,pylibkriging,fit_s,0.000581741 +py01_kernels_trends,python,lim_polynomial/matern3_2/constant,pylibkriging,rmse,0.0600911 +py01_kernels_trends,python,lim_polynomial/matern3_2/constant,pylibkriging,q2,0.998607 +py01_kernels_trends,python,lim_polynomial/matern3_2/constant,pylibkriging,nlpd,-1.34658 +py01_kernels_trends,python,lim_polynomial/matern3_2/linear,pylibkriging,fit_s,0.000698566 +py01_kernels_trends,python,lim_polynomial/matern3_2/linear,pylibkriging,rmse,0.0584376 +py01_kernels_trends,python,lim_polynomial/matern3_2/linear,pylibkriging,q2,0.998683 +py01_kernels_trends,python,lim_polynomial/matern3_2/linear,pylibkriging,nlpd,-1.34912 +py01_kernels_trends,python,lim_polynomial/matern3_2/interactive,pylibkriging,fit_s,0.000763416 +py01_kernels_trends,python,lim_polynomial/matern3_2/interactive,pylibkriging,rmse,0.0576096 +py01_kernels_trends,python,lim_polynomial/matern3_2/interactive,pylibkriging,q2,0.99872 +py01_kernels_trends,python,lim_polynomial/matern3_2/interactive,pylibkriging,nlpd,-1.35376 +py01_kernels_trends,python,lim_polynomial/matern3_2/quadratic,pylibkriging,fit_s,0.00313497 +py01_kernels_trends,python,lim_polynomial/matern3_2/quadratic,pylibkriging,rmse,0.215336 +py01_kernels_trends,python,lim_polynomial/matern3_2/quadratic,pylibkriging,q2,0.982117 +py01_kernels_trends,python,lim_polynomial/matern3_2/quadratic,pylibkriging,nlpd,-0.350837 +py01_kernels_trends,python,lim_polynomial/matern5_2/constant,pylibkriging,fit_s,0.000642776 +py01_kernels_trends,python,lim_polynomial/matern5_2/constant,pylibkriging,rmse,0.0246348 +py01_kernels_trends,python,lim_polynomial/matern5_2/constant,pylibkriging,q2,0.999766 +py01_kernels_trends,python,lim_polynomial/matern5_2/constant,pylibkriging,nlpd,-2.59598 +py01_kernels_trends,python,lim_polynomial/matern5_2/constant,pylibkriging BFGS10,rmse,0.0246338 +py01_kernels_trends,python,lim_polynomial/matern5_2/constant,pylibkriging BFGS10,q2,0.999766 +py01_kernels_trends,python,lim_polynomial/matern5_2/constant,pylibkriging BFGS10,nlpd,-2.59598 +py01_kernels_trends,python,lim_polynomial/matern5_2/linear,pylibkriging,fit_s,0.000878334 +py01_kernels_trends,python,lim_polynomial/matern5_2/linear,pylibkriging,rmse,0.0232483 +py01_kernels_trends,python,lim_polynomial/matern5_2/linear,pylibkriging,q2,0.999792 +py01_kernels_trends,python,lim_polynomial/matern5_2/linear,pylibkriging,nlpd,-2.58843 +py01_kernels_trends,python,lim_polynomial/matern5_2/interactive,pylibkriging,fit_s,0.000845909 +py01_kernels_trends,python,lim_polynomial/matern5_2/interactive,pylibkriging,rmse,0.0230248 +py01_kernels_trends,python,lim_polynomial/matern5_2/interactive,pylibkriging,q2,0.999796 +py01_kernels_trends,python,lim_polynomial/matern5_2/interactive,pylibkriging,nlpd,-2.58839 +py01_kernels_trends,python,lim_polynomial/matern5_2/quadratic,pylibkriging,fit_s,0.00176644 +py01_kernels_trends,python,lim_polynomial/matern5_2/quadratic,pylibkriging,rmse,0.209802 +py01_kernels_trends,python,lim_polynomial/matern5_2/quadratic,pylibkriging,q2,0.983025 +py01_kernels_trends,python,lim_polynomial/matern5_2/quadratic,pylibkriging,nlpd,-0.403998 +py01_kernels_trends,python,ishigami/gauss/constant,pylibkriging,fit_s,0.0820026 +py01_kernels_trends,python,ishigami/gauss/constant,pylibkriging,rmse,1.41975 +py01_kernels_trends,python,ishigami/gauss/constant,pylibkriging,q2,0.844642 +py01_kernels_trends,python,ishigami/gauss/constant,pylibkriging,nlpd,2.25894 +py01_kernels_trends,python,ishigami/gauss/linear,pylibkriging,fit_s,0.0247335 +py01_kernels_trends,python,ishigami/gauss/linear,pylibkriging,rmse,2.84579 +py01_kernels_trends,python,ishigami/gauss/linear,pylibkriging,q2,0.375811 +py01_kernels_trends,python,ishigami/gauss/linear,pylibkriging,nlpd,3.41805 +py01_kernels_trends,python,ishigami/gauss/interactive,pylibkriging,fit_s,0.0169995 +py01_kernels_trends,python,ishigami/gauss/interactive,pylibkriging,rmse,2.85366 +py01_kernels_trends,python,ishigami/gauss/interactive,pylibkriging,q2,0.372353 +py01_kernels_trends,python,ishigami/gauss/interactive,pylibkriging,nlpd,3.28643 +py01_kernels_trends,python,ishigami/gauss/quadratic,pylibkriging,fit_s,0.00751019 +py01_kernels_trends,python,ishigami/gauss/quadratic,pylibkriging,rmse,2.08982 +py01_kernels_trends,python,ishigami/gauss/quadratic,pylibkriging,q2,0.66339 +py01_kernels_trends,python,ishigami/gauss/quadratic,pylibkriging,nlpd,2.80099 +py01_kernels_trends,python,ishigami/exp/constant,pylibkriging,fit_s,0.00825953 +py01_kernels_trends,python,ishigami/exp/constant,pylibkriging,rmse,1.41632 +py01_kernels_trends,python,ishigami/exp/constant,pylibkriging,q2,0.845391 +py01_kernels_trends,python,ishigami/exp/constant,pylibkriging,nlpd,1.75239 +py01_kernels_trends,python,ishigami/exp/linear,pylibkriging,fit_s,0.00724149 +py01_kernels_trends,python,ishigami/exp/linear,pylibkriging,rmse,1.4095 +py01_kernels_trends,python,ishigami/exp/linear,pylibkriging,q2,0.846877 +py01_kernels_trends,python,ishigami/exp/linear,pylibkriging,nlpd,1.74779 +py01_kernels_trends,python,ishigami/exp/interactive,pylibkriging,fit_s,0.00750446 +py01_kernels_trends,python,ishigami/exp/interactive,pylibkriging,rmse,1.41511 +py01_kernels_trends,python,ishigami/exp/interactive,pylibkriging,q2,0.845657 +py01_kernels_trends,python,ishigami/exp/interactive,pylibkriging,nlpd,1.75024 +py01_kernels_trends,python,ishigami/exp/quadratic,pylibkriging,fit_s,0.00750661 +py01_kernels_trends,python,ishigami/exp/quadratic,pylibkriging,rmse,1.40329 +py01_kernels_trends,python,ishigami/exp/quadratic,pylibkriging,q2,0.848223 +py01_kernels_trends,python,ishigami/exp/quadratic,pylibkriging,nlpd,1.73254 +py01_kernels_trends,python,ishigami/matern3_2/constant,pylibkriging,fit_s,0.0483243 +py01_kernels_trends,python,ishigami/matern3_2/constant,pylibkriging,rmse,1.32974 +py01_kernels_trends,python,ishigami/matern3_2/constant,pylibkriging,q2,0.863717 +py01_kernels_trends,python,ishigami/matern3_2/constant,pylibkriging,nlpd,1.50266 +py01_kernels_trends,python,ishigami/matern3_2/linear,pylibkriging,fit_s,0.0462224 +py01_kernels_trends,python,ishigami/matern3_2/linear,pylibkriging,rmse,1.30244 +py01_kernels_trends,python,ishigami/matern3_2/linear,pylibkriging,q2,0.869255 +py01_kernels_trends,python,ishigami/matern3_2/linear,pylibkriging,nlpd,1.47906 +py01_kernels_trends,python,ishigami/matern3_2/interactive,pylibkriging,fit_s,0.0462408 +py01_kernels_trends,python,ishigami/matern3_2/interactive,pylibkriging,rmse,1.30591 +py01_kernels_trends,python,ishigami/matern3_2/interactive,pylibkriging,q2,0.868558 +py01_kernels_trends,python,ishigami/matern3_2/interactive,pylibkriging,nlpd,1.47384 +py01_kernels_trends,python,ishigami/matern3_2/quadratic,pylibkriging,fit_s,0.0412879 +py01_kernels_trends,python,ishigami/matern3_2/quadratic,pylibkriging,rmse,1.32197 +py01_kernels_trends,python,ishigami/matern3_2/quadratic,pylibkriging,q2,0.865305 +py01_kernels_trends,python,ishigami/matern3_2/quadratic,pylibkriging,nlpd,1.46827 +py01_kernels_trends,python,ishigami/matern5_2/constant,pylibkriging,fit_s,0.02212 +py01_kernels_trends,python,ishigami/matern5_2/constant,pylibkriging,rmse,3.00566 +py01_kernels_trends,python,ishigami/matern5_2/constant,pylibkriging,q2,0.303709 +py01_kernels_trends,python,ishigami/matern5_2/constant,pylibkriging,nlpd,3.42085 +py01_kernels_trends,python,ishigami/matern5_2/constant,pylibkriging BFGS10,rmse,1.33903 +py01_kernels_trends,python,ishigami/matern5_2/constant,pylibkriging BFGS10,q2,0.861806 +py01_kernels_trends,python,ishigami/matern5_2/constant,pylibkriging BFGS10,nlpd,1.66132 +py01_kernels_trends,python,ishigami/matern5_2/linear,pylibkriging,fit_s,0.0472796 +py01_kernels_trends,python,ishigami/matern5_2/linear,pylibkriging,rmse,1.3434 +py01_kernels_trends,python,ishigami/matern5_2/linear,pylibkriging,q2,0.860901 +py01_kernels_trends,python,ishigami/matern5_2/linear,pylibkriging,nlpd,1.65103 +py01_kernels_trends,python,ishigami/matern5_2/interactive,pylibkriging,fit_s,0.0477624 +py01_kernels_trends,python,ishigami/matern5_2/interactive,pylibkriging,rmse,1.34479 +py01_kernels_trends,python,ishigami/matern5_2/interactive,pylibkriging,q2,0.860615 +py01_kernels_trends,python,ishigami/matern5_2/interactive,pylibkriging,nlpd,1.64153 +py01_kernels_trends,python,ishigami/matern5_2/quadratic,pylibkriging,fit_s,0.0483804 +py01_kernels_trends,python,ishigami/matern5_2/quadratic,pylibkriging,rmse,1.37082 +py01_kernels_trends,python,ishigami/matern5_2/quadratic,pylibkriging,q2,0.855166 +py01_kernels_trends,python,ishigami/matern5_2/quadratic,pylibkriging,nlpd,1.62228 +py01_kernels_trends,python,hartmann3/gauss/constant,pylibkriging,fit_s,0.00703788 +py01_kernels_trends,python,hartmann3/gauss/constant,pylibkriging,rmse,0.156226 +py01_kernels_trends,python,hartmann3/gauss/constant,pylibkriging,q2,0.971116 +py01_kernels_trends,python,hartmann3/gauss/constant,pylibkriging,nlpd,-0.555623 +py01_kernels_trends,python,hartmann3/gauss/linear,pylibkriging,fit_s,0.0116901 +py01_kernels_trends,python,hartmann3/gauss/linear,pylibkriging,rmse,0.153297 +py01_kernels_trends,python,hartmann3/gauss/linear,pylibkriging,q2,0.972189 +py01_kernels_trends,python,hartmann3/gauss/linear,pylibkriging,nlpd,-0.568404 +py01_kernels_trends,python,hartmann3/gauss/interactive,pylibkriging,fit_s,0.00962472 +py01_kernels_trends,python,hartmann3/gauss/interactive,pylibkriging,rmse,0.712947 +py01_kernels_trends,python,hartmann3/gauss/interactive,pylibkriging,q2,0.398455 +py01_kernels_trends,python,hartmann3/gauss/interactive,pylibkriging,nlpd,1.74554 +py01_kernels_trends,python,hartmann3/gauss/quadratic,pylibkriging,fit_s,0.00845146 +py01_kernels_trends,python,hartmann3/gauss/quadratic,pylibkriging,rmse,0.73238 +py01_kernels_trends,python,hartmann3/gauss/quadratic,pylibkriging,q2,0.365216 +py01_kernels_trends,python,hartmann3/gauss/quadratic,pylibkriging,nlpd,2.14588 +py01_kernels_trends,python,hartmann3/exp/constant,pylibkriging,fit_s,0.00484872 +py01_kernels_trends,python,hartmann3/exp/constant,pylibkriging,rmse,0.347348 +py01_kernels_trends,python,hartmann3/exp/constant,pylibkriging,q2,0.857215 +py01_kernels_trends,python,hartmann3/exp/constant,pylibkriging,nlpd,0.352683 +py01_kernels_trends,python,hartmann3/exp/linear,pylibkriging,fit_s,0.00318956 +py01_kernels_trends,python,hartmann3/exp/linear,pylibkriging,rmse,0.347273 +py01_kernels_trends,python,hartmann3/exp/linear,pylibkriging,q2,0.857276 +py01_kernels_trends,python,hartmann3/exp/linear,pylibkriging,nlpd,0.343631 +py01_kernels_trends,python,hartmann3/exp/interactive,pylibkriging,fit_s,0.00330567 +py01_kernels_trends,python,hartmann3/exp/interactive,pylibkriging,rmse,0.378349 +py01_kernels_trends,python,hartmann3/exp/interactive,pylibkriging,q2,0.83059 +py01_kernels_trends,python,hartmann3/exp/interactive,pylibkriging,nlpd,0.385842 +py01_kernels_trends,python,hartmann3/exp/quadratic,pylibkriging,fit_s,0.00355411 +py01_kernels_trends,python,hartmann3/exp/quadratic,pylibkriging,rmse,0.355119 +py01_kernels_trends,python,hartmann3/exp/quadratic,pylibkriging,q2,0.850755 +py01_kernels_trends,python,hartmann3/exp/quadratic,pylibkriging,nlpd,0.334371 +py01_kernels_trends,python,hartmann3/matern3_2/constant,pylibkriging,fit_s,0.0107734 +py01_kernels_trends,python,hartmann3/matern3_2/constant,pylibkriging,rmse,0.227521 +py01_kernels_trends,python,hartmann3/matern3_2/constant,pylibkriging,q2,0.938737 +py01_kernels_trends,python,hartmann3/matern3_2/constant,pylibkriging,nlpd,-0.181207 +py01_kernels_trends,python,hartmann3/matern3_2/linear,pylibkriging,fit_s,0.0118749 +py01_kernels_trends,python,hartmann3/matern3_2/linear,pylibkriging,rmse,0.23038 +py01_kernels_trends,python,hartmann3/matern3_2/linear,pylibkriging,q2,0.937188 +py01_kernels_trends,python,hartmann3/matern3_2/linear,pylibkriging,nlpd,-0.162203 +py01_kernels_trends,python,hartmann3/matern3_2/interactive,pylibkriging,fit_s,0.00637007 +py01_kernels_trends,python,hartmann3/matern3_2/interactive,pylibkriging,rmse,0.315752 +py01_kernels_trends,python,hartmann3/matern3_2/interactive,pylibkriging,q2,0.88201 +py01_kernels_trends,python,hartmann3/matern3_2/interactive,pylibkriging,nlpd,0.898253 +py01_kernels_trends,python,hartmann3/matern3_2/quadratic,pylibkriging,fit_s,0.00598764 +py01_kernels_trends,python,hartmann3/matern3_2/quadratic,pylibkriging,rmse,0.291393 +py01_kernels_trends,python,hartmann3/matern3_2/quadratic,pylibkriging,q2,0.899513 +py01_kernels_trends,python,hartmann3/matern3_2/quadratic,pylibkriging,nlpd,0.53743 +py01_kernels_trends,python,hartmann3/matern5_2/constant,pylibkriging,fit_s,0.0100911 +py01_kernels_trends,python,hartmann3/matern5_2/constant,pylibkriging,rmse,0.190287 +py01_kernels_trends,python,hartmann3/matern5_2/constant,pylibkriging,q2,0.957148 +py01_kernels_trends,python,hartmann3/matern5_2/constant,pylibkriging,nlpd,-0.403054 +py01_kernels_trends,python,hartmann3/matern5_2/constant,pylibkriging BFGS10,rmse,0.190286 +py01_kernels_trends,python,hartmann3/matern5_2/constant,pylibkriging BFGS10,q2,0.957148 +py01_kernels_trends,python,hartmann3/matern5_2/constant,pylibkriging BFGS10,nlpd,-0.40306 +py01_kernels_trends,python,hartmann3/matern5_2/linear,pylibkriging,fit_s,0.0106812 +py01_kernels_trends,python,hartmann3/matern5_2/linear,pylibkriging,rmse,0.189633 +py01_kernels_trends,python,hartmann3/matern5_2/linear,pylibkriging,q2,0.957442 +py01_kernels_trends,python,hartmann3/matern5_2/linear,pylibkriging,nlpd,-0.410075 +py01_kernels_trends,python,hartmann3/matern5_2/interactive,pylibkriging,fit_s,0.00839925 +py01_kernels_trends,python,hartmann3/matern5_2/interactive,pylibkriging,rmse,0.307114 +py01_kernels_trends,python,hartmann3/matern5_2/interactive,pylibkriging,q2,0.888377 +py01_kernels_trends,python,hartmann3/matern5_2/interactive,pylibkriging,nlpd,1.50063 +py01_kernels_trends,python,hartmann3/matern5_2/quadratic,pylibkriging,fit_s,0.00724792 +py01_kernels_trends,python,hartmann3/matern5_2/quadratic,pylibkriging,rmse,0.285355 +py01_kernels_trends,python,hartmann3/matern5_2/quadratic,pylibkriging,q2,0.903634 +py01_kernels_trends,python,hartmann3/matern5_2/quadratic,pylibkriging,nlpd,0.898128 +py02_objectives_gradients,python,branin/LL,pylibkriging,rmse,71.4801 +py02_objectives_gradients,python,branin/LL,pylibkriging,q2,-1991.62 +py02_objectives_gradients,python,branin/LL,pylibkriging,nlpd,13603.3 +py02_objectives_gradients,python,branin/LOO,pylibkriging,rmse,71.5631 +py02_objectives_gradients,python,branin/LOO,pylibkriging,q2,-1996.25 +py02_objectives_gradients,python,branin/LOO,pylibkriging,nlpd,18836.2 +py02_objectives_gradients,python,branin/LMP,pylibkriging,rmse,71.5295 +py02_objectives_gradients,python,branin/LMP,pylibkriging,q2,-1994.38 +py02_objectives_gradients,python,branin/LMP,pylibkriging,nlpd,14603.3 +py02_objectives_gradients,python,hartmann3/LL,pylibkriging,rmse,1.30952 +py02_objectives_gradients,python,hartmann3/LL,pylibkriging,q2,-2013.29 +py02_objectives_gradients,python,hartmann3/LL,pylibkriging,nlpd,41.7128 +py02_objectives_gradients,python,hartmann3/LOO,pylibkriging,rmse,1.31436 +py02_objectives_gradients,python,hartmann3/LOO,pylibkriging,q2,-2028.22 +py02_objectives_gradients,python,hartmann3/LOO,pylibkriging,nlpd,49.7135 +py02_objectives_gradients,python,hartmann3/LMP,pylibkriging,rmse,1.31148 +py02_objectives_gradients,python,hartmann3/LMP,pylibkriging,q2,-2019.34 +py02_objectives_gradients,python,hartmann3/LMP,pylibkriging,nlpd,43.397 +py02_objectives_gradients,python,ishigami/LL,pylibkriging,rmse,5.11103 +py02_objectives_gradients,python,ishigami/LL,pylibkriging,q2,-1783.42 +py02_objectives_gradients,python,ishigami/LL,pylibkriging,nlpd,10.4546 +py02_objectives_gradients,python,ishigami/LOO,pylibkriging,rmse,5.81789 +py02_objectives_gradients,python,ishigami/LOO,pylibkriging,q2,-2311.12 +py02_objectives_gradients,python,ishigami/LOO,pylibkriging,nlpd,40.8323 +py02_objectives_gradients,python,ishigami/LMP,pylibkriging,rmse,5.18495 +py02_objectives_gradients,python,ishigami/LMP,pylibkriging,q2,-1835.41 +py02_objectives_gradients,python,ishigami/LMP,pylibkriging,nlpd,11.5894 +py02_objectives_gradients,python,friedman/LL,pylibkriging,rmse,6.87533 +py02_objectives_gradients,python,friedman/LL,pylibkriging,q2,-1991.33 +py02_objectives_gradients,python,friedman/LL,pylibkriging,nlpd,651.706 +py02_objectives_gradients,python,friedman/LOO,pylibkriging,rmse,6.87323 +py02_objectives_gradients,python,friedman/LOO,pylibkriging,q2,-1990.12 +py02_objectives_gradients,python,friedman/LOO,pylibkriging,nlpd,484.115 +py02_objectives_gradients,python,friedman/LMP,pylibkriging,rmse,6.87647 +py02_objectives_gradients,python,friedman/LMP,pylibkriging,q2,-1991.99 +py02_objectives_gradients,python,friedman/LMP,pylibkriging,nlpd,658.808 +py03_noise,python,forrester,nugget,rmse,0.198091 +py03_noise,python,forrester,nugget,q2,0.998087 +py03_noise,python,forrester,nugget,nlpd,0.286631 +py03_noise,python,forrester,noise-free fit,rmse,0.487916 +py03_noise,python,forrester,noise-free fit,q2,0.988392 +py03_noise,python,forrester,noise-free fit,nlpd,862.687 +py03_noise,python,friedman,nugget,rmse,0.467335 +py03_noise,python,friedman,nugget,q2,0.99089 +py03_noise,python,friedman,nugget,nlpd,1.07073 +py03_noise,python,friedman,noise-free fit,rmse,1.27156 +py03_noise,python,friedman,noise-free fit,q2,0.932559 +py03_noise,python,friedman,noise-free fit,nlpd,1.64495 +py03_noise,python,gramacy_lee_2012/hetero,known variances,rmse,0.168741 +py03_noise,python,gramacy_lee_2012/hetero,known variances,q2,0.982515 +py03_noise,python,gramacy_lee_2012/hetero,known variances,nlpd,-0.570574 +py03_noise,python,gramacy_lee_2012/hetero,nugget,rmse,0.170945 +py03_noise,python,gramacy_lee_2012/hetero,nugget,q2,0.982055 +py03_noise,python,gramacy_lee_2012/hetero,nugget,nlpd,-0.221568 +py05_warpkriging,python,gl2012,Kriging,rmse,0.167263 +py05_warpkriging,python,gl2012,Kriging,q2,0.981176 +py05_warpkriging,python,gl2012,Kriging,nlpd,-1.08917 +py05_warpkriging,python,gl2012,affine,rmse,0.167263 +py05_warpkriging,python,gl2012,affine,q2,0.981176 +py05_warpkriging,python,gl2012,affine,nlpd,-1.08917 +py05_warpkriging,python,gl2012,boxcox,rmse,0.16759 +py05_warpkriging,python,gl2012,boxcox,q2,0.981103 +py05_warpkriging,python,gl2012,boxcox,nlpd,282.422 +py05_warpkriging,python,gl2012,kumaraswamy,rmse,0.168286 +py05_warpkriging,python,gl2012,kumaraswamy,q2,0.980945 +py05_warpkriging,python,gl2012,kumaraswamy,nlpd,2.04577e+06 +py05_warpkriging,python,gl2012,knots(5),rmse,0.16791 +py05_warpkriging,python,gl2012,knots(5),q2,0.98103 +py05_warpkriging,python,gl2012,knots(5),nlpd,2.04376e+06 +py05_warpkriging,python,gl2012,neural_mono(8),rmse,0.170824 +py05_warpkriging,python,gl2012,neural_mono(8),q2,0.980366 +py05_warpkriging,python,gl2012,neural_mono(8),nlpd,-0.265141 +py05_warpkriging,python,gl2008,Kriging,rmse,0.0534649 +py05_warpkriging,python,gl2008,Kriging,q2,0.585636 +py05_warpkriging,python,gl2008,Kriging,nlpd,-1.74569 +py05_warpkriging,python,gl2008,kumaraswamy,rmse,0.0535298 +py05_warpkriging,python,gl2008,kumaraswamy,q2,0.58463 +py05_warpkriging,python,gl2008,kumaraswamy,nlpd,-1.74552 +py05_warpkriging,python,mixed,Kriging (level as number),rmse,16.7693 +py05_warpkriging,python,mixed,Kriging (level as number),q2,0.838453 +py05_warpkriging,python,mixed,Kriging (level as number),nlpd,3.81063 +py05_warpkriging,python,mixed,"categorical(5,2)",rmse,16.1438 +py05_warpkriging,python,mixed,"categorical(5,2)",q2,0.85028 +py05_warpkriging,python,mixed,"categorical(5,2)",nlpd,3.3206 +py05_warpkriging,python,ordinal,Kriging (rank as number),rmse,0.213845 +py05_warpkriging,python,ordinal,Kriging (rank as number),q2,0.953177 +py05_warpkriging,python,ordinal,Kriging (rank as number),nlpd,0.810969 +py05_warpkriging,python,ordinal,ordinal(6),rmse,0.213043 +py05_warpkriging,python,ordinal,ordinal(6),q2,0.953528 +py05_warpkriging,python,ordinal,ordinal(6),nlpd,0.78888 +py06_high_dimension_mlp,python,otl_circuit,unit cube,fit_s,0.00796986 +py06_high_dimension_mlp,python,otl_circuit,unit cube,rmse,0.0150514 +py06_high_dimension_mlp,python,otl_circuit,unit cube,q2,0.999836 +py06_high_dimension_mlp,python,otl_circuit,unit cube,nlpd,-2.90452 +py06_high_dimension_mlp,python,otl_circuit,native + normalize,rmse,0.0150514 +py06_high_dimension_mlp,python,otl_circuit,native + normalize,q2,0.999836 +py06_high_dimension_mlp,python,otl_circuit,native + normalize,nlpd,-2.90452 +py06_high_dimension_mlp,python,piston,unit cube,fit_s,0.023371 +py06_high_dimension_mlp,python,piston,unit cube,rmse,0.0104258 +py06_high_dimension_mlp,python,piston,unit cube,q2,0.994462 +py06_high_dimension_mlp,python,piston,unit cube,nlpd,-3.49038 +py06_high_dimension_mlp,python,piston,native + normalize,rmse,0.0104258 +py06_high_dimension_mlp,python,piston,native + normalize,q2,0.994462 +py06_high_dimension_mlp,python,piston,native + normalize,nlpd,-3.49038 +py06_high_dimension_mlp,python,borehole,unit cube,fit_s,0.0214174 +py06_high_dimension_mlp,python,borehole,unit cube,rmse,0.893341 +py06_high_dimension_mlp,python,borehole,unit cube,q2,0.999599 +py06_high_dimension_mlp,python,borehole,unit cube,nlpd,1.08433 +py06_high_dimension_mlp,python,borehole,native + normalize,rmse,0.893341 +py06_high_dimension_mlp,python,borehole,native + normalize,q2,0.999599 +py06_high_dimension_mlp,python,borehole,native + normalize,nlpd,1.08433 +py06_high_dimension_mlp,python,robot_arm,unit cube,fit_s,0.187081 +py06_high_dimension_mlp,python,robot_arm,unit cube,rmse,0.358507 +py06_high_dimension_mlp,python,robot_arm,unit cube,q2,0.571154 +py06_high_dimension_mlp,python,robot_arm,unit cube,nlpd,0.355106 +py06_high_dimension_mlp,python,robot_arm,native + normalize,rmse,0.358507 +py06_high_dimension_mlp,python,robot_arm,native + normalize,q2,0.571154 +py06_high_dimension_mlp,python,robot_arm,native + normalize,nlpd,0.355106 +py06_high_dimension_mlp,python,dette_pepelyshev,unit cube,fit_s,0.0181198 +py06_high_dimension_mlp,python,dette_pepelyshev,unit cube,rmse,0.424908 +py06_high_dimension_mlp,python,dette_pepelyshev,unit cube,q2,0.997757 +py06_high_dimension_mlp,python,dette_pepelyshev,unit cube,nlpd,0.510276 +py06_high_dimension_mlp,python,dette_pepelyshev,native + normalize,rmse,0.424908 +py06_high_dimension_mlp,python,dette_pepelyshev,native + normalize,q2,0.997757 +py06_high_dimension_mlp,python,dette_pepelyshev,native + normalize,nlpd,0.510276 +py06_high_dimension_mlp,python,wing_weight,unit cube,fit_s,0.0303686 +py06_high_dimension_mlp,python,wing_weight,unit cube,rmse,0.809743 +py06_high_dimension_mlp,python,wing_weight,unit cube,q2,0.999716 +py06_high_dimension_mlp,python,wing_weight,unit cube,nlpd,1.07161 +py06_high_dimension_mlp,python,wing_weight,native + normalize,rmse,0.809743 +py06_high_dimension_mlp,python,wing_weight,native + normalize,q2,0.999716 +py06_high_dimension_mlp,python,wing_weight,native + normalize,nlpd,1.07161 +py06_high_dimension_mlp,python,welch,unit cube,fit_s,0.500499 +py06_high_dimension_mlp,python,welch,unit cube,rmse,0.167729 +py06_high_dimension_mlp,python,welch,unit cube,q2,0.993452 +py06_high_dimension_mlp,python,welch,unit cube,nlpd,-0.429138 +py06_high_dimension_mlp,python,welch,native + normalize,rmse,0.167692 +py06_high_dimension_mlp,python,welch,native + normalize,q2,0.993454 +py06_high_dimension_mlp,python,welch,native + normalize,nlpd,-0.429291 +py06_high_dimension_mlp,python,projected_branin_10d,Kriging,rmse,4.42284 +py06_high_dimension_mlp,python,projected_branin_10d,Kriging,q2,0.959122 +py06_high_dimension_mlp,python,projected_branin_10d,Kriging,nlpd,2.89997 +py06_high_dimension_mlp,python,projected_branin_10d,MLPKriging tanh BFGS,fit_s,0.171914 +py06_high_dimension_mlp,python,projected_branin_10d,MLPKriging tanh BFGS,rmse,59.5068 +py06_high_dimension_mlp,python,projected_branin_10d,MLPKriging tanh BFGS,q2,-6.39973 +py06_high_dimension_mlp,python,projected_branin_10d,MLPKriging tanh BFGS,nlpd,100.745 +py06_high_dimension_mlp,python,projected_branin_10d,MLPKriging tanh BFGS10,fit_s,0.53444 +py06_high_dimension_mlp,python,projected_branin_10d,MLPKriging tanh BFGS10,rmse,4.41161 +py06_high_dimension_mlp,python,projected_branin_10d,MLPKriging tanh BFGS10,q2,0.95933 +py06_high_dimension_mlp,python,projected_branin_10d,MLPKriging tanh BFGS10,nlpd,28.4236 +py06_high_dimension_mlp,python,projected_branin_10d,MLPKriging selu BFGS,fit_s,0.145419 +py06_high_dimension_mlp,python,projected_branin_10d,MLPKriging selu BFGS,rmse,58.9844 +py06_high_dimension_mlp,python,projected_branin_10d,MLPKriging selu BFGS,q2,-6.27038 +py06_high_dimension_mlp,python,projected_branin_10d,MLPKriging selu BFGS,nlpd,46.9347 +py06_high_dimension_mlp,python,projected_branin_10d,MLPKriging selu BFGS10,fit_s,0.518803 +py06_high_dimension_mlp,python,projected_branin_10d,MLPKriging selu BFGS10,rmse,3.79093 +py06_high_dimension_mlp,python,projected_branin_10d,MLPKriging selu BFGS10,q2,0.969969 +py06_high_dimension_mlp,python,projected_branin_10d,MLPKriging selu BFGS10,nlpd,12.6814 +py07_scalability,python,hartmann3/n=1500,exact,fit_s,11.2644 +py07_scalability,python,hartmann3/n=1500,exact,rmse,0.000259233 +py07_scalability,python,hartmann3/n=1500,exact,q2,1 +py07_scalability,python,hartmann3/n=1500,exact,nlpd,-6.92184 +py07_scalability,python,hartmann3/n=1500,nested NK (20 groups),fit_s,0.324342 +py07_scalability,python,hartmann3/n=1500,nested NK (20 groups),rmse,0.0029908 +py07_scalability,python,hartmann3/n=1500,nested NK (20 groups),q2,0.999991 +py07_scalability,python,hartmann3/n=1500,nested NK (20 groups),nlpd,-4.85946 +py07_scalability,python,hartmann3/n=5000,nested NK (20 groups),fit_s,3.25297 +py07_scalability,python,hartmann3/n=5000,nested NK (20 groups),rmse,0.000222655 +py07_scalability,python,hartmann3/n=5000,nested NK (20 groups),q2,1 +py07_scalability,python,hartmann3/n=5000,nested NK (20 groups),nlpd,-7.44453 +py07_scalability,python,hartmann3/n=1500,nested rBCM (20 groups),fit_s,0.273419 +py07_scalability,python,hartmann3/n=1500,nested rBCM (20 groups),rmse,0.00304431 +py07_scalability,python,hartmann3/n=1500,nested rBCM (20 groups),q2,0.99999 +py07_scalability,python,hartmann3/n=1500,nested rBCM (20 groups),nlpd,-4.50201 +py07_scalability,python,hartmann3/n=5000,nested rBCM (20 groups),fit_s,2.67811 +py07_scalability,python,hartmann3/n=5000,nested rBCM (20 groups),rmse,0.000224517 +py07_scalability,python,hartmann3/n=5000,nested rBCM (20 groups),q2,1 +py07_scalability,python,hartmann3/n=5000,nested rBCM (20 groups),nlpd,-7.18182 +py07_scalability,python,hartmann3/n=1500,subset of data (500),fit_s,0.685586 +py07_scalability,python,hartmann3/n=1500,subset of data (500),rmse,0.00269153 +py07_scalability,python,hartmann3/n=1500,subset of data (500),q2,0.999992 +py07_scalability,python,hartmann3/n=1500,subset of data (500),nlpd,-4.68641 +py07_scalability,python,hartmann3/n=5000,subset of data (500),fit_s,0.906366 +py07_scalability,python,hartmann3/n=5000,subset of data (500),rmse,0.00247552 +py07_scalability,python,hartmann3/n=5000,subset of data (500),q2,0.999994 +py07_scalability,python,hartmann3/n=5000,subset of data (500),nlpd,-4.71359 +py07_scalability,python,otl_circuit/n=1500,exact,fit_s,10.9405 +py07_scalability,python,otl_circuit/n=1500,exact,rmse,9.81598e-05 +py07_scalability,python,otl_circuit/n=1500,exact,q2,1 +py07_scalability,python,otl_circuit/n=1500,exact,nlpd,-8.21962 +py07_scalability,python,otl_circuit/n=1500,nested NK (20 groups),fit_s,0.279359 +py07_scalability,python,otl_circuit/n=1500,nested NK (20 groups),rmse,0.00226873 +py07_scalability,python,otl_circuit/n=1500,nested NK (20 groups),q2,0.999996 +py07_scalability,python,otl_circuit/n=1500,nested NK (20 groups),nlpd,-4.90542 +py07_scalability,python,otl_circuit/n=5000,nested NK (20 groups),fit_s,3.08712 +py07_scalability,python,otl_circuit/n=5000,nested NK (20 groups),rmse,0.000261686 +py07_scalability,python,otl_circuit/n=5000,nested NK (20 groups),q2,1 +py07_scalability,python,otl_circuit/n=5000,nested NK (20 groups),nlpd,-7.20623 +py07_scalability,python,otl_circuit/n=1500,nested rBCM (20 groups),fit_s,0.251989 +py07_scalability,python,otl_circuit/n=1500,nested rBCM (20 groups),rmse,0.00291869 +py07_scalability,python,otl_circuit/n=1500,nested rBCM (20 groups),q2,0.999993 +py07_scalability,python,otl_circuit/n=1500,nested rBCM (20 groups),nlpd,-2.55512 +py07_scalability,python,otl_circuit/n=5000,nested rBCM (20 groups),fit_s,3.67768 +py07_scalability,python,otl_circuit/n=5000,nested rBCM (20 groups),rmse,0.000278149 +py07_scalability,python,otl_circuit/n=5000,nested rBCM (20 groups),q2,1 +py07_scalability,python,otl_circuit/n=5000,nested rBCM (20 groups),nlpd,-4.34927 +py07_scalability,python,otl_circuit/n=1500,subset of data (500),fit_s,0.709236 +py07_scalability,python,otl_circuit/n=1500,subset of data (500),rmse,0.0004286 +py07_scalability,python,otl_circuit/n=1500,subset of data (500),q2,1 +py07_scalability,python,otl_circuit/n=1500,subset of data (500),nlpd,-6.72678 +py07_scalability,python,otl_circuit/n=5000,subset of data (500),fit_s,0.905534 +py07_scalability,python,otl_circuit/n=5000,subset of data (500),rmse,0.000446879 +py07_scalability,python,otl_circuit/n=5000,subset of data (500),q2,1 +py07_scalability,python,otl_circuit/n=5000,subset of data (500),nlpd,-6.78844 +py07_scalability,python,branin/nested,NK,rmse,0.0343984 +py07_scalability,python,branin/nested,NK,q2,1 +py07_scalability,python,branin/nested,NK,nlpd,2.00083e+14 +py07_scalability,python,branin/nested,PoE,rmse,0.0354265 +py07_scalability,python,branin/nested,PoE,q2,1 +py07_scalability,python,branin/nested,PoE,nlpd,-1.64533 +py07_scalability,python,branin/nested,gPoE,rmse,0.0354265 +py07_scalability,python,branin/nested,gPoE,q2,1 +py07_scalability,python,branin/nested,gPoE,nlpd,-0.621836 +py07_scalability,python,branin/nested,BCM,rmse,0.0354196 +py07_scalability,python,branin/nested,BCM,q2,1 +py07_scalability,python,branin/nested,BCM,nlpd,-1.64533 +py07_scalability,python,branin/nested,rBCM,rmse,0.035287 +py07_scalability,python,branin/nested,rBCM,q2,1 +py07_scalability,python,branin/nested,rBCM,nlpd,-2.61369 +py10_compare_python_packages,python,branin,pylibkriging,q2,0.999944 +py10_compare_python_packages,python,branin,pylibkriging,nlpd,0.137225 +py10_compare_python_packages,python,branin,pylibkriging,fit_s,0.00151396 +py10_compare_python_packages,python,branin,pylibkriging BFGS10,q2,0.999945 +py10_compare_python_packages,python,branin,pylibkriging BFGS10,nlpd,0.137254 +py10_compare_python_packages,python,branin,pylibkriging BFGS10,fit_s,0.0161211 +py10_compare_python_packages,python,branin,scikit-learn,q2,0.999597 +py10_compare_python_packages,python,branin,scikit-learn,nlpd,0.928508 +py10_compare_python_packages,python,branin,scikit-learn,fit_s,0.147773 +py10_compare_python_packages,python,branin,SMT,q2,0.999944 +py10_compare_python_packages,python,branin,SMT,nlpd,0.137259 +py10_compare_python_packages,python,branin,SMT,fit_s,0.792991 +py10_compare_python_packages,python,branin,OpenTURNS,q2,0.998221 +py10_compare_python_packages,python,branin,OpenTURNS,nlpd,1.55599 +py10_compare_python_packages,python,branin,OpenTURNS,fit_s,0.00512815 +py10_compare_python_packages,python,six_hump_camel,pylibkriging,q2,0.999418 +py10_compare_python_packages,python,six_hump_camel,pylibkriging,nlpd,0.0429346 +py10_compare_python_packages,python,six_hump_camel,pylibkriging,fit_s,0.00358009 +py10_compare_python_packages,python,six_hump_camel,pylibkriging BFGS10,q2,0.999418 +py10_compare_python_packages,python,six_hump_camel,pylibkriging BFGS10,nlpd,0.0429347 +py10_compare_python_packages,python,six_hump_camel,pylibkriging BFGS10,fit_s,0.0500369 +py10_compare_python_packages,python,six_hump_camel,scikit-learn,q2,0.0163311 +py10_compare_python_packages,python,six_hump_camel,scikit-learn,nlpd,4.63763 +py10_compare_python_packages,python,six_hump_camel,scikit-learn,fit_s,0.156482 +py10_compare_python_packages,python,six_hump_camel,SMT,q2,0.999594 +py10_compare_python_packages,python,six_hump_camel,SMT,nlpd,-0.121007 +py10_compare_python_packages,python,six_hump_camel,SMT,fit_s,0.941661 +py10_compare_python_packages,python,six_hump_camel,OpenTURNS,q2,0.956713 +py10_compare_python_packages,python,six_hump_camel,OpenTURNS,nlpd,2.55446 +py10_compare_python_packages,python,six_hump_camel,OpenTURNS,fit_s,0.00930476 +py10_compare_python_packages,python,hartmann3,pylibkriging,q2,0.986148 +py10_compare_python_packages,python,hartmann3,pylibkriging,nlpd,-0.847247 +py10_compare_python_packages,python,hartmann3,pylibkriging,fit_s,0.023098 +py10_compare_python_packages,python,hartmann3,pylibkriging BFGS10,q2,0.986148 +py10_compare_python_packages,python,hartmann3,pylibkriging BFGS10,nlpd,-0.847249 +py10_compare_python_packages,python,hartmann3,pylibkriging BFGS10,fit_s,0.113887 +py10_compare_python_packages,python,hartmann3,scikit-learn,q2,0.983889 +py10_compare_python_packages,python,hartmann3,scikit-learn,nlpd,-0.782697 +py10_compare_python_packages,python,hartmann3,scikit-learn,fit_s,0.208279 +py10_compare_python_packages,python,hartmann3,SMT,q2,0.986148 +py10_compare_python_packages,python,hartmann3,SMT,nlpd,-0.847249 +py10_compare_python_packages,python,hartmann3,SMT,fit_s,1.35887 +py10_compare_python_packages,python,hartmann3,OpenTURNS,q2,0.984712 +py10_compare_python_packages,python,hartmann3,OpenTURNS,nlpd,-0.809851 +py10_compare_python_packages,python,hartmann3,OpenTURNS,fit_s,0.0169773 +py10_compare_python_packages,python,ishigami,pylibkriging,q2,0.259556 +py10_compare_python_packages,python,ishigami,pylibkriging,nlpd,4.38991 +py10_compare_python_packages,python,ishigami,pylibkriging,fit_s,0.0264924 +py10_compare_python_packages,python,ishigami,pylibkriging BFGS10,q2,0.935722 +py10_compare_python_packages,python,ishigami,pylibkriging BFGS10,nlpd,0.857443 +py10_compare_python_packages,python,ishigami,pylibkriging BFGS10,fit_s,0.322963 +py10_compare_python_packages,python,ishigami,scikit-learn,q2,0.831142 +py10_compare_python_packages,python,ishigami,scikit-learn,nlpd,1.5455 +py10_compare_python_packages,python,ishigami,scikit-learn,fit_s,0.512728 +py10_compare_python_packages,python,ishigami,SMT,q2,0.51196 +py10_compare_python_packages,python,ishigami,SMT,nlpd,3.31349 +py10_compare_python_packages,python,ishigami,SMT,fit_s,2.70852 +py10_compare_python_packages,python,ishigami,OpenTURNS,q2,0.887251 +py10_compare_python_packages,python,ishigami,OpenTURNS,nlpd,1.39348 +py10_compare_python_packages,python,ishigami,OpenTURNS,fit_s,0.0510182 +py10_compare_python_packages,python,friedman,pylibkriging,q2,0.999771 +py10_compare_python_packages,python,friedman,pylibkriging,nlpd,-1.29726 +py10_compare_python_packages,python,friedman,pylibkriging,fit_s,0.0165708 +py10_compare_python_packages,python,friedman,pylibkriging BFGS10,q2,0.999771 +py10_compare_python_packages,python,friedman,pylibkriging BFGS10,nlpd,-1.29727 +py10_compare_python_packages,python,friedman,pylibkriging BFGS10,fit_s,0.279774 +py10_compare_python_packages,python,friedman,scikit-learn,q2,0.999754 +py10_compare_python_packages,python,friedman,scikit-learn,nlpd,-1.37702 +py10_compare_python_packages,python,friedman,scikit-learn,fit_s,0.308127 +py10_compare_python_packages,python,friedman,SMT,q2,0.999899 +py10_compare_python_packages,python,friedman,SMT,nlpd,-1.871 +py10_compare_python_packages,python,friedman,SMT,fit_s,4.71948 +py10_compare_python_packages,python,friedman,OpenTURNS,q2,0.990862 +py10_compare_python_packages,python,friedman,OpenTURNS,nlpd,0.469828 +py10_compare_python_packages,python,friedman,OpenTURNS,fit_s,0.0511642 +py10_compare_python_packages,python,otl_circuit,pylibkriging,q2,0.999922 +py10_compare_python_packages,python,otl_circuit,pylibkriging,nlpd,-3.46228 +py10_compare_python_packages,python,otl_circuit,pylibkriging,fit_s,0.0136361 +py10_compare_python_packages,python,otl_circuit,pylibkriging BFGS10,q2,0.999922 +py10_compare_python_packages,python,otl_circuit,pylibkriging BFGS10,nlpd,-3.46227 +py10_compare_python_packages,python,otl_circuit,pylibkriging BFGS10,fit_s,0.168652 +py10_compare_python_packages,python,otl_circuit,scikit-learn,q2,0.999906 +py10_compare_python_packages,python,otl_circuit,scikit-learn,nlpd,-3.43062 +py10_compare_python_packages,python,otl_circuit,scikit-learn,fit_s,0.64982 +py10_compare_python_packages,python,otl_circuit,SMT,q2,0.999954 +py10_compare_python_packages,python,otl_circuit,SMT,nlpd,-3.35016 +py10_compare_python_packages,python,otl_circuit,SMT,fit_s,6.837 +py10_compare_python_packages,python,otl_circuit,OpenTURNS,q2,0.99777 +py10_compare_python_packages,python,otl_circuit,OpenTURNS,nlpd,-1.61723 +py10_compare_python_packages,python,otl_circuit,OpenTURNS,fit_s,0.0662544 +py10_compare_python_packages,python,piston,pylibkriging,q2,0.997498 +py10_compare_python_packages,python,piston,pylibkriging,nlpd,-3.80667 +py10_compare_python_packages,python,piston,pylibkriging,fit_s,0.03055 +py10_compare_python_packages,python,piston,pylibkriging BFGS10,q2,0.997498 +py10_compare_python_packages,python,piston,pylibkriging BFGS10,nlpd,-3.8067 +py10_compare_python_packages,python,piston,pylibkriging BFGS10,fit_s,0.326413 +py10_compare_python_packages,python,piston,scikit-learn,q2,0.998684 +py10_compare_python_packages,python,piston,scikit-learn,nlpd,-4.10497 +py10_compare_python_packages,python,piston,scikit-learn,fit_s,0.856336 +py10_compare_python_packages,python,piston,SMT,q2,0.997511 +py10_compare_python_packages,python,piston,SMT,nlpd,-3.86618 +py10_compare_python_packages,python,piston,SMT,fit_s,12.0067 +py10_compare_python_packages,python,piston,OpenTURNS,q2,0.990631 +py10_compare_python_packages,python,piston,OpenTURNS,nlpd,-2.9906 +py10_compare_python_packages,python,piston,OpenTURNS,fit_s,0.137117 +py10_compare_python_packages,python,borehole,pylibkriging,q2,0.999694 +py10_compare_python_packages,python,borehole,pylibkriging,nlpd,1.00118 +py10_compare_python_packages,python,borehole,pylibkriging,fit_s,0.0166521 +py10_compare_python_packages,python,borehole,pylibkriging BFGS10,q2,0.999694 +py10_compare_python_packages,python,borehole,pylibkriging BFGS10,nlpd,1.00121 +py10_compare_python_packages,python,borehole,pylibkriging BFGS10,fit_s,0.200685 +py10_compare_python_packages,python,borehole,scikit-learn,q2,0.999929 +py10_compare_python_packages,python,borehole,scikit-learn,nlpd,0.194455 +py10_compare_python_packages,python,borehole,scikit-learn,fit_s,1.18352 +py10_compare_python_packages,python,borehole,SMT,q2,0.999925 +py10_compare_python_packages,python,borehole,SMT,nlpd,0.279122 +py10_compare_python_packages,python,borehole,SMT,fit_s,12.3314 +py10_compare_python_packages,python,borehole,OpenTURNS,q2,0.996206 +py10_compare_python_packages,python,borehole,OpenTURNS,nlpd,2.55431 +py10_compare_python_packages,python,borehole,OpenTURNS,fit_s,0.121907 +py10_compare_python_packages,python,wing_weight,pylibkriging,q2,0.99972 +py10_compare_python_packages,python,wing_weight,pylibkriging,nlpd,1.11517 +py10_compare_python_packages,python,wing_weight,pylibkriging,fit_s,0.0297599 +py10_compare_python_packages,python,wing_weight,pylibkriging BFGS10,q2,0.99972 +py10_compare_python_packages,python,wing_weight,pylibkriging BFGS10,nlpd,1.11512 +py10_compare_python_packages,python,wing_weight,pylibkriging BFGS10,fit_s,0.335854 +py10_compare_python_packages,python,wing_weight,scikit-learn,q2,0.998048 +py10_compare_python_packages,python,wing_weight,scikit-learn,nlpd,2.57304 +py10_compare_python_packages,python,wing_weight,scikit-learn,fit_s,0.833252 +py10_compare_python_packages,python,wing_weight,SMT,q2,0.999898 +py10_compare_python_packages,python,wing_weight,SMT,nlpd,0.604023 +py10_compare_python_packages,python,wing_weight,SMT,fit_s,26.7889 +py10_compare_python_packages,python,wing_weight,OpenTURNS,q2,0.993145 +py10_compare_python_packages,python,wing_weight,OpenTURNS,nlpd,2.83627 +py10_compare_python_packages,python,wing_weight,OpenTURNS,fit_s,0.253929 +py10_compare_python_packages,python,hartmann6,pylibkriging,q2,0.422939 +py10_compare_python_packages,python,hartmann6,pylibkriging,nlpd,0.0965158 +py10_compare_python_packages,python,hartmann6,pylibkriging,fit_s,0.068651 +py10_compare_python_packages,python,hartmann6,pylibkriging BFGS10,q2,0.72352 +py10_compare_python_packages,python,hartmann6,pylibkriging BFGS10,nlpd,-0.156898 +py10_compare_python_packages,python,hartmann6,pylibkriging BFGS10,fit_s,0.877097 +py10_compare_python_packages,python,hartmann6,scikit-learn,q2,0.69049 +py10_compare_python_packages,python,hartmann6,scikit-learn,nlpd,-0.137845 +py10_compare_python_packages,python,hartmann6,scikit-learn,fit_s,1.43153 +py10_compare_python_packages,python,hartmann6,SMT,q2,0.708463 +py10_compare_python_packages,python,hartmann6,SMT,nlpd,-0.0177081 +py10_compare_python_packages,python,hartmann6,SMT,fit_s,11.2226 +py10_compare_python_packages,python,hartmann6,OpenTURNS,q2,0.696177 +py10_compare_python_packages,python,hartmann6,OpenTURNS,nlpd,-0.132838 +py10_compare_python_packages,python,hartmann6,OpenTURNS,fit_s,0.141383 diff --git a/data/borehole_X.csv b/data/borehole_X.csv new file mode 100644 index 0000000..fcc35f3 --- /dev/null +++ b/data/borehole_X.csv @@ -0,0 +1,80 @@ +0.017315140363590921,0.34810328286366077,0.52113184961397885,0.61230499400028382,0.85197029620993292,0.98524761505608716,0.50795743396967485,0.74508070757782807 +0.72375790748518831,0.13498865872561727,0.067640508555753376,0.90478990607720289,0.3509872369953323,0.22198429007544349,0.49612143134603137,0.94418601000738978 +0.88780472252964326,0.66206822291695433,0.5647102873597587,0.86083142574266047,0.26421480623741617,0.38224624798877649,0.15931847750843336,0.3140630269984222 +0.10817302203271725,0.41037975937866189,0.11282305953725118,0.14664543342531278,0.67164921413040468,0.41161397769712299,0.19972692960858918,0.29669743200074417 +0.16032788691691899,0.22335243395614746,0.39032791335376826,0.36953254195018853,0.10431792864846676,0.31851947828835819,0.6186289979763232,0.7663854127961589 +0.41124298711267582,0.047690161516542137,0.98383549478148047,0.45169279988580929,0.69535426273966849,0.41457628711338812,0.57391279424403707,0.70894843334845015 +0.2348355331783254,0.24616853589685378,0.54129851270646989,0.96893654709667698,0.43424104934328522,0.041233051929850946,0.076488887176965031,0.79552733368392126 +0.5930523805714718,0.57965241741162798,0.96889240235602014,0.8821375514324703,0.80117533160238064,0.30542836186217709,0.21561038053279469,0.37216837961072829 +0.31189734526692592,0.38043609100891229,0.83837009737996593,0.63001932302394958,0.21461161412026622,0.20883319928083077,0.18682179570624344,0.98576554844266528 +0.49597377766898781,0.28241251611859897,0.70234200643336475,0.38733154432716405,0.92974270697264316,0.5567880812381546,0.96763656234055195,0.24658639537774177 +0.064975512467892946,0.48275871362259692,0.055039720416198513,0.56792723611643703,0.38322509252848502,0.50408538717721307,0.38445424851046173,0.076641340705933481 +0.91620087931188743,0.0094235343778137255,0.14712474180255045,0.52335466699141364,0.63706891969883439,0.58503803554473699,0.17420422454429518,0.60900159112080732 +0.37780180281451586,0.41500714451935627,0.2956767785683751,0.010910645708621257,0.42447131117220033,0.89875280178053085,0.90620775664595743,0.91787368088969035 +0.092334379815928985,0.031181689554096604,0.48725476235481846,0.78854082811545201,0.49691468521311233,0.47131819474334102,0.95567913089135492,0.42718203070284738 +0.13985358558640684,0.45957708183050394,0.85194312060665178,0.20482400873236234,0.89951930124143398,0.58868656777521289,0.36690379250660271,0.63865908765049972 +0.6858045118220214,0.76265832003875977,0.0899991480318518,0.47659943901433577,0.70988725923463103,0.26310790187534899,0.84296491175789634,0.30070071479132809 +0.80466527314897041,0.1951074522677953,0.68642867761080528,0.64966491923805247,0.44612116635057381,0.49925285099216177,0.2358147642654071,0.89024538977743206 +0.54295954513342892,0.25116845898761292,0.11186463525065531,0.87138250204055612,0.76671595277135784,0.8753450857205326,0.048958669702908894,0.46482920908820269 +0.7881480927158121,0.076579813552882625,0.42468046958732242,0.94038558412428819,0.13992375614022695,0.61670065808888042,0.39676464171473147,0.23651207103628061 +0.026215372709436423,0.87185424944376566,0.89158427545650398,0.32546888645260003,0.31255247376418166,0.75610977614511665,0.79590092080290953,0.40995055185222345 +0.72638363963149766,0.8929737116931602,0.57651794726171701,0.50203764035800913,0.038656413649791686,0.86072210230899926,0.53314034276965483,0.9651465325132399 +0.11603230918849794,0.11755880800456098,0.86368156369948823,0.65274765019812364,0.17283376587182298,0.021122067881225292,0.86881086322049494,0.14247197419743751 +0.22485950108088937,0.80637747750968336,0.62083764391833396,0.89593312931146607,0.91604705493664207,0.69850254707470527,0.60207451438622928,0.1296947947077933 +0.81303046766391096,0.71656624174953165,0.13418695218651699,0.049545847740734485,0.24288264686734617,0.54645121744317993,0.62667342280984828,0.032819313593866556 +0.3278619219259033,0.15900602814099674,0.95925578073167694,0.49988995386364959,0.79392555769042761,0.64743331911239399,0.52255467307210957,0.012066486277763754 +0.47772089756032854,0.72740135148113239,0.42947537827416749,0.13019059098004981,0.9859632941081854,0.44656445512819209,0.14749111986341717,0.80860263198982896 +0.75943494661677169,0.81485062048505463,0.2573773040165776,0.19784410455598023,0.64094238916356683,0.036862876412423209,0.1052271152542805,0.20371386052095955 +0.64395842556086758,0.70536233434179407,0.69066446825420935,0.71903213249890796,0.72332350875277185,0.63658193696816734,0.76687473114040405,0.8789793243501961 +0.36563386396908093,0.51945250087204764,0.80558363480593731,0.080341071150435692,0.40327041581682488,0.60385042802350031,0.47823817640469057,0.49761196394919488 +0.50846454180598866,0.82582591792621618,0.15440968762329768,0.95321639506435329,0.0086356079417305911,0.009051989149911923,0.33556299541805351,0.59766676852197542 +0.27577820392913982,0.74092827918847493,0.78573951924637331,0.83323173255705019,0.23528104236557151,0.84370665490954555,0.98673607024758403,0.32750151991771226 +0.62050441385388311,0.55940738267715662,0.90711742017660657,0.75640943923684645,0.073299130552726077,0.77171727707623217,0.068331598549948816,0.11842100352269769 +0.13498454521277389,0.85858161746734607,0.014060634647304471,0.67883441218652729,0.53793845354240322,0.6785912507081967,0.12507561256257951,0.75931207998105088 +0.63464561787805329,0.088382639034930266,0.66087076181679927,0.39475858189804047,0.61299228478454793,0.82027815515891267,0.26792835717342139,0.35663318168516311 +0.92729463827795799,0.46861879080169783,0.50231816391238149,0.59456893851947634,0.90484224936121005,0.096582255511394072,0.70588273836123228,0.16589823488845926 +0.87949401115075942,0.17157193744680899,0.83261904303031908,0.10230655884878179,0.94068874404422309,0.86892768697160283,0.023431550876115419,0.52416022560128817 +0.58320835134121962,0.6338049337976942,0.08375310086519043,0.23591822219502728,0.83919118933918413,0.83238771582210069,0.55279904876283614,0.92875984526388522 +0.20131157874126826,0.44642102109321735,0.60961612932474352,0.18294361931978523,0.39222109107018371,0.11237032114642301,0.42678663838891079,0.41450849178136534 +0.18468040177841011,0.17520897738997676,0.22390029914387891,0.44891877991182938,0.028672284314457952,0.80033273768136259,0.738450447424564,0.55644465546459032 +0.66067494686822159,0.92005477777928424,0.2419326686243132,0.01724710504477877,0.1529124510656836,0.70325090574149685,0.23982677220927875,0.65386883345667957 +0.31388822452460635,0.69403222442407575,0.63280820194251197,0.46731237165918965,0.50563636073246487,0.94729514988833086,0.29652448250322483,0.54537531822691554 +0.74582679723704515,0.49990290096737422,0.38509883720409238,0.82313394829738606,0.58802724448933874,0.13444784816128244,0.25384698396637473,0.71318367547419137 +0.26481875459343623,0.77541884328173405,0.94039104371044502,0.062841618822904233,0.74351644077754386,0.075457678220115815,0.44901593082797503,0.84885555054125184 +0.16846951495039789,0.92824479955006223,0.53133508400486618,0.34745004810099955,0.83220961525106019,0.14117890173918987,0.30414091608468674,0.043557728860334634 +0.77602830418258617,0.64592031355654966,0.66634082728983912,0.054457379508347337,0.87887742395196589,0.39973726951084393,0.88347296370321682,0.66562810858835841 +0.52063810736204896,0.94206967200098146,0.41001222512191554,0.032218875247831361,0.11458179055740544,0.35184956724957817,0.8151088709875639,0.38270592369415313 +0.0045380710433107953,0.61195057623249483,0.33795902568067604,0.67341481182705398,0.60774303767115501,0.16293837239649403,0.75371230183734661,0.95587371840404756 +0.95644272932528707,0.54753670500977958,0.79890596038378026,0.21701210456309217,0.08646765565076317,0.11334286863085356,0.68878563107146973,0.86569033271400841 +0.55619648651555409,0.7948782381266023,0.5598545269355012,0.43477889458000885,0.3490660480604536,0.074576264748874674,0.94790720897398639,0.7300812760009856 +0.38841485503391304,0.50038566469099599,0.18105037555768583,0.92632918767552552,0.28453585762600336,0.57477815934985743,0.86201687658158366,0.77622667043885241 +0.83282210649599886,0.88487912186084616,0.88101335836788564,0.80016249588407717,0.58618267900314536,0.23840929271605141,0.92339675445277669,0.50747438936161104 +0.45371483901965892,0.61753823730815327,0.001853261341052724,0.68963481626767431,0.31091510807902584,0.95626063453702359,0.45745691417381129,0.1888154531494744 +0.69732154186961748,0.30596093156156329,0.19610835659379805,0.74488840700022352,0.81552643124670321,0.050509931847235158,0.59415988982718315,0.39404465773931796 +0.86504477504702815,0.31967445649363591,0.35189134000884337,0.41927213994536716,0.25328134014292925,0.73568032987530141,0.089530365927370084,0.90364973608026433 +0.71098949187598781,0.99234226976054762,0.98876954364903613,0.40330074133916993,0.37435082969208627,0.48239562436614547,0.27708530572410583,0.18474849454672554 +0.19323228581497121,0.59560771310004279,0.16291633462177307,0.31399965044529055,0.13321123781651611,0.65110745719228036,0.34541018399546219,0.8536217595094433 +0.23882605698920706,0.67670212194371637,0.28476849235774371,0.53875663049398337,0.017929657190391458,0.29510052631366468,0.54740574674064357,0.27900357923811547 +0.43904225105924377,0.057180027599366148,0.27163468132161472,0.77591834820868266,0.99555295828300872,0.28141075127003595,0.41465560875959495,0.62081057369769588 +0.97531523639279794,0.95454495712988618,0.3305044981603748,0.6150066737903146,0.96695582788795331,0.79354595060394384,0.36040296094317814,0.44728782667161032 +0.25441966956833478,0.14140642687205213,0.045926112789273318,0.25095707268289386,0.8670780859699283,0.19106049970444336,0.89532462933040546,0.68072408402576301 +0.46631778660104961,0.97638570139169933,0.71496220507544828,0.57931438632094789,0.48469317396344891,0.16001027256734293,0.58514372673257808,0.5682813644371193 +0.91162067168716354,0.3509191879629337,0.93084590729663541,0.92078934042789684,0.090805921469379144,0.93382135644258724,0.73438556160002877,0.69846334480611616 +0.42590901074547605,0.23720159638589955,0.74757935772220407,0.98118550411310168,0.6579885169049583,0.91247555553144566,0.65837026365371165,0.066902447722907377 +0.044128637266163659,0.074949978125162145,0.45069302975652048,0.093716451839574347,0.33573598479414246,0.99818255747730211,0.20488744177506701,0.26240336550806653 +0.41580326689250374,0.29732095998452179,0.49864085819652881,0.24466284094005206,0.052308925056974306,0.26170851552317453,0.027700461594000393,0.060442039168098392 +0.29908806426452916,0.90848529905979658,0.2018848434199183,0.29908603837873604,0.56421015235957439,0.91736580604844709,0.67365636074389923,0.15731983221197451 +0.05899914048784085,0.56705250376650984,0.31491022516915179,0.98886618608531696,0.95992070110768091,0.34130318072116334,0.71934236267286267,0.3423245049070015 +0.94155210827523406,0.36562948913139254,0.23643824516861689,0.30557709947224782,0.53469893618055253,0.33396511749155006,0.40672414665314055,0.022532295025624299 +0.083257713977681516,0.84701065133553877,0.75374146813643583,0.73552705345302427,0.20664976191331741,0.43303407228684365,0.059358203298531212,0.63622424817954504 +0.99525685196011648,0.53284777741489753,0.46278624161528342,0.35317769946885236,0.68018265966240199,0.96658141307094014,0.93609288509798461,0.57708544648204529 +0.60660881928749399,0.42972260678841046,0.3655720380853682,0.77326996808719495,0.4722747144990973,0.7794380115964038,0.4749421294583952,0.10449513722627671 +0.76761664248844796,0.20842407761856813,0.31056080320134016,0.15909853943869595,0.51685726790285613,0.71946593041133955,0.9939247594587719,0.21643876892071018 +0.56710713221160136,0.015086924409817019,0.59044109520709531,0.2627472696579799,0.17857612797249653,0.51954551936013471,0.78184574210436675,0.451355080234904 +0.8527816708242193,0.75796144687116629,0.72812332327731411,0.56097282115132496,0.7546211103076681,0.5302981121664162,0.0059822558556910568,0.099258443230797827 +0.84752034599427239,0.26444024541141686,0.7724758774257281,0.12273468676077211,0.29197309464805304,0.18130284265739935,0.63782547891390107,0.27317006976029706 +0.97340113248751936,0.39800541377902882,0.027322578844861889,0.70513112554413471,0.1918516344111553,0.45474982123565261,0.80104365015277179,0.53669086181947623 +0.33789989878711146,0.97270467803845673,0.43909273023867018,0.84797926500094412,0.77914905027844805,0.73788429819241708,0.83359203663052173,0.81483312130984342 +0.67002923930507807,0.10395769566387203,0.6474684007931667,0.53548067137263344,0.73741832135461705,0.3637246718672334,0.31377215832110567,0.83523970232878919 +0.35574782078199441,0.66582574527594174,0.8187470253847634,0.28099663614429538,0.45956472112818325,0.22693565845789943,0.11859195903219569,0.48527474614432087 +0.53220232167295933,0.32641497323648222,0.91426032058937656,0.16759690648940931,0.5521247637673421,0.66477095220702442,0.68705939369236413,0.99512523525613883 diff --git a/data/borehole_Xt.csv b/data/borehole_Xt.csv new file mode 100644 index 0000000..63978c4 --- /dev/null +++ b/data/borehole_Xt.csv @@ -0,0 +1,200 @@ +0.44217227295272243,0.78901759462748711,0.92883517101843105,0.17042606197312338,0.85842780900467719,0.78008118189651876,0.61572682952990154,0.5515917799926181 +0.090008414040849916,0.066436011213980128,0.36395440971819548,0.56382552625339921,0.59603640790456369,0.46942500529135189,0.87341965920427,0.27891743261638047 +0.91470114458629015,0.12875580368063,0.72330765612661896,0.94299476206174915,0.0071488025233706898,0.0070146754608084727,0.65611115848812707,0.55445433290798118 +0.94651428377004299,0.48982475344992249,0.72627075322195833,0.63531945212934016,0.84154166038552913,0.87694418203435021,0.58085516499613843,0.81587202329207398 +0.69891279386383909,0.75791265396917706,0.18119157938347696,0.60311037652189481,0.14234958597001457,0.60032353632313884,0.7626038059122805,0.086820184077650908 +0.11186104036238775,0.74418916658213197,0.32062256074155193,0.19989839643733331,0.75007045378192894,0.19795292432136735,0.83072372731131638,0.82198204613595083 +0.053916963829623965,0.52273796131998462,0.06484583577276648,0.72666405218749575,0.0058732290974637946,0.94800955054377922,0.43537510251961531,0.71952125426558322 +0.20064650389128846,0.99076045724539419,0.084426903612385806,0.93958462242323071,0.61830845161439763,0.095757230018457151,0.14303257531275559,0.50325844688795984 +0.24117907360732416,0.82757768573576806,0.15501613891033339,0.45614272588285909,0.06757893150272487,0.19169136032447587,0.74394765454437939,0.42198575465404942 +0.79566433908098122,0.022432735582047836,0.45526027961659254,0.22670543264604381,0.041969134457666701,0.76797985085979048,0.75152717450358009,0.70027393045195352 +0.96431894999931078,0.71933691302467628,0.0083535630910052117,0.99577654434371621,0.49107197448063877,0.61224636436388424,0.3034401566374697,0.3576014547910763 +0.61338323716802057,0.7400432125811105,0.49722147209535672,0.5401526792567648,0.42882614983424183,0.79263106477152101,0.81662464835723969,0.99006174398007007 +0.051079393166953757,0.10305214156860165,0.088007351038719706,0.97598737516737288,0.20669571756588578,0.88706819131737635,0.78471862813803983,0.89799767861774571 +0.018750356594929407,0.0207711338308052,0.88592018667820327,0.929288750591311,0.91997170735839207,0.028439447273798768,0.19307172125323935,0.76951207043301029 +0.36299400492802036,0.026775567824404156,0.37115676600116632,0.14199663417590525,0.026832884623721798,0.9247216401204863,0.32301841517045682,0.33125143832922455 +0.77341326510301378,0.16933829864247008,0.055361531615479187,0.94052206655136317,0.79823699901359413,0.1706929874189751,0.058114782212122629,0.25603132525880468 +0.83719249918581173,0.87951861569513778,0.55581040738549514,0.16028806653964967,0.13601257793440014,0.97113546286683772,0.038006819017857851,0.72798416175824521 +0.78764745715918405,0.82311590735054008,0.15091833908836205,0.51382553647050577,0.78473567086699736,0.39662721649426991,0.11213203725839793,0.084201672054377896 +0.76902252303557894,0.67020564886877032,0.16420304940658359,0.34089838593329702,0.95308665451471919,0.42939797005730729,0.37560315982637904,0.46861740632813664 +0.90424178672680777,0.34788392337639784,0.77691560498790158,0.094489469117504776,0.15774909680215321,0.069962651281692656,0.80431459508952852,0.77203192874816617 +0.069217453967550524,0.70778546282554078,0.14042714760127439,0.73941084152373415,0.027547748316445131,0.64153621751536549,0.87848815126631119,0.18961070438471372 +0.63460597662510088,0.55744114580299786,0.31876176905004472,0.59772052057089065,0.1714939792670247,0.830576390685829,0.85561387892704721,0.60780675640160864 +0.83868485053539532,0.84832295939756008,0.58414593733301567,0.63177931856820291,0.71935086663360759,0.22990871470043284,0.89805754888889422,0.26428062486215154 +0.59442668440972568,0.71810648571140645,0.28522064665601476,0.88445798380784091,0.72809476916014881,0.13816718483617652,0.53822096846412115,0.27485705761574097 +0.438360416048446,0.78083629079733796,0.55920798299151964,0.055494012131647397,0.99823093169026944,0.77586784324529623,0.11198378257637132,0.85288615748942231 +0.40175513784851757,0.95757972182259288,0.90994983631532522,0.89179018757098727,0.88044050020888409,0.056091759467511104,0.36913982376475396,0.49712887776359471 +0.903073595780708,0.65300006886845552,0.13684116946487057,0.4955818059240078,0.35770027943443372,0.69926323197080609,0.71928705467098963,0.73161976946167728 +0.628561338397021,0.39341656595279551,0.1216973572747202,0.041119778717247923,0.37262318934523342,0.68044401276901534,0.54751122917157846,0.75576396515555433 +0.99789773101401624,0.14669331613891079,0.38516255091568152,0.46666822201279567,0.81840005805064731,0.9796538701898152,0.29071799279316068,0.1311310141822658 +0.42702936816549431,0.026150618155328154,0.84248706515183547,0.016683831236958824,0.53705704856426772,0.66264776493275201,0.41947305364252541,0.81743342743120073 +0.88046155051935504,0.63867680496949697,0.53664959001243018,0.25602224163988496,0.36451494535291451,0.68500917375131598,0.52413037160987364,0.11190905368843318 +0.76825221449813075,0.85876164123686083,0.35912254707983404,0.28465627959627482,0.26775242430060964,0.21837134941219793,0.89273880089008062,0.46320701864374059 +0.63641694396865356,0.49444250824172298,0.87211383416274912,0.031539296562799746,0.87076508373449524,0.48052446873597876,0.64192779885974338,0.095065021673584771 +0.8231433992053161,0.39175164168258059,0.33087687520954856,0.32143638706046096,0.018411316809590406,0.87319854586237666,0.92155373656305606,0.20581596094541221 +0.55188496454699454,0.48585080808920156,0.41974822079656038,0.57661563134774729,0.5446928112307059,0.24089044474948751,0.34466111301214686,0.3506568223103127 +0.2338606360193981,0.46557562048313106,0.88450809440822453,0.50715507176959351,0.14045160316228333,0.48509668009589624,0.360258546636122,0.96076386513093515 +0.56134634728644706,0.5340975518039186,0.50190811473723018,0.36381673371817891,0.37477072331755812,0.96508344731737683,0.39084217320433012,0.60933856666296793 +0.65122668995859001,0.23352871928950392,0.64063349602723219,0.99864098121415945,0.46209347036148074,0.12687596706969639,0.16318878439176177,0.15547728302456199 +0.23066575492329078,0.89434669536302658,0.49417315039274734,0.4662367306790135,0.65456402562107496,0.71058961869572645,0.1697284684164404,0.62702325115012358 +0.51030799374590718,0.39954206433522288,0.50285327550788073,0.73315356431785661,0.58604851504040334,0.13354311518037121,0.96578812327744989,0.11193849270848499 +0.083077899070327921,0.52442785427259542,0.93376825844193123,0.43383100103336025,0.94009652569179214,0.43501246449891506,0.47880682700956556,0.22037005458370629 +0.90536028488981424,0.31812781713096183,0.66675349078305979,0.3576766775612692,0.85944984250491008,0.82166195954009547,0.65694772385011224,0.55353558851297924 +0.020719990036436653,0.76842448392307683,0.78046209677104172,0.96397398901165343,0.36895580318514731,0.18426348727066455,0.47832392489354769,0.92493425117829353 +0.746115337111384,0.69967584280146111,0.3630711474317343,0.10918933553520471,0.29246668272083998,0.34602499230663553,0.48161928414880717,0.8415320136043527 +0.812147080924843,0.77126044170551766,0.078417122269472839,0.41628642897226109,0.39150900051524684,0.26150089118819875,0.74408763066647698,0.45085208448287306 +0.71285997807277601,0.2753306296098561,0.12898795235915839,0.34480539498689122,0.51982539467826849,0.91287365307545121,0.62585087258273753,0.88634058763789325 +0.45935240399237376,0.13251939589417994,0.086633350583403601,0.65651377323830051,0.62771050020820296,0.6797193086843597,0.083760739416245067,0.052430589482174716 +0.14747024041364887,0.43763308889551578,0.24296988898622252,0.13258663209207755,0.46523994895009613,0.24497744473370631,0.50496531693052349,0.67757190725172611 +0.71404821792359652,0.030626970897620343,0.94777396510898293,0.44567518604116774,0.17306471962612224,0.68298843561453237,0.36841050863965308,0.58994133912522584 +0.45936162343167131,0.53382226460184778,0.65002328895004424,0.10864465814886792,0.01942201492659601,0.76871317861074151,0.57692470809450225,0.48999684806854216 +0.35212090470108326,0.37408894416049887,0.22425360703023356,0.99199739291729983,0.79243589991150853,0.95534664543425973,0.057222604827159684,0.17337978654682307 +0.039427541404710831,0.14796363757148145,0.042743153993095917,0.85033417900154784,0.090283821204099612,0.60211540875303704,0.45826510070619542,0.97349262079913423 +0.048908932107002867,0.94179869200259558,0.22597852949657726,0.55279069088148436,0.85821145832999857,0.91872393589636125,0.67122847744385783,0.50109020180830954 +0.25223028027912497,0.17884147966941855,0.45599282138873942,0.22160909066511836,0.3186815135049762,0.02241249130413725,0.55773921291192075,0.38470098605538539 +0.022533572253770617,0.6050657075846082,0.043487613941568748,0.53934104885917711,0.2048044474554539,0.94910685929557614,0.097665495628938448,0.06057537661403456 +0.71700211525279567,0.65741920395293185,0.65794164196968086,0.35149971818475356,0.57547215780940353,0.087961234899303964,0.7781823974288542,0.74953302192572147 +0.63402143637809749,0.0034196367376099213,0.77070281331927726,0.1672559086248796,0.44526581792866182,0.83706659052125232,0.91078429152817331,0.1012808804706522 +0.73914808091126138,0.40854152210420125,0.63993167458299149,0.22520281850781543,0.47000318227460902,0.59929212221298844,0.98392019982401435,0.041155673701470885 +0.1190314594746964,0.99759038488170293,0.68235615733725707,0.18515863222281304,0.051831662179558036,0.38119375500461883,0.8487201410216858,0.36909893970598429 +0.90339893345591704,0.90182547532260471,0.61715655970918815,0.0097326073486683207,0.94038167174886045,0.86156765053749718,0.71930243613700473,0.87715670858166883 +0.22703695207567265,0.14161948706908112,0.74195050612809177,0.78085154440425664,0.78874025746431087,0.93057904211114939,0.29183740899010713,0.31095399060180351 +0.14695806170176184,0.88492340806357972,0.34925993492283247,0.85339396238931886,0.80163748078099806,0.7905610355581868,0.089924089264773799,0.73181692511633911 +0.22067059670934686,0.48735839477942899,0.098171496800493974,0.90675394017506905,0.069557294218870247,0.29097149092197205,0.72153013438523284,0.066503145201721736 +0.35312318593519987,0.9316807688165416,0.63936217566431408,0.735063372648463,0.94773068861385878,0.75631427041136379,0.52584791059636649,0.010543887407105879 +0.26303186664623135,0.36704837788007072,0.55744390330797899,0.18137362317991101,0.20461573382027953,0.035663941053448722,0.86734566249117118,0.68820239436232844 +0.18894522921359036,0.13793503477272884,0.2992724093786121,0.1033144512351708,0.61661691128291896,0.37718905345000275,0.18403654079553633,0.81884407314419161 +0.038227921629746975,0.77827057523899035,0.95559910361534939,0.9680367654491332,0.34413978459763217,0.65740085362300327,0.66738990604949111,0.2196608562894693 +0.82417232049083256,0.51559227582518863,0.86281306922552536,0.79908207705099499,0.62387497079999055,0.053929081730902717,0.48520601035204247,0.6815889692064212 +0.39069062396181164,0.48752675783748556,0.86363627145154276,0.94258203134459617,0.3438459442937547,0.88575784606477337,0.24673919626437923,0.11552216009078442 +0.81557778206623033,0.85147269494307232,0.29789800564710833,0.92137862449980523,0.43824762226175451,0.72211597104877523,0.61593717771593581,0.096236728196555599 +0.35524254697284363,0.090596855022192613,0.0052066592098156672,0.51197944610236534,0.71671354920978192,0.041369372745930111,0.18728004180511326,0.48797309456244464 +0.31041525280588955,0.38059334668669942,0.32347118376854567,0.4332118004061899,0.35469937052280043,0.59253875183505034,0.23446675682369533,0.36515867001065516 +0.23374602937207489,0.084866827899760189,0.86600386958042808,0.43821434737536791,0.17174863438835963,0.26280015327115558,0.48449978001673233,0.7255442833110054 +0.90821030764840693,0.71799816252840154,0.70948774121983949,0.58554834348194562,0.32636025957107351,0.82728362015938073,0.93560659543072677,0.45552524752096912 +0.050735310033296122,0.89609171757556638,0.70200978162362926,0.9593139683032641,0.08084401957295928,0.2055759450644209,0.032777485077280843,0.36923928401289874 +0.19562966593519204,0.67000987719163774,0.87797959340292653,0.91844558599993142,0.62749639708681149,0.48385771775668107,0.15657583516124118,0.31621307808415744 +0.023117826601279923,0.39018681787170595,0.88469695886663868,0.97410867088107178,0.63328279107388896,0.75383776376417999,0.17181079139539812,0.4859058368775957 +0.56490224579450377,0.67578022275089666,0.44838810619699032,0.77982377675830628,0.24582790037062197,0.34774287389834713,0.041440219818951185,0.87985957254836777 +0.42337189425547295,0.93685380430211251,0.57069314050569941,0.20144107825120194,0.23728723335854529,0.30594455490165928,0.8474504513121327,0.97053055503169405 +0.69613551352744996,0.85514532302406854,0.15947970995904992,0.29775244122404743,0.52164915568217574,0.41316694365458873,0.049324026155127987,0.40708616853581536 +0.19176145776066167,0.84139889183939254,0.96556365296488245,0.73181355634999901,0.69182938530202165,0.33190144621183271,0.31275211703589023,0.76889587169076756 +0.13446659114924453,0.59411596264439948,0.64976245379243125,0.9028897334515259,0.56068058180073299,0.47909671700562229,0.796119231512738,0.54759501298570901 +0.18059589527966124,0.7223896089572891,0.96471557609319381,0.040103130756295013,0.63861796653838787,0.069425191046543766,0.86189910308882034,0.53763450466049834 +0.10721909444783606,0.68671313839982162,0.22695555921241539,0.1511436390326274,0.58218362093428278,0.39867222230833077,0.97834076834871275,0.79419496945355939 +0.91688846467829155,0.10088753757059987,0.061061158050419895,0.67994784976851819,0.94473002488885538,0.48693371523909112,0.45230551948952658,0.86347275022214831 +0.73483144811492507,0.61597462372316369,0.50133213737902849,0.37047674441140865,0.93889451221873688,0.050517763859100784,0.40702045195936709,0.2520128613827759 +0.91048791909451443,0.6210800686899729,0.13293039403613427,0.58018436606904966,0.86709540445991629,0.2386789350796219,0.7611470916245201,0.83316604425063645 +0.72542682513110157,0.15182502861306413,0.11414082457865204,0.22301499358470256,0.97907239586245076,0.026828753687180962,0.68582980356557144,0.8257867815794887 +0.86321998962116198,0.068965574504487814,0.91365671035527862,0.67804381101798528,0.51271187788444672,0.32992202914947044,0.018593411057336695,0.98649494701104423 +0.99575981585553908,0.15449282819993471,0.0075029401545437802,0.0853378009980722,0.26938891150741306,0.89877445878559159,0.89482668519223396,0.50183461933017914 +0.89772161183359811,0.91576585919468545,0.94936120288973735,0.8006132457256766,0.82996224763659288,0.31378355366453536,0.46650679843440934,0.048753038691927708 +0.21783379289612492,0.18581071774583136,0.67946401684430324,0.87387870989271377,0.48242319638247833,0.10105266437421923,0.37374916281514636,0.30309233667316526 +0.56694523412393139,0.64956008799177822,0.58385576007038009,0.22667026416431113,0.035404301590516107,0.44375984715984662,0.96390861450255327,0.28781177105547895 +0.23927446828956567,0.39193415515326679,0.12659765990794758,0.35498077559143448,0.74298058546744905,0.30481885330080538,0.36533004094400434,0.92348815219573122 +0.92063031319191424,0.10319537359310382,0.72953854419588859,0.14119033530321234,0.51169459899501479,0.59631189908614024,0.36379823030362701,0.31954671312304739 +0.62839337995717526,0.85593516656389845,0.082475141169999766,0.72132538758238129,0.32261722513657021,0.84045739404870579,0.86030807052748182,0.65905577724684761 +0.38881656283350885,0.76828342213397205,0.087840165047588181,0.54490615915149576,0.70090603475589408,0.16071779263884556,0.64981221389800747,0.72896270716066769 +0.30010128734379748,0.42030552358465134,0.023834029280044078,0.72770535170143702,0.041065478093853525,0.58829055572290223,0.38977633533641498,0.12394911786575369 +0.75505511340810594,0.2516528178335985,0.81691505980908796,0.69422556543084346,0.33686592923689118,0.34085315989803178,0.10997690436711094,0.014615730536159832 +0.90140026987105737,0.70582528758884555,0.033948926123963385,0.96780092115233163,0.48098427107239827,0.80704925231076086,0.4016138563079743,0.47761334182649695 +0.54478012478765458,0.79279165463013923,0.8119440137195727,0.48390443797968641,0.043205209947738465,0.18923682408040665,0.63136182624828263,0.36063140012597406 +0.84046531669042512,0.50453699496169035,0.37876978662052663,0.89391899988227275,0.30766786855964323,0.69728586123579284,0.12794892735664287,0.86048543559990642 +0.3667486524913498,0.85129424712553947,0.61903184137359601,0.47941939819362334,0.028951289357288856,0.88357945915227987,0.46152872502924891,0.40630351400190245 +0.80438192536214281,0.83929464175771673,0.59201420849470854,0.84892570599595485,0.31762397122187636,0.65119808025248571,0.074918947297574201,0.42430200739887991 +0.68319137780224126,0.026385705695325612,0.78443919907425774,0.84226661745847287,0.7388340968521504,0.19768508223489634,0.70718691928919974,0.03282856076125773 +0.050421917217190582,0.69109925367859393,0.91270131219360884,0.052336807224433346,0.028399105424875115,0.95786572596658759,0.53103684530405582,0.12441641597413466 +0.45054279204975844,0.24387187346127104,0.71992509700707119,0.35674916058041628,0.82961737375837608,0.61393608699210434,0.94579291150500933,0.56279509237920433 +0.36120436250107413,0.94652080281476558,0.03843713656802783,0.75959031707576974,0.87437311903398052,0.7826298948077971,0.93266265213537558,0.6395496467121129 +0.079647500740178123,0.23851633551525819,0.40083526511666412,0.30824960165983017,0.46591343539191576,0.029110584260529704,0.55903086708913119,0.62870504795616367 +0.19987232402341448,0.99050508950152183,0.80934789042512845,0.18889673732461143,0.86923523130183011,0.98282929428628019,0.55337935818177231,0.23830369928024575 +0.055001152598691427,0.6188277861334307,0.4507960458537531,0.88137978253280591,0.15912585948848723,0.77092669570045647,0.7847948366931381,0.48959215032588133 +0.45663956371196401,0.97184250386187487,0.19433505102092086,0.92874576130787923,0.80743892749148494,0.085658229249642059,0.19677816653433489,0.7405131385723368 +0.0041907453513444493,0.093642119157934522,0.17599325273462463,0.44031544778058695,0.83067534197192172,0.033216904361171307,0.11536314079123389,0.5102245080163077 +0.19975112738002865,0.47334358882048322,0.3892065888026135,0.47136128607269001,0.70634787103884056,0.54831799980223583,0.32118601218049914,0.7127945499431414 +0.63041430807405863,0.16569460626807175,0.16467161191532376,0.51788647221253425,0.12321578772179198,0.62635913076423166,0.3473813448187425,0.013672929948132961 +0.96722920280071789,0.58083477150455731,0.23410249450737397,0.14943842268776031,0.87270580540630671,0.13351920794120509,0.13839938104898442,0.94945686047652811 +0.35781798311027213,0.29042048844606638,0.25659301491676823,0.51113766718523024,0.069944319042078318,0.72956998055276978,0.24046945366911276,0.47926709176862203 +0.86150930024220562,0.93776137063029896,0.79961375311870964,0.57352066347839303,0.54687475122806206,0.79701385928830137,0.24049999507281461,0.8923586512130115 +0.88595316541027214,0.7011924428707581,0.18086625815434398,0.22427904669059784,0.2172542482382156,0.25587863317403803,0.49572774951863241,0.70608566234036385 +0.32747976366673992,0.74569487886630648,0.2525031568953543,0.051859747826207281,0.74998658677599617,0.12725236482497748,0.74432618566669639,0.94554634537489302 +0.53521700907806102,0.4292642983712478,0.8362033661423105,0.90786359259667515,0.36569056760258634,0.69810575079842341,0.46857246829945842,0.77541842292209096 +0.41839559157955608,0.91399241412279653,0.44672437217750061,0.929993753115961,0.1827743058783794,0.75168291245225727,0.99092399795516783,0.6547922256280948 +0.62593000924033271,0.97172193190022982,0.49426505564929013,0.074673729150325663,0.026643871818220144,0.28555075490696369,0.93431449589252202,0.36267420243642123 +0.043263642931501534,0.49482710905529148,0.68145334890461029,0.86734524868509466,0.48991150036119724,0.15036155373951821,0.070699988013595894,0.89092188654327276 +0.41068427588057155,0.12904950055791686,0.69480302829489937,0.63811020194559054,0.91645550073587567,0.80519115086716908,0.20448242260775351,0.65580934314829409 +0.31610984258610797,0.71882020623546794,0.49933191141932265,0.17370058475240502,0.43664258477117079,0.13213499611154855,0.57241200696193018,0.64659183916080187 +0.40434200997818226,0.13706664014649761,0.94688717379212495,0.3456492750002581,0.4097899759128949,0.6800439294463565,0.39162593481651997,0.53413350489466993 +0.16586847401344718,0.75004337084359929,0.16880412045512316,0.55111807718709793,0.04124219864380263,0.097387769016449743,0.53597843997870231,0.96906732612953117 +0.033759297611836825,0.080543627269731632,0.30751999374680328,0.90671689149995993,0.71676463320806938,0.45668113516834119,0.86045634448611841,0.67817457949070359 +0.76830519894521954,0.94151978121710789,0.90158141824004079,0.4350335622514202,0.15162898604538966,0.57115468157057248,0.49682434529215125,0.47504690685260709 +0.40643948497113924,0.58488253805387713,0.52598456595070342,0.083234337347750431,0.25972916074751795,0.32479992053604689,0.87976216641538307,0.18281629421166912 +0.60392680893505768,0.91502705153329733,0.43761627976593176,0.80402706064220264,0.6273339145980138,0.91964097832028024,0.0069584446486300466,0.35858935042458795 +0.99776579023296974,0.7323106124664065,0.99444462067086303,0.55064076720201616,0.71939543855946153,0.62808702365087288,0.90713340981631985,0.42434953499263839 +0.93352118483345892,0.017882640552630558,0.11284835462198939,0.90593461379764106,0.97130593278351851,0.88437487478432852,0.29144714740062461,0.51169737233458612 +0.90264150197047111,0.084603800550850261,0.047578061348511169,0.39185415987882333,0.56579641705722716,0.9703654809895027,0.19162801281176312,0.90942963948342381 +0.78936801934838474,0.96366512280364558,0.79546593890566186,0.51733728330674489,0.53131372842752622,0.058047284954782619,0.5510772172123295,0.65550318812135622 +0.38847954488973191,0.20952386730577777,0.80298268666000816,0.56386534327119386,0.17702671758543154,0.52157483705587826,0.63490662224693062,0.67355732771199317 +0.86694370389754416,0.82664105523217957,0.61290885664942873,0.70453214097933803,0.057231918716460384,0.6356421796118229,0.25177116384795695,0.026212725189714181 +0.016332821808610842,0.67417874241694964,0.29587373750678081,0.60391582960635115,0.31060116437129026,0.5148496315152985,0.39519336740118216,0.9437901148496084 +0.068512495585680178,0.10202691734057756,0.87438986974275668,0.4044480483208176,0.3685948147671122,0.49743580637592844,0.56648106716672342,0.0663101445444787 +0.70621096223442148,0.78807750016456113,0.24614380757596077,0.19484757623423465,0.7407319611530323,0.23318265165834229,0.96815638318953501,0.89814593040818691 +0.17026309949355967,0.72357573313558776,0.91385990336862355,0.88894099766731061,0.92411026260362639,0.78121779355801568,0.11438992060412345,0.39207367439807572 +0.33613695489064233,0.14399554259385206,0.40273485677983234,0.52376121799828113,0.34245397428486735,0.1150028789727513,0.45570771502642549,0.19598429104075388 +0.61569976967822571,0.89769875441783908,0.51837735923761041,0.48726460537185312,0.31807454995711926,0.029483566217309853,0.2781932543050919,0.38524745626844503 +0.75111062013684271,0.098987349494230603,0.64538644785786936,0.59707018330581374,0.90519726845275605,0.55281969873704251,0.1355291667418157,0.54622204312372336 +0.34644435831582443,0.011798206828782409,0.033624094666158455,0.13111743614643834,0.93603263346549592,0.25383384436455003,0.089321005177919965,0.36246965586368962 +0.17282340990041845,0.29667443322592668,0.30109520068203777,0.53805030661366138,0.43837640338587136,0.37811696104088988,0.2228190908919494,0.2395078365420924 +0.4877752618543284,0.10133006020089352,0.80213383266877658,0.89330135769306229,0.516591662836415,0.2665953691470716,0.34799564525466264,0.083593657450046988 +0.21986175388036688,0.91735603319940828,0.32311664810286811,0.78571978455261315,0.69882957820690517,0.38134381502748993,0.8731953076335266,0.70847543251200062 +0.36706615561146438,0.15047533613460107,0.58838048858907388,0.72142234610318901,0.83310000614149171,0.99348567886576322,0.88626076722980962,0.58600384594427168 +0.071064618405297653,0.45022896624802711,0.61609271018660083,0.57020580461244486,0.40867053409891707,0.084786105750508822,0.1261390549865341,0.77315390085067481 +0.048642258402415095,0.51357533654142173,0.7134156110195744,0.85618922371762696,0.28669346411736907,0.69066218911038757,0.78540007097325581,0.54095364696099824 +0.97693470407831917,0.11047936343123899,0.0039574991424570216,0.20906389440804529,0.419520335001628,0.98306568147979256,0.29447929497269831,0.18004496981367957 +0.92308388556053156,0.37318280351831146,0.37133070921852529,0.32771215056177772,0.048543406758702945,0.082178926209890313,0.022553256124886234,0.97804368579032264 +0.15707730865507974,0.35264200827806458,0.5494017387618283,0.27715751614132822,0.99968970496257303,0.51613319470867447,0.46384160559205645,0.90071638906040719 +0.64019945916917109,0.083880918490503609,0.014673582286673703,0.93527608055813904,0.50334605718037295,0.58065118099725477,0.90010773799194821,0.40677096486187847 +0.14980690865529567,0.21879624052434443,0.3383333607546537,0.63703710406742797,0.51944877591564798,0.30541143964406736,0.068279547082978453,0.13504003659347297 +0.46543616001525123,0.24360397691905999,0.4569859247211312,0.48745270216499992,0.11719329537449574,0.18659198836429958,0.088591415467569878,0.80785550422589203 +0.5203501400211421,0.078227991059971691,0.3967793082294383,0.20112073222820692,0.33071669250705105,0.89460186941146091,0.44242173111097094,0.64280918848416702 +0.078158529302830404,0.057433465432000785,0.61977782222415623,0.60159456416898138,0.81945147287972009,0.88949149256402904,0.44256229774907319,0.095515370151494139 +0.66198591493673697,0.26553021835494561,0.93838284941652195,0.44829690078027151,0.90161272992592623,0.12719461064858373,0.52885140773545647,0.028437434718284749 +0.23218535415322095,0.35057799013979196,0.82654896701313374,0.29602537301849774,0.5609117377016799,0.82575640590929922,0.41907906131370776,0.099734135471285845 +0.263516248128428,0.17039079021163439,0.68131728348481235,0.13015771941664511,0.52129640759000062,0.31092457857323896,0.95568319550503222,0.60032331939653172 +0.11959036821517344,0.45023225013144363,0.61877374756693559,0.36408222776856525,0.97594868413947289,0.61418443054373573,0.35318423217077133,0.16176635301398734 +0.27131467126368103,0.45826215243268842,0.099335544554979727,0.51421571104676711,0.35165527698100363,0.63080141332765272,0.92949954393035605,0.58346198539704364 +0.75418680278459116,0.57507964757200847,0.026746021581144808,0.18986895896498379,0.68655319957786254,0.30666133449057698,0.78901391069445026,0.98260441997100079 +0.70506762706878123,0.19638641225687037,0.28279738346229899,0.13610867799891713,0.39988482787193924,0.73655505226419238,0.61022858042433392,0.53982948473622838 +0.5364029977431507,0.49993741005092607,0.50438661073282298,0.47108570008705131,0.41399738016055054,0.48636951308340293,0.92242047650162529,0.6620380845292686 +0.89537317370329994,0.44606231161437282,0.26672879751112566,0.079200550145315907,0.29024629216737075,0.53529711948052727,0.50184417603338571,0.45302197204632366 +0.18360987096329928,0.32945974256008392,0.76330987594779465,0.0019189673453937628,0.58893257660505716,0.71207978554267881,0.5199031429024652,0.25722429221210275 +0.43242169164747624,0.42860117700126021,0.18381516568864587,0.26345822194136403,0.56283920951444366,0.62001785090349115,0.34483490378826465,0.91855869465229811 +0.23861845158652262,0.31726648876831853,0.37051571833054253,0.42892097601040535,0.33729565610681878,0.77680169087175155,0.377390729781792,0.072142987020896499 +0.84899358059453212,0.73848916663753161,0.81892793242302875,0.82298476451471259,0.4737007484160658,0.94240703348959387,0.64543660570230066,0.16712184443718126 +0.51302891799220351,0.55025789541546477,0.040998120973373742,0.029511115918076314,0.89801241977212842,0.42259591423166176,0.13845550710277954,0.88842184331064811 +0.73252483912896715,0.82333936893582937,0.99990415461188176,0.61465768135713394,0.2037860726237124,0.7278518499352209,0.56732964243598549,0.4049969252153317 +0.20511197426770744,0.32405797067182129,0.2823869297088254,0.11935552529507121,0.47326605380078668,0.0068486466150113889,0.81314588783246244,0.83268335893763523 +0.63505965103507067,0.066755431275853927,0.34009399288588371,0.46122143558914108,0.84464997350101756,0.11313043198878581,0.051678455756629216,0.021960453240060573 +0.1201545118817845,0.169332271379941,0.35810625984311784,0.84782666332974466,0.12780243475728348,0.75379862307493828,0.1827755780116328,0.67584947919513438 +0.20650234596239636,0.22848670989831199,0.78618876651725889,0.86042684791796598,0.9142992336903758,0.93619308043801663,0.69092288695200565,0.28414837407897386 +0.89303030414726214,0.36079475583669152,0.26319701618441405,0.39482166732266144,0.1951946060269224,0.89886725099347176,0.03144113423571282,0.39641464383877745 +0.1540399117674035,0.48697197595130615,0.3760069600199164,0.031259192090522858,0.049146560310475929,0.93393399461294369,0.25915064751333017,0.83210305899945258 +0.81814471361521679,0.80685904008690446,0.29479639437137484,0.27390796372475834,0.76383199592698758,0.025340844430415954,0.6210257850787535,0.48390300507788708 +0.15822101814143219,0.60241929930088656,0.35993747692045519,0.58598217001847452,0.62575160665788587,0.85645654158136209,0.70257090805642253,0.89400610988684792 +0.88411862212519543,0.44680777722871823,0.85302783572318541,0.51246289306602699,0.74636524960673201,0.13075653858888614,0.48810142638732812,0.47066692368482754 +0.43500062573468767,0.33276606108584539,0.0043503671805481536,0.68706094458541622,0.79985100385606978,0.14144280502176587,0.20916734954896721,0.14018008647456093 +0.73536056263394611,0.68287300743876123,0.94126718150280231,0.14717243128499669,0.00086513592592085686,0.10303821767059962,0.57552133553919127,0.25257163603811594 +0.27082663649055116,0.84297478083338451,0.43696055554872415,0.32497711725867218,0.26082803054396297,0.7398051345818657,0.2594321201544737,0.39116953555338652 +0.31654454628864481,0.75614665911664081,0.36407362393620601,0.84744872256712289,0.27935136815580286,0.7326104648406675,0.2906406400472249,0.89936777212349073 +0.96749833169767074,0.4937502314891552,0.71243974909587771,0.89595856160170706,0.43965468854205259,0.97347310944553023,0.93587641523256582,0.097493100709422964 +0.34168593110030232,0.53528798774960518,0.63430273825273431,0.28120678612555305,0.94791628851677923,0.52200089067470934,0.49730514703506901,0.24511882499303184 +0.84915609663078273,0.20718612690320115,0.054277789564034951,0.72962755095852094,0.72079282063465921,0.35776490664193816,0.72378342151822272,0.17054552979609794 +0.088214735325851668,0.59975752470398724,0.77102307481947874,0.48057160507284968,0.68322377027038217,0.54536303205896364,0.093961440084399883,0.033122574212533729 +0.26299044114663561,0.66628566556398838,0.0024049354089710206,0.12776250549730461,0.41189772087839649,0.15346375265719381,0.77424916119398257,0.10202204791392644 +0.29861798994087607,0.26345159252448236,0.73145975465341395,0.33839036147055912,0.13833264589690686,0.60198792066589313,0.37633881457370344,0.061030717643603238 +0.23507915888740383,0.73017085904660628,0.3886385676930314,0.98653414338232637,0.22440119712388651,0.88665954879561659,0.89648886318961984,0.81013471525594605 +0.60388135657950204,0.24039289302414213,0.63874227154469654,0.24354715127340387,0.40695808730500538,0.50512838914829428,0.23086976500973411,0.45599089792523506 +0.49669140056929428,0.80494506420395251,0.60737558130261049,0.068334458192574221,0.4461301414413632,0.74236635699702891,0.77054399664301421,0.72241847848767426 +0.60293630680270371,0.69276296230436385,0.078757972302510471,0.46149749982366406,0.16814850412565285,0.88263983132508872,0.8860061988943676,0.11085978684752795 +0.59628173662958117,0.3223188874491727,0.83235124702589858,0.31041338190083312,0.037710722864766955,0.99455467878947812,0.92510438852644239,0.11084741465637471 +0.76022534932309049,0.92091786594799074,0.093295547264289125,0.041981999095378697,0.8467714659365394,0.70125208085742485,0.4592278979043406,0.93519512149670758 diff --git a/data/borehole_y.csv b/data/borehole_y.csv new file mode 100644 index 0000000..3a91ba7 --- /dev/null +++ b/data/borehole_y.csv @@ -0,0 +1,80 @@ +16.841943481641277 +147.98987681904245 +180.84286662842061 +25.533591328603766 +31.807360190425864 +60.588513865214203 +67.643778067397037 +115.24528099323008 +68.813745944502827 +52.393103767584336 +22.363285265582384 +162.34444801908407 +32.246090353265252 +23.543673289943943 +26.585244581792463 +91.535923945196245 +153.71021477653295 +93.212360179499427 +131.24913149131055 +14.21503362103139 +97.238304300770253 +27.546701911349423 +35.950732716963223 +83.406065633835965 +40.903365364035771 +72.797265500263677 +133.46911939606625 +93.15716113904918 +41.990163143801958 +109.0223826883678 +34.394544680240834 +98.253135028452135 +35.455696599719253 +80.450122024444681 +149.83276361700757 +113.91569578614394 +66.578724619233498 +36.416462055643692 +26.27475589695057 +78.233884562508521 +41.382498668321062 +162.87478082500809 +45.136377913901519 +34.000537425963699 +88.505420682200622 +55.490142242971658 +21.7353034649325 +154.9864233861731 +80.535419382369952 +59.113737191301553 +132.54667184275937 +54.714290035604819 +123.32390572499261 +147.09088804306151 +103.02944869128422 +33.498391584372762 +40.001604539550719 +79.999770436523562 +149.0513531979056 +37.10613681506841 +76.139193316093326 +132.54995371821491 +53.779941155846302 +14.232777429002541 +66.286668244519234 +29.399766668553198 +25.105866826387849 +136.15663452360963 +33.91115128633497 +101.84262347516136 +80.469183685294226 +68.849061240365984 +64.223387389709146 +149.89136914088795 +113.16486517566811 +151.62276838851952 +48.895170776256492 +119.68545996976678 +62.458862552317214 +61.320805917309187 diff --git a/data/borehole_yt.csv b/data/borehole_yt.csv new file mode 100644 index 0000000..a623b2e --- /dev/null +++ b/data/borehole_yt.csv @@ -0,0 +1,200 @@ +45.489337121460018 +21.405352741476424 +185.9052669499778 +136.90866244412595 +84.421002257094202 +25.053020381180396 +21.227026521085389 +54.919261488775362 +38.802060005088272 +86.967344262234278 +184.40243699124721 +76.557260368927629 +21.707111123230874 +31.589545072174484 +37.142212918071067 +174.68838034511717 +106.85926716801386 +133.29529845765336 +114.7789554071702 +131.85074550178388 +19.527583815129812 +73.565478049404177 +121.54388004033021 +104.49525377500417 +55.540739593739659 +81.911469871378202 +122.20329785400116 +68.341404175883198 +126.17105666475069 +49.11518552520667 +101.00343447661814 +100.13162754635255 +64.84005909911123 +76.744502848442636 +92.330918252615973 +44.603942307727586 +63.977821989655126 +137.27303561057627 +40.266821053130009 +70.30138745150488 +23.022029350664447 +107.29891277446352 +27.916818818940392 +106.30269815772684 +116.6977902931413 +81.618663669000142 +70.62999189663239 +29.364160298681764 +100.95236897730423 +45.842878745002025 +59.436478328182517 +25.398502916938099 +17.031268983019771 +41.547432809602029 +17.527502326787911 +109.78136849968195 +52.16840759407021 +69.933713085506625 +21.386508396989456 +89.085583773827409 +36.995935204447278 +38.202557608387416 +38.834237360596269 +45.764220486586808 +39.66010301590493 +36.417667014535056 +20.575720373145447 +168.88579548296016 +59.141343679753156 +116.26164071494566 +70.753203268254154 +47.390306274577895 +43.127847877582731 +106.2611569504068 +33.862089168425975 +45.297467042960648 +24.383690916666833 +124.30457332019562 +55.529209277873107 +115.143001947375 +45.141530197525839 +30.370609135176618 +28.635935626627727 +20.922103259580069 +167.59600421007042 +120.67129657234585 +154.94171331098437 +114.06543631154662 +203.61854816597065 +90.411148632991143 +152.94679064162821 +48.895215469017614 +59.218376367808176 +45.43876748886926 +117.89163832042404 +77.234273247392636 +64.119257706175659 +46.462105046121032 +136.35865267767602 +152.96058082411662 +79.979529090827342 +172.0608530624439 +44.165991734177311 +153.70864580927903 +105.34637680141833 +12.304677455979688 +49.274043156386007 +45.728276313286202 +26.71395027347906 +21.698348493046495 +20.521336348694366 +104.36555948401065 +25.033986740991793 +37.828030122505993 +82.971792039662688 +194.41898655054814 +52.696156964708301 +142.5323393836203 +139.07672175140297 +46.741092693320361 +88.093696457204757 +55.330511547931046 +67.758917141707542 +35.420109092091351 +64.086456649904918 +48.10410651003852 +52.612005006642043 +40.311440598994913 +21.772776978232049 +106.56118185420202 +42.660319671695319 +99.313845113496598 +129.86333417405572 +160.47283750151246 +130.22176244999326 +141.14710127023122 +55.916287966493613 +135.72944495424713 +22.339626199633447 +19.669595096264285 +92.189140217167818 +38.592413524596964 +55.486462193214969 +112.17200234821834 +133.27309197601198 +56.245416339935318 +36.567616503201769 +85.079925018229943 +38.974007992103481 +41.771995051822614 +34.522509229970282 +20.70179043358473 +106.60361023300661 +211.2158038357428 +30.324129957295867 +89.417429973534624 +38.246785518268041 +94.161518213975285 +55.380742017626147 +19.71305588135246 +97.107943223667661 +28.282221651081514 +33.076569269142553 +24.251667106781191 +34.898576965122793 +104.06359803301251 +74.529139525557895 +66.809621036120163 +110.14974321989844 +22.146050138795538 +60.702693857949143 +31.901505665449573 +107.77248075930829 +76.706382754450914 +97.609794468559784 +35.052123482321683 +115.13147659996028 +33.654450146730959 +30.569570338697929 +129.98158171976885 +22.449167940150428 +128.80475628062854 +27.298491767936138 +156.53044959339363 +79.864875448779372 +101.06301226500194 +37.797996255295452 +58.355199862493535 +115.2412330772008 +43.054404379670729 +127.45006273206197 +25.921473857701308 +33.942776861764806 +38.699564763419119 +36.628790313821561 +85.390428201308765 +47.847390526708189 +56.798195030840667 +48.063688068517102 +90.206930186932311 diff --git a/data/branin_X.csv b/data/branin_X.csv new file mode 100644 index 0000000..cb4c01c --- /dev/null +++ b/data/branin_X.csv @@ -0,0 +1,25 @@ +0.056449627378410791,0.26943178240238125 +0.435784572810065,0.99276704745212585 +0.58513364076693897,0.50139523202364711 +0.17296674648074872,0.88802135612900668 +0.51086597720977056,0.45532769944704887 +0.24407090662753825,0.42926538696100086 +0.36771272054128568,0.32500168639495103 +0.29612710533183523,0.19260851685293484 +0.72378100383836252,0.67431072738879794 +0.4676219187647323,0.12140279166878087 +0.23174624354457879,0.82229097731628653 +0.79147370617064128,0.22698601379669112 +0.14887834401031938,0.071427071950469698 +0.86927748521729487,0.39321523001171804 +0.60016791604538133,0.029030199829788779 +0.92097511209485849,0.092733837920627393 +0.80444231847874037,0.93737075795896674 +0.98604909686289544,0.7971647286307938 +0.55019182917366938,0.74051020555790859 +0.036850978649520041,0.71683862731609793 +0.66127923812261569,0.84802286246194014 +0.34131164744698639,0.61784671831119309 +0.71981481426686456,0.30854726749829542 +0.88904932273205839,0.56664411876284182 +0.10561412433242597,0.55904510843487853 diff --git a/data/branin_Xt.csv b/data/branin_Xt.csv new file mode 100644 index 0000000..7a87967 --- /dev/null +++ b/data/branin_Xt.csv @@ -0,0 +1,200 @@ +0.44217227295272243,0.78901759462748711 +0.92883517101843105,0.17042606197312338 +0.85842780900467719,0.78008118189651876 +0.61572682952990154,0.5515917799926181 +0.090008414040849916,0.066436011213980128 +0.36395440971819548,0.56382552625339921 +0.59603640790456369,0.46942500529135189 +0.87341965920427,0.27891743261638047 +0.91470114458629015,0.12875580368063 +0.72330765612661896,0.94299476206174915 +0.0071488025233706898,0.0070146754608084727 +0.65611115848812707,0.55445433290798118 +0.94651428377004299,0.48982475344992249 +0.72627075322195833,0.63531945212934016 +0.84154166038552913,0.87694418203435021 +0.58085516499613843,0.81587202329207398 +0.69891279386383909,0.75791265396917706 +0.18119157938347696,0.60311037652189481 +0.14234958597001457,0.60032353632313884 +0.7626038059122805,0.086820184077650908 +0.11186104036238775,0.74418916658213197 +0.32062256074155193,0.19989839643733331 +0.75007045378192894,0.19795292432136735 +0.83072372731131638,0.82198204613595083 +0.053916963829623965,0.52273796131998462 +0.06484583577276648,0.72666405218749575 +0.0058732290974637946,0.94800955054377922 +0.43537510251961531,0.71952125426558322 +0.20064650389128846,0.99076045724539419 +0.084426903612385806,0.93958462242323071 +0.61830845161439763,0.095757230018457151 +0.14303257531275559,0.50325844688795984 +0.24117907360732416,0.82757768573576806 +0.15501613891033339,0.45614272588285909 +0.06757893150272487,0.19169136032447587 +0.74394765454437939,0.42198575465404942 +0.79566433908098122,0.022432735582047836 +0.45526027961659254,0.22670543264604381 +0.041969134457666701,0.76797985085979048 +0.75152717450358009,0.70027393045195352 +0.96431894999931078,0.71933691302467628 +0.0083535630910052117,0.99577654434371621 +0.49107197448063877,0.61224636436388424 +0.3034401566374697,0.3576014547910763 +0.61338323716802057,0.7400432125811105 +0.49722147209535672,0.5401526792567648 +0.42882614983424183,0.79263106477152101 +0.81662464835723969,0.99006174398007007 +0.051079393166953757,0.10305214156860165 +0.088007351038719706,0.97598737516737288 +0.20669571756588578,0.88706819131737635 +0.78471862813803983,0.89799767861774571 +0.018750356594929407,0.0207711338308052 +0.88592018667820327,0.929288750591311 +0.91997170735839207,0.028439447273798768 +0.19307172125323935,0.76951207043301029 +0.36299400492802036,0.026775567824404156 +0.37115676600116632,0.14199663417590525 +0.026832884623721798,0.9247216401204863 +0.32301841517045682,0.33125143832922455 +0.77341326510301378,0.16933829864247008 +0.055361531615479187,0.94052206655136317 +0.79823699901359413,0.1706929874189751 +0.058114782212122629,0.25603132525880468 +0.83719249918581173,0.87951861569513778 +0.55581040738549514,0.16028806653964967 +0.13601257793440014,0.97113546286683772 +0.038006819017857851,0.72798416175824521 +0.78764745715918405,0.82311590735054008 +0.15091833908836205,0.51382553647050577 +0.78473567086699736,0.39662721649426991 +0.11213203725839793,0.084201672054377896 +0.76902252303557894,0.67020564886877032 +0.16420304940658359,0.34089838593329702 +0.95308665451471919,0.42939797005730729 +0.37560315982637904,0.46861740632813664 +0.90424178672680777,0.34788392337639784 +0.77691560498790158,0.094489469117504776 +0.15774909680215321,0.069962651281692656 +0.80431459508952852,0.77203192874816617 +0.069217453967550524,0.70778546282554078 +0.14042714760127439,0.73941084152373415 +0.027547748316445131,0.64153621751536549 +0.87848815126631119,0.18961070438471372 +0.63460597662510088,0.55744114580299786 +0.31876176905004472,0.59772052057089065 +0.1714939792670247,0.830576390685829 +0.85561387892704721,0.60780675640160864 +0.83868485053539532,0.84832295939756008 +0.58414593733301567,0.63177931856820291 +0.71935086663360759,0.22990871470043284 +0.89805754888889422,0.26428062486215154 +0.59442668440972568,0.71810648571140645 +0.28522064665601476,0.88445798380784091 +0.72809476916014881,0.13816718483617652 +0.53822096846412115,0.27485705761574097 +0.438360416048446,0.78083629079733796 +0.55920798299151964,0.055494012131647397 +0.99823093169026944,0.77586784324529623 +0.11198378257637132,0.85288615748942231 +0.40175513784851757,0.95757972182259288 +0.90994983631532522,0.89179018757098727 +0.88044050020888409,0.056091759467511104 +0.36913982376475396,0.49712887776359471 +0.903073595780708,0.65300006886845552 +0.13684116946487057,0.4955818059240078 +0.35770027943443372,0.69926323197080609 +0.71928705467098963,0.73161976946167728 +0.628561338397021,0.39341656595279551 +0.1216973572747202,0.041119778717247923 +0.37262318934523342,0.68044401276901534 +0.54751122917157846,0.75576396515555433 +0.99789773101401624,0.14669331613891079 +0.38516255091568152,0.46666822201279567 +0.81840005805064731,0.9796538701898152 +0.29071799279316068,0.1311310141822658 +0.42702936816549431,0.026150618155328154 +0.84248706515183547,0.016683831236958824 +0.53705704856426772,0.66264776493275201 +0.41947305364252541,0.81743342743120073 +0.88046155051935504,0.63867680496949697 +0.53664959001243018,0.25602224163988496 +0.36451494535291451,0.68500917375131598 +0.52413037160987364,0.11190905368843318 +0.76825221449813075,0.85876164123686083 +0.35912254707983404,0.28465627959627482 +0.26775242430060964,0.21837134941219793 +0.89273880089008062,0.46320701864374059 +0.63641694396865356,0.49444250824172298 +0.87211383416274912,0.031539296562799746 +0.87076508373449524,0.48052446873597876 +0.64192779885974338,0.095065021673584771 +0.8231433992053161,0.39175164168258059 +0.33087687520954856,0.32143638706046096 +0.018411316809590406,0.87319854586237666 +0.92155373656305606,0.20581596094541221 +0.55188496454699454,0.48585080808920156 +0.41974822079656038,0.57661563134774729 +0.5446928112307059,0.24089044474948751 +0.34466111301214686,0.3506568223103127 +0.2338606360193981,0.46557562048313106 +0.88450809440822453,0.50715507176959351 +0.14045160316228333,0.48509668009589624 +0.360258546636122,0.96076386513093515 +0.56134634728644706,0.5340975518039186 +0.50190811473723018,0.36381673371817891 +0.37477072331755812,0.96508344731737683 +0.39084217320433012,0.60933856666296793 +0.65122668995859001,0.23352871928950392 +0.64063349602723219,0.99864098121415945 +0.46209347036148074,0.12687596706969639 +0.16318878439176177,0.15547728302456199 +0.23066575492329078,0.89434669536302658 +0.49417315039274734,0.4662367306790135 +0.65456402562107496,0.71058961869572645 +0.1697284684164404,0.62702325115012358 +0.51030799374590718,0.39954206433522288 +0.50285327550788073,0.73315356431785661 +0.58604851504040334,0.13354311518037121 +0.96578812327744989,0.11193849270848499 +0.083077899070327921,0.52442785427259542 +0.93376825844193123,0.43383100103336025 +0.94009652569179214,0.43501246449891506 +0.47880682700956556,0.22037005458370629 +0.90536028488981424,0.31812781713096183 +0.66675349078305979,0.3576766775612692 +0.85944984250491008,0.82166195954009547 +0.65694772385011224,0.55353558851297924 +0.020719990036436653,0.76842448392307683 +0.78046209677104172,0.96397398901165343 +0.36895580318514731,0.18426348727066455 +0.47832392489354769,0.92493425117829353 +0.746115337111384,0.69967584280146111 +0.3630711474317343,0.10918933553520471 +0.29246668272083998,0.34602499230663553 +0.48161928414880717,0.8415320136043527 +0.812147080924843,0.77126044170551766 +0.078417122269472839,0.41628642897226109 +0.39150900051524684,0.26150089118819875 +0.74408763066647698,0.45085208448287306 +0.71285997807277601,0.2753306296098561 +0.12898795235915839,0.34480539498689122 +0.51982539467826849,0.91287365307545121 +0.62585087258273753,0.88634058763789325 +0.45935240399237376,0.13251939589417994 +0.086633350583403601,0.65651377323830051 +0.62771050020820296,0.6797193086843597 +0.083760739416245067,0.052430589482174716 +0.14747024041364887,0.43763308889551578 +0.24296988898622252,0.13258663209207755 +0.46523994895009613,0.24497744473370631 +0.50496531693052349,0.67757190725172611 +0.71404821792359652,0.030626970897620343 +0.94777396510898293,0.44567518604116774 +0.17306471962612224,0.68298843561453237 +0.36841050863965308,0.58994133912522584 +0.45936162343167131,0.53382226460184778 +0.65002328895004424,0.10864465814886792 +0.01942201492659601,0.76871317861074151 +0.57692470809450225,0.48999684806854216 diff --git a/data/branin_y.csv b/data/branin_y.csv new file mode 100644 index 0000000..560b783 --- /dev/null +++ b/data/branin_y.csv @@ -0,0 +1,25 @@ +121.48190599474232 +132.03742032150166 +34.640118861733328 +10.415272536214367 +18.716568889555884 +15.906681327590452 +18.468017595964984 +34.461435671415757 +99.827412317368413 +8.1210955299534646 +23.477267434509667 +23.007704402098216 +107.58906048281199 +27.102349910019207 +5.3393982799416833 +2.511675114913217 +182.0881798494803 +84.874781734255166 +79.985148463123508 +31.302134533810641 +142.46598142595028 +31.479573472413108 +30.832792916092949 +51.656097368202694 +21.531893250415575 diff --git a/data/branin_yt.csv b/data/branin_yt.csv new file mode 100644 index 0000000..d5bc6df --- /dev/null +++ b/data/branin_yt.csv @@ -0,0 +1,200 @@ +74.843579760045472 +1.7544465779275278 +114.28579756826056 +50.453527769751005 +158.6967763729435 +28.599798564014584 +31.467022034411301 +14.403513807506229 +2.6825094424612068 +188.61021452089682 +293.10424256092097 +60.157534311067643 +26.220073670456962 +89.824755787706778 +150.95530498884841 +109.38481199029145 +120.96940611068656 +5.3145755385084739 +7.6003704831372971 +19.523416724730112 +2.9567897813343507 +30.382012367004243 +23.094742414914815 +134.96622414937272 +55.612760190598337 +16.920324174699644 +19.269921963420138 +58.358285147812765 +32.942124488014912 +2.1549724202027205 +5.9441436861106007 +17.16781400176702 +27.534632546345016 +20.253103179187576 +136.37857600542623 +46.899598143114957 +18.337880210874808 +7.5636227885014709 +22.018924684017641 +108.04476731154379 +68.98220060255008 +15.12921001492143 +41.909190885143175 +20.545148639076547 +95.676452528928152 +29.949504417789399 +73.800083421600107 +200.40672548773304 +188.04909774873107 +2.8243682191064057 +21.46636389419038 +170.35745823117904 +268.46799724225298 +156.16087820694523 +4.6838913414182315 +7.7519315635634758 +42.832838905502292 +27.149790972970873 +13.638277537630941 +21.127270648188279 +21.128215949084911 +5.6394203181091633 +19.299044250963043 +124.20339602325819 +153.04175931500077 +0.65758895751351965 +7.9796925681660928 +29.092687888623527 +143.5678155608615 +14.241755407452708 +41.552052432277094 +131.4203526021173 +99.165666085236595 +35.227907253783002 +17.059931631385947 +21.681773614407177 +15.126515965553352 +19.033239534844853 +102.3799429935243 +124.49174212939948 +17.259796048923036 +1.0466142387926283 +49.691956805548578 +8.4384065003504798 +55.982964334384597 +26.194976656049221 +6.1057119046810744 +68.818817497723671 +141.78626940099844 +60.495551197664369 +23.901592192833501 +9.1815347437450399 +84.558432428150567 +53.822568781101786 +19.915920605068717 +3.6387192963507644 +72.354006056967876 +2.2712283018044737 +76.825451033806331 +0.5579069250375106 +112.21425064188696 +134.89570676722005 +7.3202244752462526 +23.423387151446242 +67.256753354792636 +19.757650938481376 +44.504767589764732 +115.64482312430772 +26.809730581531404 +138.17254592886709 +43.98988810442895 +83.527889279501338 +2.3775041112671422 +21.506014341859604 +195.76858317019244 +43.748752558020378 +24.72903941989486 +13.340578360446653 +58.146109810652085 +78.288232837112972 +69.737809739016612 +2.666260103950993 +43.483715650836018 +1.4507470992022284 +157.77106357519907 +20.178116272459512 +34.811780781685741 +32.200089427219545 +44.19862346443081 +9.0466714935021049 +39.862390363345689 +9.2000532009673339 +35.785707278227378 +21.126440007447567 +21.019192719648171 +3.2340494896762362 +26.670528817653093 +32.810146490984067 +2.2533983676230998 +19.687982948920411 +13.573027407363762 +41.369841356117433 +20.137397926604343 +100.42786510877386 +36.095789029143901 +9.1986662306091507 +106.45952576761508 +35.754341664144079 +15.180384164131263 +193.18629458481399 +8.8765159853802924 +75.483507957521809 +31.990754834382695 +19.496018400217302 +98.034024346360582 +4.213680665779405 +12.455148129458184 +69.456331600923605 +2.3832707464308625 +1.1367482733206735 +37.287457842101858 +20.258502631327715 +19.466522715025839 +4.5156718692658382 +12.131519978006285 +29.493918197026986 +127.13147299354395 +60.143737612509305 +32.600075922630147 +196.25318716954339 +24.132185579956026 +119.55809342221517 +107.8649964976714 +32.189697029558971 +21.216594854418858 +94.867147336503521 +122.80650435695073 +62.302468066013468 +16.791733962988104 +51.620193077591502 +26.9996905339049 +48.312820279188884 +124.93514984608022 +145.97283878810975 +9.131325236586088 +16.381457551458439 +82.963889873493784 +170.62076328631503 +24.7749250785327 +53.086259176394805 +6.292965348733885 +56.731552760542399 +18.513084006404185 +19.790674888303577 +3.0009108323109617 +31.631426320643641 +27.725178135794117 +10.439999349696391 +33.270072843857477 +31.230154962590639 diff --git a/data/hartmann3_X.csv b/data/hartmann3_X.csv new file mode 100644 index 0000000..aff192e --- /dev/null +++ b/data/hartmann3_X.csv @@ -0,0 +1,40 @@ +0.62302414207256118,0.18209410987328403,0.52144420461799867 +0.3442912357561066,0.16028101711150675,0.43857616247630904 +0.81981149348128712,0.62440319277179912,0.40795864870039217 +0.7207595187573238,0.043143874893617984,0.20312605399684439 +0.77236535800629058,0.38585672372435426,0.70352064117875313 +0.38893186082267911,0.51789686669690016,0.23415617193985661 +0.7757997975742229,0.21645018505697,0.16281887847369286 +0.31436005519873228,0.90964801746472457,0.14980673101613307 +0.94194755533797569,0.091866258622931934,0.65482045033830361 +0.9937164065970624,0.53982503223719802,0.92995102493578585 +0.72888624121330359,0.83949249597755293,0.89988425891679036 +0.91829086685062555,0.35565582670753659,0.069841402215361753 +0.36337202001477947,0.42364359141248692,0.60254431664221131 +0.64453698471548748,0.79477451846926217,0.020380323033084274 +0.0023631273989765779,0.33615419894449566,0.34100882770776619 +0.13001334758062916,0.47134013952059411,0.19780059828775781 +0.29803186165595003,0.81409103872026944,0.95759702541293978 +0.22466308865432807,0.28975460803315523,0.86810397829982389 +0.55237313555042555,0.64495175485145617,0.57394262234440063 +0.24396858015088693,0.73574736177938083,0.3180031822416145 +0.48702671673577147,0.88044179512065701,0.63737100822620341 +0.062363379108193208,0.93587218201782463,0.49184815523713626 +0.15415257422677614,0.2433948640014883,0.59165359447409049 +0.69157868085579344,0.48503864604789915,0.45957981215440552 +0.043482098686570346,0.56804896500644964,0.74226369922795776 +0.58536531932480385,0.13122076106558939,0.7527764490492127 +0.11027312468110409,0.066280685539309672,0.92322795539424596 +0.86967106635665081,0.26964191996904352,0.81127272146492968 +0.076401429582656222,0.67560944505928655,0.096793508905553508 +0.26882937042622473,0.11285537394320555,0.040708525479336871 +0.66329952382663482,0.6704794046575786,0.83991382834762263 +0.53310421655046791,0.30087674479298804,0.29055631031782514 +0.83507944083239705,0.97170933398282722,0.35501428903871257 +0.18449421008955752,0.76886989323354338,0.6858544722328741 +0.89051742724519389,0.72248597422535166,0.54402130267107762 +0.96784204218643466,0.87379469053385184,0.26801205628199348 +0.43831977965436647,0.00010494752836333054,0.37708165623090401 +0.41541184738830622,0.99950952470963694,0.784668759631858 +0.52007607108947396,0.59329842826080925,0.12162952853081421 +0.45742772104191909,0.43725799595264636,0.9965310968853307 diff --git a/data/hartmann3_Xt.csv b/data/hartmann3_Xt.csv new file mode 100644 index 0000000..5d5c526 --- /dev/null +++ b/data/hartmann3_Xt.csv @@ -0,0 +1,200 @@ +0.44217227295272243,0.78901759462748711,0.92883517101843105 +0.17042606197312338,0.85842780900467719,0.78008118189651876 +0.61572682952990154,0.5515917799926181,0.090008414040849916 +0.066436011213980128,0.36395440971819548,0.56382552625339921 +0.59603640790456369,0.46942500529135189,0.87341965920427 +0.27891743261638047,0.91470114458629015,0.12875580368063 +0.72330765612661896,0.94299476206174915,0.0071488025233706898 +0.0070146754608084727,0.65611115848812707,0.55445433290798118 +0.94651428377004299,0.48982475344992249,0.72627075322195833 +0.63531945212934016,0.84154166038552913,0.87694418203435021 +0.58085516499613843,0.81587202329207398,0.69891279386383909 +0.75791265396917706,0.18119157938347696,0.60311037652189481 +0.14234958597001457,0.60032353632313884,0.7626038059122805 +0.086820184077650908,0.11186104036238775,0.74418916658213197 +0.32062256074155193,0.19989839643733331,0.75007045378192894 +0.19795292432136735,0.83072372731131638,0.82198204613595083 +0.053916963829623965,0.52273796131998462,0.06484583577276648 +0.72666405218749575,0.0058732290974637946,0.94800955054377922 +0.43537510251961531,0.71952125426558322,0.20064650389128846 +0.99076045724539419,0.084426903612385806,0.93958462242323071 +0.61830845161439763,0.095757230018457151,0.14303257531275559 +0.50325844688795984,0.24117907360732416,0.82757768573576806 +0.15501613891033339,0.45614272588285909,0.06757893150272487 +0.19169136032447587,0.74394765454437939,0.42198575465404942 +0.79566433908098122,0.022432735582047836,0.45526027961659254 +0.22670543264604381,0.041969134457666701,0.76797985085979048 +0.75152717450358009,0.70027393045195352,0.96431894999931078 +0.71933691302467628,0.0083535630910052117,0.99577654434371621 +0.49107197448063877,0.61224636436388424,0.3034401566374697 +0.3576014547910763,0.61338323716802057,0.7400432125811105 +0.49722147209535672,0.5401526792567648,0.42882614983424183 +0.79263106477152101,0.81662464835723969,0.99006174398007007 +0.051079393166953757,0.10305214156860165,0.088007351038719706 +0.97598737516737288,0.20669571756588578,0.88706819131737635 +0.78471862813803983,0.89799767861774571,0.018750356594929407 +0.0207711338308052,0.88592018667820327,0.929288750591311 +0.91997170735839207,0.028439447273798768,0.19307172125323935 +0.76951207043301029,0.36299400492802036,0.026775567824404156 +0.37115676600116632,0.14199663417590525,0.026832884623721798 +0.9247216401204863,0.32301841517045682,0.33125143832922455 +0.77341326510301378,0.16933829864247008,0.055361531615479187 +0.94052206655136317,0.79823699901359413,0.1706929874189751 +0.058114782212122629,0.25603132525880468,0.83719249918581173 +0.87951861569513778,0.55581040738549514,0.16028806653964967 +0.13601257793440014,0.97113546286683772,0.038006819017857851 +0.72798416175824521,0.78764745715918405,0.82311590735054008 +0.15091833908836205,0.51382553647050577,0.78473567086699736 +0.39662721649426991,0.11213203725839793,0.084201672054377896 +0.76902252303557894,0.67020564886877032,0.16420304940658359 +0.34089838593329702,0.95308665451471919,0.42939797005730729 +0.37560315982637904,0.46861740632813664,0.90424178672680777 +0.34788392337639784,0.77691560498790158,0.094489469117504776 +0.15774909680215321,0.069962651281692656,0.80431459508952852 +0.77203192874816617,0.069217453967550524,0.70778546282554078 +0.14042714760127439,0.73941084152373415,0.027547748316445131 +0.64153621751536549,0.87848815126631119,0.18961070438471372 +0.63460597662510088,0.55744114580299786,0.31876176905004472 +0.59772052057089065,0.1714939792670247,0.830576390685829 +0.85561387892704721,0.60780675640160864,0.83868485053539532 +0.84832295939756008,0.58414593733301567,0.63177931856820291 +0.71935086663360759,0.22990871470043284,0.89805754888889422 +0.26428062486215154,0.59442668440972568,0.71810648571140645 +0.28522064665601476,0.88445798380784091,0.72809476916014881 +0.13816718483617652,0.53822096846412115,0.27485705761574097 +0.438360416048446,0.78083629079733796,0.55920798299151964 +0.055494012131647397,0.99823093169026944,0.77586784324529623 +0.11198378257637132,0.85288615748942231,0.40175513784851757 +0.95757972182259288,0.90994983631532522,0.89179018757098727 +0.88044050020888409,0.056091759467511104,0.36913982376475396 +0.49712887776359471,0.903073595780708,0.65300006886845552 +0.13684116946487057,0.4955818059240078,0.35770027943443372 +0.69926323197080609,0.71928705467098963,0.73161976946167728 +0.628561338397021,0.39341656595279551,0.1216973572747202 +0.041119778717247923,0.37262318934523342,0.68044401276901534 +0.54751122917157846,0.75576396515555433,0.99789773101401624 +0.14669331613891079,0.38516255091568152,0.46666822201279567 +0.81840005805064731,0.9796538701898152,0.29071799279316068 +0.1311310141822658,0.42702936816549431,0.026150618155328154 +0.84248706515183547,0.016683831236958824,0.53705704856426772 +0.66264776493275201,0.41947305364252541,0.81743342743120073 +0.88046155051935504,0.63867680496949697,0.53664959001243018 +0.25602224163988496,0.36451494535291451,0.68500917375131598 +0.52413037160987364,0.11190905368843318,0.76825221449813075 +0.85876164123686083,0.35912254707983404,0.28465627959627482 +0.26775242430060964,0.21837134941219793,0.89273880089008062 +0.46320701864374059,0.63641694396865356,0.49444250824172298 +0.87211383416274912,0.031539296562799746,0.87076508373449524 +0.48052446873597876,0.64192779885974338,0.095065021673584771 +0.8231433992053161,0.39175164168258059,0.33087687520954856 +0.32143638706046096,0.018411316809590406,0.87319854586237666 +0.92155373656305606,0.20581596094541221,0.55188496454699454 +0.48585080808920156,0.41974822079656038,0.57661563134774729 +0.5446928112307059,0.24089044474948751,0.34466111301214686 +0.3506568223103127,0.2338606360193981,0.46557562048313106 +0.88450809440822453,0.50715507176959351,0.14045160316228333 +0.48509668009589624,0.360258546636122,0.96076386513093515 +0.56134634728644706,0.5340975518039186,0.50190811473723018 +0.36381673371817891,0.37477072331755812,0.96508344731737683 +0.39084217320433012,0.60933856666296793,0.65122668995859001 +0.23352871928950392,0.64063349602723219,0.99864098121415945 +0.46209347036148074,0.12687596706969639,0.16318878439176177 +0.15547728302456199,0.23066575492329078,0.89434669536302658 +0.49417315039274734,0.4662367306790135,0.65456402562107496 +0.71058961869572645,0.1697284684164404,0.62702325115012358 +0.51030799374590718,0.39954206433522288,0.50285327550788073 +0.73315356431785661,0.58604851504040334,0.13354311518037121 +0.96578812327744989,0.11193849270848499,0.083077899070327921 +0.52442785427259542,0.93376825844193123,0.43383100103336025 +0.94009652569179214,0.43501246449891506,0.47880682700956556 +0.22037005458370629,0.90536028488981424,0.31812781713096183 +0.66675349078305979,0.3576766775612692,0.85944984250491008 +0.82166195954009547,0.65694772385011224,0.55353558851297924 +0.020719990036436653,0.76842448392307683,0.78046209677104172 +0.96397398901165343,0.36895580318514731,0.18426348727066455 +0.47832392489354769,0.92493425117829353,0.746115337111384 +0.69967584280146111,0.3630711474317343,0.10918933553520471 +0.29246668272083998,0.34602499230663553,0.48161928414880717 +0.8415320136043527,0.812147080924843,0.77126044170551766 +0.078417122269472839,0.41628642897226109,0.39150900051524684 +0.26150089118819875,0.74408763066647698,0.45085208448287306 +0.71285997807277601,0.2753306296098561,0.12898795235915839 +0.34480539498689122,0.51982539467826849,0.91287365307545121 +0.62585087258273753,0.88634058763789325,0.45935240399237376 +0.13251939589417994,0.086633350583403601,0.65651377323830051 +0.62771050020820296,0.6797193086843597,0.083760739416245067 +0.052430589482174716,0.14747024041364887,0.43763308889551578 +0.24296988898622252,0.13258663209207755,0.46523994895009613 +0.24497744473370631,0.50496531693052349,0.67757190725172611 +0.71404821792359652,0.030626970897620343,0.94777396510898293 +0.44567518604116774,0.17306471962612224,0.68298843561453237 +0.36841050863965308,0.58994133912522584,0.45936162343167131 +0.53382226460184778,0.65002328895004424,0.10864465814886792 +0.01942201492659601,0.76871317861074151,0.57692470809450225 +0.48999684806854216,0.35212090470108326,0.37408894416049887 +0.22425360703023356,0.99199739291729983,0.79243589991150853 +0.95534664543425973,0.057222604827159684,0.17337978654682307 +0.039427541404710831,0.14796363757148145,0.042743153993095917 +0.85033417900154784,0.090283821204099612,0.60211540875303704 +0.45826510070619542,0.97349262079913423,0.048908932107002867 +0.94179869200259558,0.22597852949657726,0.55279069088148436 +0.85821145832999857,0.91872393589636125,0.67122847744385783 +0.50109020180830954,0.25223028027912497,0.17884147966941855 +0.45599282138873942,0.22160909066511836,0.3186815135049762 +0.02241249130413725,0.55773921291192075,0.38470098605538539 +0.022533572253770617,0.6050657075846082,0.043487613941568748 +0.53934104885917711,0.2048044474554539,0.94910685929557614 +0.097665495628938448,0.06057537661403456,0.71700211525279567 +0.65741920395293185,0.65794164196968086,0.35149971818475356 +0.57547215780940353,0.087961234899303964,0.7781823974288542 +0.74953302192572147,0.63402143637809749,0.0034196367376099213 +0.77070281331927726,0.1672559086248796,0.44526581792866182 +0.83706659052125232,0.91078429152817331,0.1012808804706522 +0.73914808091126138,0.40854152210420125,0.63993167458299149 +0.22520281850781543,0.47000318227460902,0.59929212221298844 +0.98392019982401435,0.041155673701470885,0.1190314594746964 +0.99759038488170293,0.68235615733725707,0.18515863222281304 +0.051831662179558036,0.38119375500461883,0.8487201410216858 +0.36909893970598429,0.90339893345591704,0.90182547532260471 +0.61715655970918815,0.0097326073486683207,0.94038167174886045 +0.86156765053749718,0.71930243613700473,0.87715670858166883 +0.22703695207567265,0.14161948706908112,0.74195050612809177 +0.78085154440425664,0.78874025746431087,0.93057904211114939 +0.29183740899010713,0.31095399060180351,0.14695806170176184 +0.88492340806357972,0.34925993492283247,0.85339396238931886 +0.80163748078099806,0.7905610355581868,0.089924089264773799 +0.73181692511633911,0.22067059670934686,0.48735839477942899 +0.098171496800493974,0.90675394017506905,0.069557294218870247 +0.29097149092197205,0.72153013438523284,0.066503145201721736 +0.35312318593519987,0.9316807688165416,0.63936217566431408 +0.735063372648463,0.94773068861385878,0.75631427041136379 +0.52584791059636649,0.010543887407105879,0.26303186664623135 +0.36704837788007072,0.55744390330797899,0.18137362317991101 +0.20461573382027953,0.035663941053448722,0.86734566249117118 +0.68820239436232844,0.18894522921359036,0.13793503477272884 +0.2992724093786121,0.1033144512351708,0.61661691128291896 +0.37718905345000275,0.18403654079553633,0.81884407314419161 +0.038227921629746975,0.77827057523899035,0.95559910361534939 +0.9680367654491332,0.34413978459763217,0.65740085362300327 +0.66738990604949111,0.2196608562894693,0.82417232049083256 +0.51559227582518863,0.86281306922552536,0.79908207705099499 +0.62387497079999055,0.053929081730902717,0.48520601035204247 +0.6815889692064212,0.39069062396181164,0.48752675783748556 +0.86363627145154276,0.94258203134459617,0.3438459442937547 +0.88575784606477337,0.24673919626437923,0.11552216009078442 +0.81557778206623033,0.85147269494307232,0.29789800564710833 +0.92137862449980523,0.43824762226175451,0.72211597104877523 +0.61593717771593581,0.096236728196555599,0.35524254697284363 +0.090596855022192613,0.0052066592098156672,0.51197944610236534 +0.71671354920978192,0.041369372745930111,0.18728004180511326 +0.48797309456244464,0.31041525280588955,0.38059334668669942 +0.32347118376854567,0.4332118004061899,0.35469937052280043 +0.59253875183505034,0.23446675682369533,0.36515867001065516 +0.23374602937207489,0.084866827899760189,0.86600386958042808 +0.43821434737536791,0.17174863438835963,0.26280015327115558 +0.48449978001673233,0.7255442833110054,0.90821030764840693 +0.71799816252840154,0.70948774121983949,0.58554834348194562 +0.32636025957107351,0.82728362015938073,0.93560659543072677 +0.45552524752096912,0.050735310033296122,0.89609171757556638 +0.70200978162362926,0.9593139683032641,0.08084401957295928 +0.2055759450644209,0.032777485077280843,0.36923928401289874 diff --git a/data/hartmann3_y.csv b/data/hartmann3_y.csv new file mode 100644 index 0000000..d82a2b0 --- /dev/null +++ b/data/hartmann3_y.csv @@ -0,0 +1,40 @@ +-0.23613813851434406 +-0.43717764125912023 +-0.22504523156942588 +-0.57726407325352436 +-1.8118217063860205 +-0.22460601752279963 +-0.39739797665366339 +-0.020301993517470543 +-0.31313778772470774 +-2.998175496845195 +-1.6265154734237697 +-0.071047937940617839 +-1.0567344607321085 +-0.0015206087924520368 +-0.39577549438705789 +-0.22077827927097554 +-1.5523605796870705 +-1.9853018986047837 +-1.3605569543754574 +-0.4579805385249231 +-1.852520147040978 +-2.5598792804084955 +-0.48947467702299186 +-0.31568052471948593 +-3.0047528425641081 +-0.70899020309299809 +-0.32856690047212928 +-1.7567644922017924 +-0.018027061827510019 +-0.20793908318458354 +-3.215940701275847 +-0.65595645227084165 +-0.16998136442497755 +-2.4925597650486582 +-0.55018914703329125 +-0.029061637802246118 +-0.60078336460847137 +-0.81341091106545438 +-0.05408521487595766 +-1.7936922862355713 diff --git a/data/hartmann3_yt.csv b/data/hartmann3_yt.csv new file mode 100644 index 0000000..c05ce94 --- /dev/null +++ b/data/hartmann3_yt.csv @@ -0,0 +1,200 @@ +-1.984026384124711 +-1.826363591237814 +-0.049832337836311073 +-0.65548545518453982 +-3.462357802696443 +-0.012648268415514776 +-0.00021242133331988186 +-2.0868493877192842 +-2.3245282111002501 +-1.6981911123271107 +-1.5951559607198642 +-0.3662801938298702 +-3.2096965411070224 +-0.6026014492894034 +-1.1083514130258139 +-2.000838739617735 +-0.042507354959886126 +-0.14844000505713159 +-0.062963986913777206 +-0.32778155724103142 +-0.51977612847708587 +-1.5787491474383792 +-0.083827071758918653 +-1.4879897953157262 +-0.19477597955488055 +-0.36281386252147996 +-2.1756373720810465 +-0.1008769683261455 +-0.22755069875656944 +-2.8633885551660283 +-0.4976705280310782 +-1.1617957771678655 +-0.28102553352367038 +-1.1019795748206898 +-0.00034651675386555451 +-1.2166540750917425 +-0.31514292277640954 +-0.059491341242546986 +-0.17534829429958801 +-0.23590179348473969 +-0.15477193578395754 +-0.0070133728757716708 +-1.7221901251212395 +-0.049040035592249058 +-0.0010231606387752332 +-2.098439058135003 +-3.4364257307975312 +-0.36485053670296014 +-0.026629282081892788 +-1.4991581095075972 +-3.2964585028472362 +-0.0092465755969006417 +-0.47525696482095636 +-0.37106500955987365 +-0.0037660982142699246 +-0.025525334057298329 +-0.20014653155148809 +-1.0170390188966705 +-3.4514548423026574 +-1.1438601557712904 +-1.2729049849287062 +-2.7138263915992273 +-1.7955828332568837 +-0.23774854597117323 +-2.1505373316502494 +-0.99191447645247166 +-1.4868250289475962 +-1.0227886437872311 +-0.32407189521206597 +-1.6857733788831648 +-0.3892183347898997 +-1.9798258277919596 +-0.20190855565344101 +-1.6266057502121531 +-1.4634485584507144 +-0.42517231843457581 +-0.073555897958854838 +-0.056488786417656672 +-0.097053561527257887 +-3.1058053525953775 +-0.50227941216265681 +-1.6396693607375405 +-0.64051796873392941 +-0.27349896786792077 +-1.2446303153739648 +-1.1706983732761538 +-0.28774035681088184 +-0.027206076568089543 +-0.24120750952683684 +-0.26127567866181595 +-0.23420165192389869 +-0.79457120347400201 +-0.66442879581888226 +-0.35275099867155918 +-0.061414060705475185 +-1.832633681704162 +-0.65604309302432851 +-1.8968412246097801 +-2.0201935890392515 +-1.9982803375123201 +-0.70321831553073577 +-1.3385654182229703 +-1.6086688755165075 +-0.43023407574277223 +-0.422806281031539 +-0.04550707949071419 +-0.12403332296691955 +-1.1165898623682362 +-0.18502304529465236 +-0.53569262343167046 +-2.6117694853559441 +-0.67405699080355952 +-2.4719430018602555 +-0.14941690875331878 +-1.2365502649120748 +-0.18588571864544975 +-0.39015468512519191 +-1.5313726462571575 +-0.38100694269691843 +-1.745987092544838 +-0.30755095437980501 +-3.4252366951589748 +-1.0351662440265978 +-0.32070986860375295 +-0.01373683109764008 +-0.33562118920194822 +-0.33298334732591411 +-2.1269839024484094 +-0.19227618466863255 +-0.68394979037207415 +-0.93077316221974504 +-0.027990238363607645 +-2.8142818469753941 +-0.44812892675262689 +-0.92995679213196714 +-0.26392494357391072 +-0.15755518978433217 +-0.20509450787129582 +-0.0010261335228726805 +-0.25320493023286617 +-0.65036333596228835 +-0.62556403678178418 +-0.81632419352156282 +-0.53261878996692591 +-0.014901928048360847 +-0.84942649547296167 +-0.36543929849734436 +-0.2589260034907544 +-0.53873746985096738 +-0.0055888600197336906 +-0.26038162476761167 +-0.001680459107081895 +-1.1731060046498494 +-1.2660765896565738 +-0.15695878447375508 +-0.013464654826596936 +-2.9301026486764896 +-1.1832042831948488 +-0.16511789901104021 +-2.7368318978305002 +-0.74385377521349483 +-1.882857459358718 +-0.43750805225205036 +-2.4763342910631549 +-0.0033946446046906288 +-0.22674618565497384 +-0.0030299246827193353 +-0.0092729704571478735 +-2.1370217488248917 +-0.77953863660509115 +-0.82893679343960003 +-0.12902976958099174 +-0.31548063612001986 +-0.42335355857221574 +-0.27494498370220893 +-1.1236784105996722 +-1.856337835201586 +-1.1168444858602753 +-1.3778872620608635 +-1.5495063355959935 +-0.21725244060856083 +-0.29771311215775975 +-0.1373064832095417 +-0.19000904218691975 +-0.0949998318092663 +-2.174182760204916 +-0.66018073795530452 +-0.14476343749704723 +-0.54216809856665171 +-0.49115184642615417 +-0.40998262230792631 +-0.57597275376735024 +-0.49584386362219357 +-0.95739122352820716 +-2.6891356737483862 +-1.0798451638334057 +-1.6267512448558112 +-0.32399360756735285 +-0.0013613999011384428 +-0.63203754118163336 diff --git a/known_issues.csv b/known_issues.csv new file mode 100644 index 0000000..3fcf230 --- /dev/null +++ b/known_issues.csv @@ -0,0 +1,11 @@ +suite,check,issue +py07_scalability,vecchia(n-1) LL == exact LL (same theta),"With m >= n-1 neighbours the Vecchia likelihood should be the exact one; it matches for n=20 but not for n=40 (-111.349 vs -110.834)" +py07_scalability,hartmann3/n=1500/nystrom(100): Q2,"Nystrom predictor degrades as n grows at fixed k (theta estimates are fine: the exact predictor at the Nystrom theta has Q2 ~ 1)" +py07_scalability,hartmann3/n=5000/nystrom(100): Q2,"Nystrom predictor degrades as n grows at fixed k (theta estimates are fine: the exact predictor at the Nystrom theta has Q2 ~ 1)" +py07_scalability,otl_circuit/n=1500/nystrom(100): Q2,"Nystrom predictor degrades as n grows at fixed k (theta estimates are fine: the exact predictor at the Nystrom theta has Q2 ~ 1)" +py07_scalability,otl_circuit/n=5000/nystrom(100): Q2,"Nystrom predictor degrades as n grows at fixed k (theta estimates are fine: the exact predictor at the Nystrom theta has Q2 ~ 1)" +py09_robustness,NaN in y is rejected,"Kriging accepts NaN in y without raising" +py09_robustness,Inf in y is rejected,"Kriging accepts Inf in y without raising" +py09_robustness,WarpKriging with 1 spec for 2 columns is rejected,"WarpKriging accepts fewer warping specs than columns of X" +py09_robustness,predict(stdev) time growth exponent in q (8000 vs 4000),"predict(return_stdev=True, return_cov=False) is O(q^2) in time and memory (10 000 points: 1.6 GB; 30 000: out of memory)" +py10_compare_python_packages,sklearn RBF == libKriging gauss: simple kriging stdev (beta fixed),"With beta fixed (is_beta_estim=False) predict still adds the trend-estimation variance (universal kriging) instead of the simple kriging variance" diff --git a/lkbench/__init__.py b/lkbench/__init__.py new file mode 100644 index 0000000..bebe556 --- /dev/null +++ b/lkbench/__init__.py @@ -0,0 +1,4 @@ +"""Helpers shared by the libKriging benchmark notebooks.""" +from . import checks, compat, design, functions, metrics # noqa: F401 +from .checks import Suite # noqa: F401 +from .functions import TESTS # noqa: F401 diff --git a/lkbench/checks.py b/lkbench/checks.py new file mode 100644 index 0000000..6e960e9 --- /dev/null +++ b/lkbench/checks.py @@ -0,0 +1,166 @@ +"""Record checks and metrics from a notebook, and fail it at the end if needed. + +A notebook creates one ``Suite``, records any number of checks (assertions that +do *not* stop the notebook, so every problem of a run is reported at once) and +metrics, then calls ``suite.finalize()`` in its last cell: it writes + + results/.checks.csv suite,lang,check,severity,status,value,expected,detail + results/.metrics.csv suite,lang,case,package,metric,value + +and raises ``AssertionError`` if any check of severity "error" failed, which makes +the notebook (hence the CI job) fail. Checks of severity "warn" are reported but +never fail the run: they flag performance drifts or comparisons with other +packages, where a threshold is a judgement call rather than a correctness bound. + +Statuses: PASS, FAIL (error-level), WARN (warn-level), SKIP (feature missing in +the installed version), KNOWN (a failure listed in ``known_issues.csv``: reported +with its issue, but does not fail the run). A listed check that passes is +reported as FIXED by ``tools/report.py``, so that the list does not rot. + +The R, Octave and Julia notebooks write the same two CSV formats with a few lines +of their own code (see notebooks/*/checks.*), so that ``tools/report.py`` can +aggregate every language in one report. +""" +import csv +import math +import os +import time +import traceback +from contextlib import contextmanager +from pathlib import Path + +import numpy as np + +ROOT = Path(__file__).resolve().parents[1] +RESULTS_DIR = Path(os.environ.get("LKBENCH_RESULTS", ROOT / "results")) +KNOWN_ISSUES_FILE = ROOT / "known_issues.csv" + + +def known_issues(): + """{(suite, check): issue} from known_issues.csv.""" + if not KNOWN_ISSUES_FILE.exists(): + return {} + with open(KNOWN_ISSUES_FILE, newline="") as fh: + return {(r["suite"], r["check"]): r["issue"] for r in csv.DictReader(fh)} +CHECK_FIELDS = ["suite", "lang", "check", "severity", "status", "value", "expected", "detail"] +METRIC_FIELDS = ["suite", "lang", "case", "package", "metric", "value"] + + +def _fmt(v): + if v is None: + return "" + if isinstance(v, (bool, np.bool_)): + return str(bool(v)) + if isinstance(v, (int, float, np.integer, np.floating)): + return f"{float(v):.6g}" + return str(v) + + +class Suite: + def __init__(self, name, lang="python"): + self.name, self.lang = name, lang + self.checks, self.metrics = [], [] + self.t0 = time.time() + self.known = known_issues() + + # -- checks --------------------------------------------------------------- + + def _add(self, check, ok, value=None, expected=None, detail="", severity="error"): + status = "PASS" if ok else ("FAIL" if severity == "error" else "WARN") + if not ok and (self.name, check) in self.known: + status, detail = "KNOWN", f"{self.known[self.name, check]} | {detail}".rstrip(" |") + self.checks.append(dict(suite=self.name, lang=self.lang, check=check, severity=severity, + status=status, value=_fmt(value), expected=_fmt(expected), detail=detail)) + if not ok: + print(f"[{status}] {check}: value={_fmt(value)} expected {_fmt(expected)} {detail}") + return ok + + def true(self, check, cond, detail="", severity="error"): + return self._add(check, bool(cond), cond, True, detail, severity) + + def less(self, check, value, bound, detail="", severity="error"): + """value < bound (NaN fails).""" + value = float(value) + return self._add(check, math.isfinite(value) and value < bound, value, f"< {bound:g}", detail, severity) + + def greater(self, check, value, bound, detail="", severity="error"): + value = float(value) + return self._add(check, math.isfinite(value) and value > bound, value, f"> {bound:g}", detail, severity) + + def between(self, check, value, lo, hi, detail="", severity="error"): + value = float(value) + return self._add(check, math.isfinite(value) and lo <= value <= hi, value, f"in [{lo:g}, {hi:g}]", + detail, severity) + + def close(self, check, a, b, rtol=1e-6, atol=0.0, detail="", severity="error"): + """All |a - b| <= atol + rtol |b| (element-wise); reports the worst relative gap.""" + a, b = np.asarray(a, float), np.asarray(b, float) + if a.shape != b.shape: + return self._add(check, False, f"shape {a.shape}", f"shape {b.shape}", detail, severity) + gap = np.abs(a - b) + ok = bool(np.all(np.isfinite(a)) and np.all(gap <= atol + rtol * np.abs(b))) + rel = float(np.max(gap / np.maximum(np.abs(b), 1e-300))) if a.size else 0.0 + return self._add(check, ok, rel, f"rtol {rtol:g}, atol {atol:g}", detail, severity) + + def finite(self, check, a, detail="", severity="error"): + a = np.asarray(a, float) + return self._add(check, bool(a.size and np.all(np.isfinite(a))), f"{np.sum(~np.isfinite(a))} non-finite", + "all finite", detail, severity) + + def skip(self, check, reason): + self.checks.append(dict(suite=self.name, lang=self.lang, check=check, severity="info", + status="SKIP", value="", expected="", detail=reason)) + print(f"[SKIP] {check}: {reason}") + + @contextmanager + def runs(self, check, severity="error"): + """``with suite.runs("x"): ...`` records a FAIL (with the exception) instead of stopping.""" + try: + yield + except Exception as exc: # noqa: BLE001 - every failure is a finding here + self._add(check, False, type(exc).__name__, "no exception", + str(exc).splitlines()[0][:200] if str(exc) else traceback.format_exc(limit=1), severity) + else: + self._add(check, True, "ok", "no exception", "", severity) + + def raises(self, check, fn, *args, severity="error", **kwargs): + """``fn(*args)`` must raise (invalid input must be rejected, not silently accepted).""" + try: + fn(*args, **kwargs) + except Exception as exc: # noqa: BLE001 + return self._add(check, True, type(exc).__name__, "an exception", "", severity) + return self._add(check, False, "no exception", "an exception", "", severity) + + # -- metrics -------------------------------------------------------------- + + def metric(self, case, package, **values): + for k, v in values.items(): + self.metrics.append(dict(suite=self.name, lang=self.lang, case=case, package=package, + metric=k, value=_fmt(v))) + + def table(self): + """Metrics as a wide pandas DataFrame (one row per case x package).""" + import pandas as pd + if not self.metrics: + return pd.DataFrame() + df = pd.DataFrame(self.metrics) + df["value"] = pd.to_numeric(df["value"], errors="coerce") + return df.pivot_table(index=["case", "package"], columns="metric", values="value", sort=False) + + # -- output --------------------------------------------------------------- + + def finalize(self): + RESULTS_DIR.mkdir(parents=True, exist_ok=True) + self.metric("_suite", "_", elapsed_s=time.time() - self.t0) + for rows, fields, ext in ((self.checks, CHECK_FIELDS, "checks"), (self.metrics, METRIC_FIELDS, "metrics")): + with open(RESULTS_DIR / f"{self.name}.{ext}.csv", "w", newline="") as fh: + w = csv.DictWriter(fh, fieldnames=fields) + w.writeheader() + w.writerows(rows) + counts = {s: sum(c["status"] == s for c in self.checks) for s in ("PASS", "FAIL", "WARN", "KNOWN", "SKIP")} + print(f"{self.name}: " + ", ".join(f"{k} {v}" for k, v in counts.items())) + failed = [c for c in self.checks if c["status"] == "FAIL"] + if failed: + raise AssertionError(f"{len(failed)} check(s) failed in {self.name}: " + + "; ".join(c["check"] for c in failed[:20])) + return counts diff --git a/lkbench/compat.py b/lkbench/compat.py new file mode 100644 index 0000000..1fae0f4 --- /dev/null +++ b/lkbench/compat.py @@ -0,0 +1,61 @@ +"""What the installed pylibkriging offers, so that one notebook runs against both +the last release and the development head: a feature missing from the installed +version is recorded as SKIP (with the reason), never as a failure.""" +import functools +import importlib + +import numpy as np + + +def version(): + import pylibkriging as lk + v = getattr(lk, "__version__", None) + if v is None: + try: + from importlib.metadata import version as _v + v = _v("pylibkriging") + except Exception: # noqa: BLE001 + v = "unknown" + return v + + +@functools.lru_cache(None) +def has(feature): + """Probe a feature on a tiny problem (cached).""" + import pylibkriging as lk + X = np.linspace(0, 1, 6).reshape(-1, 1) + y = np.sin(3 * X[:, 0]) + probes = { + "method": lambda: lk.Kriging(y, X, "matern5_2", method="vecchia(3)"), + "noise": lambda: lk.Kriging(y, X, "matern5_2", noise="nugget"), + "WarpKriging": lambda: lk.WarpKriging(y, X, ["none"], "matern5_2"), + "MLPKriging": lambda: lk.MLPKriging(y, X, hidden_dims=[2], d_out=1, kernel="gauss"), + "NestedKriging": lambda: lk.NestedKriging(y, X, "matern5_2", nb_groups=2), + "subsetOfData": lambda: lk.Kriging.subsetOfData(X, n_max=3, method="kmeans", seed=1), + "save": lambda: lk.Kriging(y, X, "matern5_2").save, + "pickle": lambda: __import__("pickle").dumps(lk.Kriging(y, X, "matern5_2")), + "NestedKriging.save": lambda: lk.NestedKriging(y, X, "matern5_2", nb_groups=2).save, + "load(noise)": lambda: _roundtrip(lk, lk.Kriging(y, X, "matern5_2", noise="nugget")), + "sklearn": lambda: importlib.import_module("pylibkriging.sklearn").KrigingRegressor, + } + try: + probes[feature]() + return True + except Exception: # noqa: BLE001 + return False + + +def _roundtrip(lk, k): + import os + import tempfile + f = os.path.join(tempfile.mkdtemp(), "k.json") + k.save(f) + return lk.load(f) + + +def optional(module): + """Import an optional comparison package, or return None.""" + try: + return importlib.import_module(module) + except Exception: # noqa: BLE001 - broken installs count as absent + return None diff --git a/lkbench/design.py b/lkbench/design.py new file mode 100644 index 0000000..e4452f3 --- /dev/null +++ b/lkbench/design.py @@ -0,0 +1,31 @@ +"""Seeded designs of experiments, all on [0,1]^d (scale with ``TestFunction.scale``).""" +import numpy as np +from scipy.stats import qmc + + +def lhs(n, d, seed=0, optimized=True): + """Latin hypercube; ``optimized`` applies scipy's random-CD improvement (space filling).""" + sampler = qmc.LatinHypercube(d=d, seed=seed, optimization="random-cd" if optimized and d > 1 else None) + return sampler.random(n) + + +def sobol(n, d, seed=0): + return qmc.Sobol(d=d, scramble=True, seed=seed).random(n) + + +def uniform(n, d, seed=0): + return np.random.default_rng(seed).random((n, d)) + + +def dataset(tf, n, n_test=1000, seed=0, noise_sd=0.0): + """Training/test sets for test function ``tf`` on the unit hypercube. + + Returns (X, y, Xt, yt) with X in [0,1]^d (the GP sees the unit cube; the + function is evaluated in its native domain). ``noise_sd`` adds Gaussian + noise to y only (yt stays noise-free, it is the target).""" + X = lhs(n, tf.d, seed=seed) + Xt = uniform(n_test, tf.d, seed=10_000 + seed) + y = tf.unit(X) + if noise_sd > 0: + y = y + np.random.default_rng(20_000 + seed).normal(0.0, noise_sd, size=n) + return X, y, Xt, tf.unit(Xt) diff --git a/lkbench/functions.py b/lkbench/functions.py new file mode 100644 index 0000000..c96ff1c --- /dev/null +++ b/lkbench/functions.py @@ -0,0 +1,241 @@ +"""Test functions for surrogate modelling, after the Virtual Library of Simulation +Experiments (Surjanovic & Bingham, https://www.sfu.ca/~ssurjano/). + +Every function takes ``X`` of shape (n, d) in its *native* domain and returns a +length-n vector. ``TESTS`` gathers the metadata (dimension, domain, a reference +point with its known value) used by ``notebooks/python/00_test_functions.ipynb`` +to check the implementations themselves before anything is benchmarked on them. +""" +from dataclasses import dataclass, field + +import numpy as np + + +# --- 1D ------------------------------------------------------------------------- + +def forrester(X): + x = X[:, 0] + return (6 * x - 2) ** 2 * np.sin(12 * x - 4) + + +def gramacy_lee_2012(X): + x = X[:, 0] + return np.sin(10 * np.pi * x) / (2 * x) + (x - 1) ** 4 + + +# --- 2D ------------------------------------------------------------------------- + +def branin(X): + x1, x2 = X[:, 0], X[:, 1] + a, b, c = 1.0, 5.1 / (4 * np.pi ** 2), 5 / np.pi + r, s, t = 6.0, 10.0, 1 / (8 * np.pi) + return a * (x2 - b * x1 ** 2 + c * x1 - r) ** 2 + s * (1 - t) * np.cos(x1) + s + + +def six_hump_camel(X): + x1, x2 = X[:, 0], X[:, 1] + return (4 - 2.1 * x1 ** 2 + x1 ** 4 / 3) * x1 ** 2 + x1 * x2 + (-4 + 4 * x2 ** 2) * x2 ** 2 + + +def goldstein_price(X): + x1, x2 = X[:, 0], X[:, 1] + a = 1 + (x1 + x2 + 1) ** 2 * (19 - 14 * x1 + 3 * x1 ** 2 - 14 * x2 + 6 * x1 * x2 + 3 * x2 ** 2) + b = 30 + (2 * x1 - 3 * x2) ** 2 * (18 - 32 * x1 + 12 * x1 ** 2 + 48 * x2 - 36 * x1 * x2 + 27 * x2 ** 2) + return a * b + + +def log_goldstein_price(X): + """Logarithmic Goldstein-Price (Picheny et al. 2013), on [0,1]^2.""" + Z = 4 * X - 2 + return (np.log(goldstein_price(Z)) - 8.693) / 2.427 + + +def currin_exp(X): + x1, x2 = X[:, 0], X[:, 1] + return (1 - np.exp(-1 / (2 * x2))) * (2300 * x1 ** 3 + 1900 * x1 ** 2 + 2092 * x1 + 60) / ( + 100 * x1 ** 3 + 500 * x1 ** 2 + 4 * x1 + 20) + + +def lim_polynomial(X): + x1, x2 = X[:, 0], X[:, 1] + return (9 + 5 / 2 * x1 - 35 / 2 * x2 + 5 / 2 * x1 * x2 + 19 * x2 ** 2 - 15 / 2 * x1 ** 3 + - 5 / 2 * x1 * x2 ** 2 - 11 / 2 * x2 ** 4 + x1 ** 3 * x2 ** 2) + + +def gramacy_lee_2008(X): + """Non-stationary 2D function x1 exp(-x1^2 - x2^2) on [-2, 6]^2.""" + x1, x2 = X[:, 0], X[:, 1] + return x1 * np.exp(-x1 ** 2 - x2 ** 2) + + +# --- 3D to 5D ------------------------------------------------------------------- + +def ishigami(X, a=7.0, b=0.1): + x1, x2, x3 = X[:, 0], X[:, 1], X[:, 2] + return np.sin(x1) + a * np.sin(x2) ** 2 + b * x3 ** 4 * np.sin(x1) + + +_H3_ALPHA = np.array([1.0, 1.2, 3.0, 3.2]) +_H3_A = np.array([[3, 10, 30], [0.1, 10, 35], [3, 10, 30], [0.1, 10, 35]], float) +_H3_P = 1e-4 * np.array([[3689, 1170, 2673], [4699, 4387, 7470], + [1091, 8732, 5547], [381, 5743, 8828]], float) + + +def hartmann3(X): + diff2 = (X[:, None, :] - _H3_P[None, :, :]) ** 2 + return -np.exp(-np.sum(_H3_A[None, :, :] * diff2, axis=2)) @ _H3_ALPHA + + +def friedman(X): + x1, x2, x3, x4, x5 = (X[:, i] for i in range(5)) + return 10 * np.sin(np.pi * x1 * x2) + 20 * (x3 - 0.5) ** 2 + 10 * x4 + 5 * x5 + + +# --- 6D to 8D ------------------------------------------------------------------- + +_H6_ALPHA = np.array([1.0, 1.2, 3.0, 3.2]) +_H6_A = np.array([[10, 3, 17, 3.5, 1.7, 8], + [0.05, 10, 17, 0.1, 8, 14], + [3, 3.5, 1.7, 10, 17, 8], + [17, 8, 0.05, 10, 0.1, 14]], float) +_H6_P = 1e-4 * np.array([[1312, 1696, 5569, 124, 8283, 5886], + [2329, 4135, 8307, 3736, 1004, 9991], + [2348, 1451, 3522, 2883, 3047, 6650], + [4047, 8828, 8732, 5743, 1091, 381]], float) + + +def hartmann6(X): + diff2 = (X[:, None, :] - _H6_P[None, :, :]) ** 2 + return -np.exp(-np.sum(_H6_A[None, :, :] * diff2, axis=2)) @ _H6_ALPHA + + +def otl_circuit(X): + Rb1, Rb2, Rf, Rc1, Rc2, beta = (X[:, i] for i in range(6)) + Vb1 = 12 * Rb2 / (Rb1 + Rb2) + den = beta * (Rc2 + 9) + Rf + return ((Vb1 + 0.74) * beta * (Rc2 + 9) / den + 11.35 * Rf / den + + 0.74 * Rf * beta * (Rc2 + 9) / (den * Rc1)) + + +def piston(X): + M, S, V0, k, P0, Ta, T0 = (X[:, i] for i in range(7)) + A = P0 * S + 19.62 * M - k * V0 / S + V = S / (2 * k) * (np.sqrt(A ** 2 + 4 * k * P0 * V0 * Ta / T0) - A) + return 2 * np.pi * np.sqrt(M / (k + S ** 2 * P0 * V0 * Ta / (T0 * V ** 2))) + + +def borehole(X): + rw, r, Tu, Hu, Tl, Hl, L, Kw = (X[:, i] for i in range(8)) + lnr = np.log(r / rw) + return 2 * np.pi * Tu * (Hu - Hl) / (lnr * (1 + 2 * L * Tu / (lnr * rw ** 2 * Kw) + Tu / Tl)) + + +def robot_arm(X): + theta, L = X[:, :4], X[:, 4:] + angles = np.cumsum(theta, axis=1) + u = np.sum(L * np.cos(angles), axis=1) + v = np.sum(L * np.sin(angles), axis=1) + return np.sqrt(u ** 2 + v ** 2) + + +def dette_pepelyshev(X): + x = X + s = 0.0 + for i in range(4, 9): # i = 4..8, inner sum over j = 3..i (1-based) + s = s + i * np.log(1 + np.sum(x[:, 2:i], axis=1)) + return (4 * (x[:, 0] - 2 + 8 * x[:, 1] - 8 * x[:, 1] ** 2) ** 2 + (3 - 4 * x[:, 1]) ** 2 + + 16 * np.sqrt(x[:, 2] + 1) * (2 * x[:, 2] - 1) ** 2 + s) + + +# --- 10D and more --------------------------------------------------------------- + +def wing_weight(X): + Sw, Wfw, A, Lam, q, lam, tc, Nz, Wdg, Wp = (X[:, i] for i in range(10)) + Lam = np.deg2rad(Lam) + return (0.036 * Sw ** 0.758 * Wfw ** 0.0035 * (A / np.cos(Lam) ** 2) ** 0.6 * q ** 0.006 + * lam ** 0.04 * (100 * tc / np.cos(Lam)) ** -0.3 * (Nz * Wdg) ** 0.49 + Sw * Wp) + + +def welch(X): + x = [None] + [X[:, i] for i in range(20)] # 1-based, as in the reference + return (5 * x[12] / (1 + x[1]) + 5 * (x[4] - x[20]) ** 2 + x[5] + 40 * x[19] ** 3 - 5 * x[19] + + 0.05 * x[2] + 0.08 * x[3] - 0.03 * x[6] + 0.03 * x[7] - 0.09 * x[9] - 0.01 * x[10] + - 0.07 * x[11] + 0.25 * x[13] ** 2 - 0.04 * x[14] + 0.06 * x[15] - 0.01 * x[17] + - 0.03 * x[18]) + + +def ackley(X, a=20.0, b=0.2, c=2 * np.pi): + d = X.shape[1] + return (-a * np.exp(-b * np.sqrt(np.sum(X ** 2, axis=1) / d)) + - np.exp(np.sum(np.cos(c * X), axis=1) / d) + a + np.e) + + +@dataclass +class TestFunction: + name: str + f: callable + lower: np.ndarray + upper: np.ndarray + # (x, f(x)) pairs taken from the reference pages (global optima, nominal values) + known: list = field(default_factory=list) + tags: tuple = () + + @property + def d(self): + return len(self.lower) + + def __call__(self, X): + return self.f(np.atleast_2d(X)) + + def scale(self, U): + """Map points of [0,1]^d to the native domain.""" + return self.lower + np.asarray(U) * (self.upper - self.lower) + + def unit(self, U): + """The function seen on [0,1]^d.""" + return self.f(self.scale(np.atleast_2d(U))) + + +def _box(*bounds): + b = np.array(bounds, float) + return b[:, 0], b[:, 1] + + +def _tf(name, f, bounds, known=(), tags=()): + lo, up = _box(*bounds) + return TestFunction(name, f, lo, up, [(np.array(x, float), v) for x, v in known], tuple(tags)) + + +TESTS = {t.name: t for t in [ + _tf("forrester", forrester, [(0, 1)], [([0.757249], -6.02074)], ["1d"]), + _tf("gramacy_lee_2012", gramacy_lee_2012, [(0.5, 2.5)], [([0.548563], -0.869011)], ["1d", "nonstationary"]), + _tf("branin", branin, [(-5, 10), (0, 15)], + [([-np.pi, 12.275], 0.397887), ([np.pi, 2.275], 0.397887), ([9.42478, 2.475], 0.397887)], ["2d"]), + _tf("six_hump_camel", six_hump_camel, [(-3, 3), (-2, 2)], + [([0.0898, -0.7126], -1.0316), ([-0.0898, 0.7126], -1.0316)], ["2d"]), + _tf("goldstein_price", goldstein_price, [(-2, 2), (-2, 2)], [([0, -1], 3.0)], ["2d", "large-range"]), + _tf("log_goldstein_price", log_goldstein_price, [(0, 1), (0, 1)], + [([0.5, 0.25], (np.log(3.0) - 8.693) / 2.427)], ["2d"]), + _tf("currin_exp", currin_exp, [(0, 1), (1e-6, 1)], [], ["2d"]), + _tf("lim_polynomial", lim_polynomial, [(0, 1), (0, 1)], [([0, 0], 9.0)], ["2d", "polynomial"]), + _tf("gramacy_lee_2008", gramacy_lee_2008, [(-2, 6), (-2, 6)], + [([-1 / np.sqrt(2), 0], -np.exp(-0.5) / np.sqrt(2))], ["2d", "nonstationary"]), + _tf("ishigami", ishigami, [(-np.pi, np.pi)] * 3, [([0, 0, 0], 0.0), ([0, np.pi / 2, 0], 7.0)], ["3d"]), + _tf("hartmann3", hartmann3, [(0, 1)] * 3, [([0.114614, 0.555649, 0.852547], -3.86278)], ["3d"]), + _tf("friedman", friedman, [(0, 1)] * 5, [([0, 0, 0.5, 0, 0], 0.0)], ["5d"]), + _tf("hartmann6", hartmann6, [(0, 1)] * 6, + [([0.20169, 0.150011, 0.476874, 0.275332, 0.311652, 0.6573], -3.32237)], ["6d"]), + _tf("otl_circuit", otl_circuit, [(50, 150), (25, 70), (0.5, 3), (1.2, 2.5), (0.25, 1.2), (50, 300)], + [], ["6d", "engineering"]), + _tf("piston", piston, [(30, 60), (0.005, 0.020), (0.002, 0.010), (1000, 5000), (90000, 110000), + (290, 296), (340, 360)], [], ["7d", "engineering"]), + _tf("borehole", borehole, [(0.05, 0.15), (100, 50000), (63070, 115600), (990, 1110), + (63.1, 116), (700, 820), (1120, 1680), (9855, 12045)], [], ["8d", "engineering"]), + _tf("robot_arm", robot_arm, [(0, 2 * np.pi)] * 4 + [(0, 1)] * 4, + [([0, 0, 0, 0, 1, 1, 1, 1], 4.0), ([0, np.pi, 0, 0, 1, 1, 1, 1], 2.0)], ["8d", "oscillating"]), + _tf("dette_pepelyshev", dette_pepelyshev, [(0, 1)] * 8, [], ["8d", "curved"]), + _tf("wing_weight", wing_weight, [(150, 200), (220, 300), (6, 10), (-10, 10), (16, 45), (0.5, 1), + (0.08, 0.18), (2.5, 6), (1700, 2500), (0.025, 0.08)], [], ["10d", "engineering"]), + _tf("welch", welch, [(-0.5, 0.5)] * 20, [([0] * 20, 0.0)], ["20d", "screening"]), + _tf("ackley4", ackley, [(-2, 2)] * 4, [([0, 0, 0, 0], 0.0)], ["4d", "multimodal"]), +]} diff --git a/lkbench/metrics.py b/lkbench/metrics.py new file mode 100644 index 0000000..aa5aa62 --- /dev/null +++ b/lkbench/metrics.py @@ -0,0 +1,50 @@ +"""Accuracy and calibration metrics for probabilistic predictions.""" +import numpy as np +from scipy.stats import norm + + +def rmse(y, m): + return float(np.sqrt(np.mean((np.asarray(y) - np.asarray(m)) ** 2))) + + +def q2(y, m): + """Predictivity coefficient 1 - SSE/SST on a test set (1 is perfect).""" + y, m = np.asarray(y), np.asarray(m) + return float(1 - np.sum((y - m) ** 2) / np.sum((y - y.mean()) ** 2)) + + +def nlpd(y, m, s): + """Mean negative log predictive density under N(m, s^2).""" + s = np.maximum(np.asarray(s), 1e-12) + return float(np.mean(0.5 * np.log(2 * np.pi * s ** 2) + 0.5 * ((np.asarray(y) - m) / s) ** 2)) + + +def coverage(y, m, s, level=0.95): + """Fraction of test values inside the central ``level`` prediction interval.""" + z = norm.ppf(0.5 + level / 2) + return float(np.mean(np.abs(np.asarray(y) - m) <= z * np.asarray(s))) + + +def crps(y, m, s): + """Mean continuous ranked probability score of N(m, s^2) (lower is better).""" + s = np.maximum(np.asarray(s), 1e-12) + z = (np.asarray(y) - m) / s + return float(np.mean(s * (z * (2 * norm.cdf(z) - 1) + 2 * norm.pdf(z) - 1 / np.sqrt(np.pi)))) + + +def all_metrics(y, m, s=None): + out = {"rmse": rmse(y, m), "q2": q2(y, m)} + if s is not None: + out.update(nlpd=nlpd(y, m, s), coverage95=coverage(y, m, s), crps=crps(y, m, s)) + return out + + +def finite_diff_grad(f, x, h=1e-6): + """Central finite-difference gradient of a scalar function, relative step.""" + x = np.asarray(x, float) + g = np.zeros_like(x) + for i in range(x.size): + e = np.zeros_like(x) + e[i] = h * max(1.0, abs(x[i])) + g[i] = (f(x + e) - f(x - e)) / (2 * e[i]) + return g diff --git a/notebooks/julia/01_cross_binding.ipynb b/notebooks/julia/01_cross_binding.ipynb new file mode 100644 index 0000000..62b6668 --- /dev/null +++ b/notebooks/julia/01_cross_binding.ipynb @@ -0,0 +1,150 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "c000", + "metadata": {}, + "source": [ + "# Julia 01 — Cross-binding consistency, and a comparison with GaussianProcesses.jl\n", + "\n", + "1. Same protocol as `notebooks/python/11_cross_binding.ipynb` (see there), on\n", + " the shared datasets of `data/`; writes `results/xbind_julia.csv`, compared\n", + " with the Python values by `tools/report.py`.\n", + "2. Default fits against [GaussianProcesses.jl](https://github.com/STOR-i/GaussianProcesses.jl)\n", + " (Matérn 5/2 ARD, constant mean, MLE), when it is installed (warn-level).\n", + "\n", + "The module is `JLibKriging` (the packaged binding, [JLibKriging.jl](https://github.com/libKriging/JLibKriging.jl)),\n", + "imported under a short name so that its exported accessors (`X`, `y`, `theta`, ...)\n", + "do not clash with the variables of the notebook." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c001", + "metadata": {}, + "outputs": [], + "source": [ + "import JLibKriging as LK\n", + "include(\"checks.jl\")\n", + "s = Suite(\"jl01_cross_binding\")\n", + "rows = Vector{Vector{String}}()\n", + "record!(dataset, quantity, values) =\n", + " for (i, v) in enumerate(vec(Float64.(collect(values))))\n", + " push!(rows, [\"julia\", dataset, quantity, string(i), @sprintf(\"%.17g\", v)])\n", + " end" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c002", + "metadata": {}, + "outputs": [], + "source": [ + "for name in (\"branin\", \"hartmann3\", \"borehole\")\n", + " Xd = read_data(name, \"X\"); yd = vec(read_data(name, \"y\")); Xt = read_data(name, \"Xt\")[1:50, :]\n", + " d = size(Xd, 2)\n", + " theta0 = fill(0.3, d)\n", + " kf = LK.Kriging(yd, Xd, \"matern5_2\"; regmodel=\"constant\", optim=\"none\", theta=theta0, is_theta_estim=false)\n", + " record!(name, \"fixed_sigma2\", LK.sigma2(kf))\n", + " record!(name, \"fixed_beta\", LK.beta(kf))\n", + " record!(name, \"fixed_ll\", LK.log_likelihood(kf))\n", + " record!(name, \"fixed_loo\", LK.leave_one_out(kf))\n", + " p = LK.predict(kf, Xt; return_stdev=true)\n", + " record!(name, \"fixed_mean\", p.mean)\n", + " record!(name, \"fixed_stdev\", p.stdev)\n", + " llf = LK.log_likelihood_fun(kf, theta0; return_grad=true)\n", + " record!(name, \"llfun\", llf.ll)\n", + " record!(name, \"llfun_grad\", llf.grad)\n", + " check_close!(s, \"$name: log_likelihood() == log_likelihood_fun(theta0)\", LK.log_likelihood(kf), llf.ll; rtol=1e-10)\n", + " k = LK.Kriging(yd, Xd, \"matern5_2\")\n", + " record!(name, \"fit_theta\", LK.theta(k))\n", + " record!(name, \"fit_ll\", LK.log_likelihood(k))\n", + " record!(name, \"fit_mean\", LK.predict(k, Xt; return_stdev=false).mean)\n", + " record!(name, \"sim\", LK.simulate(kf, 3, 123, Xt[1:5, :]))\n", + "end\n", + "open(joinpath(RESULTS_DIR, \"xbind_julia.csv\"), \"w\") do io\n", + " println(io, \"lang,dataset,quantity,index,value\")\n", + " for r in rows; println(io, join(r, \",\")); end\n", + "end\n", + "println(length(rows), \" values written\")" + ] + }, + { + "cell_type": "markdown", + "id": "c003", + "metadata": {}, + "source": [ + "## Default fits vs GaussianProcesses.jl" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c004", + "metadata": {}, + "outputs": [], + "source": [ + "have_gp = try\n", + " @eval import GaussianProcesses as GP\n", + " @eval import Optim\n", + " true\n", + "catch\n", + " false\n", + "end\n", + "for name in (\"branin\", \"hartmann3\", \"borehole\")\n", + " Xd = read_data(name, \"X\"); yd = vec(read_data(name, \"y\")); Xt = read_data(name, \"Xt\"); yt = vec(read_data(name, \"yt\"))\n", + " t = @elapsed k = LK.Kriging(yd, Xd, \"matern5_2\")\n", + " q_lk = q2(yt, LK.predict(k, Xt; return_stdev=false).mean)\n", + " add_metric!(s, name, \"JLibKriging\"; q2=q_lk, fit_s=t)\n", + " t = @elapsed k10 = LK.Kriging(yd, Xd, \"matern5_2\"; optim=\"BFGS10\")\n", + " q_lk10 = q2(yt, LK.predict(k10, Xt; return_stdev=false).mean)\n", + " add_metric!(s, name, \"JLibKriging BFGS10\"; q2=q_lk10, fit_s=t)\n", + " add_check!(s, \"$name: Q2(JLibKriging BFGS10) > 0.9\", q_lk10 > 0.9; value=q_lk10, expected=\"> 0.9\")\n", + " if have_gp\n", + " try\n", + " d = size(Xd, 2)\n", + " t = @elapsed begin\n", + " gp = GP.GP(permutedims(Xd), yd, GP.MeanConst(sum(yd) / length(yd)),\n", + " GP.Mat52Ard(zeros(d), log(sum((yd .- sum(yd) / length(yd)) .^ 2) / length(yd)) / 2), -8.0)\n", + " GP.optimize!(gp; noise=false)\n", + " end\n", + " mg, _ = GP.predict_f(gp, permutedims(Xt))\n", + " q_gp = q2(yt, mg)\n", + " add_metric!(s, name, \"GaussianProcesses.jl\"; q2=q_gp, fit_s=t)\n", + " add_check!(s, \"$name: Q2(JLibKriging) - Q2(GaussianProcesses.jl)\", q_lk - q_gp > -0.02;\n", + " value=q_lk - q_gp, expected=\"> -0.02\", severity=\"warn\")\n", + " catch e\n", + " add_check!(s, \"$name: GaussianProcesses.jl fit\", false; value=string(typeof(e)), severity=\"warn\")\n", + " end\n", + " end\n", + "end\n", + "have_gp || add_check!(s, \"GaussianProcesses.jl available\", false; value=\"absent\", severity=\"warn\")\n", + "foreach(r -> println(join(r[3:6], \" \")), s.metrics)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c005", + "metadata": {}, + "outputs": [], + "source": [ + "finalize!(s)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Julia 1.11", + "language": "julia", + "name": "julia-1.11" + }, + "language_info": { + "name": "julia" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/julia/checks.jl b/notebooks/julia/checks.jl new file mode 100644 index 0000000..88c1645 --- /dev/null +++ b/notebooks/julia/checks.jl @@ -0,0 +1,75 @@ +# Same CSV outputs as lkbench/checks.py (see there), for the Julia notebooks. +using DelimitedFiles, Printf + +const RESULTS_DIR = get(ENV, "LKBENCH_RESULTS", joinpath("..", "..", "results")) +mkpath(RESULTS_DIR) + +function read_known() + f = joinpath("..", "..", "known_issues.csv") + isfile(f) || return Dict{Tuple{String,String},String}() + d = Dict{Tuple{String,String},String}() + for l in readlines(f)[2:end] + p = split(l, ","; limit=3) + length(p) == 3 && (d[(p[1], p[2])] = p[3]) + end + d +end + +mutable struct Suite + name::String + checks::Vector{Vector{String}} + metrics::Vector{Vector{String}} + t0::Float64 + known::Dict{Tuple{String,String},String} +end +Suite(name) = Suite(name, [], [], time(), read_known()) + +fmt(v::Real) = @sprintf("%.6g", v) +fmt(v) = string(v) + +function add_check!(s::Suite, check, ok; value="", expected="", detail="", severity="error") + status = ok ? "PASS" : (severity == "error" ? "FAIL" : "WARN") + if !ok && haskey(s.known, (s.name, check)) + status = "KNOWN"; detail = s.known[(s.name, check)] * " | " * detail + end + ok || println("[$status] $check: value=$(fmt(value)) expected $expected $detail") + push!(s.checks, [s.name, "julia", check, severity, status, fmt(value), expected, detail]) + ok +end + +function check_close!(s::Suite, check, a, b; rtol=1e-6, atol=0.0, detail="", severity="error") + a = vec(Float64.(collect(a))); b = vec(Float64.(collect(b))) + length(a) == length(b) || return add_check!(s, check, false; value="length $(length(a))", + expected="length $(length(b))", detail, severity) + gap = abs.(a .- b) + ok = all(isfinite, a) && all(gap .<= atol .+ rtol .* abs.(b)) + rel = maximum(gap ./ max.(abs.(b), 1e-300)) + add_check!(s, check, ok; value=rel, expected="rtol $rtol, atol $atol", detail, severity) +end + +add_metric!(s::Suite, case, package; kw...) = + for (k, v) in kw; push!(s.metrics, [s.name, "julia", case, package, string(k), fmt(v)]); end + +csvq(x) = "\"" * replace(string(x), "\"" => "\"\"") * "\"" + +function write_rows(f, header, rows) + open(f, "w") do io + println(io, join(header, ",")) + for r in rows; println(io, join(csvq.(r), ",")); end + end +end + +function finalize!(s::Suite) + add_metric!(s, "_suite", "_"; elapsed_s=time() - s.t0) + write_rows(joinpath(RESULTS_DIR, s.name * ".checks.csv"), + ["suite", "lang", "check", "severity", "status", "value", "expected", "detail"], s.checks) + write_rows(joinpath(RESULTS_DIR, s.name * ".metrics.csv"), + ["suite", "lang", "case", "package", "metric", "value"], s.metrics) + st = [c[5] for c in s.checks] + println(s.name, ": ", join(["$k $(count(==(k), st))" for k in ("PASS", "FAIL", "WARN", "KNOWN", "SKIP")], ", ")) + failed = [c[3] for c in s.checks if c[5] == "FAIL"] + isempty(failed) || error("$(length(failed)) check(s) failed in $(s.name): " * join(first(failed, 20), "; ")) +end + +read_data(name, stem) = readdlm(joinpath("..", "..", "data", "$(name)_$(stem).csv"), ',', Float64) +q2(y, m) = 1 - sum((y .- m) .^ 2) / sum((y .- sum(y) / length(y)) .^ 2) diff --git a/notebooks/julia/src/01_cross_binding.jl b/notebooks/julia/src/01_cross_binding.jl new file mode 100644 index 0000000..17740e4 --- /dev/null +++ b/notebooks/julia/src/01_cross_binding.jl @@ -0,0 +1,95 @@ +# %% [markdown] +# # Julia 01 — Cross-binding consistency, and a comparison with GaussianProcesses.jl +# +# 1. Same protocol as `notebooks/python/11_cross_binding.ipynb` (see there), on +# the shared datasets of `data/`; writes `results/xbind_julia.csv`, compared +# with the Python values by `tools/report.py`. +# 2. Default fits against [GaussianProcesses.jl](https://github.com/STOR-i/GaussianProcesses.jl) +# (Matérn 5/2 ARD, constant mean, MLE), when it is installed (warn-level). +# +# The module is `JLibKriging` (the packaged binding, [JLibKriging.jl](https://github.com/libKriging/JLibKriging.jl)), +# imported under a short name so that its exported accessors (`X`, `y`, `theta`, ...) +# do not clash with the variables of the notebook. + +# %% +import JLibKriging as LK +include("checks.jl") +s = Suite("jl01_cross_binding") +rows = Vector{Vector{String}}() +record!(dataset, quantity, values) = + for (i, v) in enumerate(vec(Float64.(collect(values)))) + push!(rows, ["julia", dataset, quantity, string(i), @sprintf("%.17g", v)]) + end + +# %% +for name in ("branin", "hartmann3", "borehole") + Xd = read_data(name, "X"); yd = vec(read_data(name, "y")); Xt = read_data(name, "Xt")[1:50, :] + d = size(Xd, 2) + theta0 = fill(0.3, d) + kf = LK.Kriging(yd, Xd, "matern5_2"; regmodel="constant", optim="none", theta=theta0, is_theta_estim=false) + record!(name, "fixed_sigma2", LK.sigma2(kf)) + record!(name, "fixed_beta", LK.beta(kf)) + record!(name, "fixed_ll", LK.log_likelihood(kf)) + record!(name, "fixed_loo", LK.leave_one_out(kf)) + p = LK.predict(kf, Xt; return_stdev=true) + record!(name, "fixed_mean", p.mean) + record!(name, "fixed_stdev", p.stdev) + llf = LK.log_likelihood_fun(kf, theta0; return_grad=true) + record!(name, "llfun", llf.ll) + record!(name, "llfun_grad", llf.grad) + check_close!(s, "$name: log_likelihood() == log_likelihood_fun(theta0)", LK.log_likelihood(kf), llf.ll; rtol=1e-10) + k = LK.Kriging(yd, Xd, "matern5_2") + record!(name, "fit_theta", LK.theta(k)) + record!(name, "fit_ll", LK.log_likelihood(k)) + record!(name, "fit_mean", LK.predict(k, Xt; return_stdev=false).mean) + record!(name, "sim", LK.simulate(kf, 3, 123, Xt[1:5, :])) +end +open(joinpath(RESULTS_DIR, "xbind_julia.csv"), "w") do io + println(io, "lang,dataset,quantity,index,value") + for r in rows; println(io, join(r, ",")); end +end +println(length(rows), " values written") + +# %% [markdown] +# ## Default fits vs GaussianProcesses.jl + +# %% +have_gp = try + @eval import GaussianProcesses as GP + @eval import Optim + true +catch + false +end +for name in ("branin", "hartmann3", "borehole") + Xd = read_data(name, "X"); yd = vec(read_data(name, "y")); Xt = read_data(name, "Xt"); yt = vec(read_data(name, "yt")) + t = @elapsed k = LK.Kriging(yd, Xd, "matern5_2") + q_lk = q2(yt, LK.predict(k, Xt; return_stdev=false).mean) + add_metric!(s, name, "JLibKriging"; q2=q_lk, fit_s=t) + t = @elapsed k10 = LK.Kriging(yd, Xd, "matern5_2"; optim="BFGS10") + q_lk10 = q2(yt, LK.predict(k10, Xt; return_stdev=false).mean) + add_metric!(s, name, "JLibKriging BFGS10"; q2=q_lk10, fit_s=t) + add_check!(s, "$name: Q2(JLibKriging BFGS10) > 0.9", q_lk10 > 0.9; value=q_lk10, expected="> 0.9") + if have_gp + try + d = size(Xd, 2) + t = @elapsed begin + gp = GP.GP(permutedims(Xd), yd, GP.MeanConst(sum(yd) / length(yd)), + GP.Mat52Ard(zeros(d), log(sum((yd .- sum(yd) / length(yd)) .^ 2) / length(yd)) / 2), -8.0) + GP.optimize!(gp; noise=false) + end + mg, _ = GP.predict_f(gp, permutedims(Xt)) + q_gp = q2(yt, mg) + add_metric!(s, name, "GaussianProcesses.jl"; q2=q_gp, fit_s=t) + add_check!(s, "$name: Q2(JLibKriging) - Q2(GaussianProcesses.jl)", q_lk - q_gp > -0.02; + value=q_lk - q_gp, expected="> -0.02", severity="warn") + catch e + add_check!(s, "$name: GaussianProcesses.jl fit", false; value=string(typeof(e)), severity="warn") + end + end +end +have_gp || add_check!(s, "GaussianProcesses.jl available", false; value="absent", severity="warn") +foreach(r -> println(join(r[3:6], " ")), s.metrics) + +# %% +finalize!(s) diff --git a/notebooks/octave/01_cross_binding.ipynb b/notebooks/octave/01_cross_binding.ipynb new file mode 100644 index 0000000..ea5e878 --- /dev/null +++ b/notebooks/octave/01_cross_binding.ipynb @@ -0,0 +1,102 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "c000", + "metadata": {}, + "source": [ + "# Octave 01 — Cross-binding consistency: the Octave side\n", + "\n", + "Same protocol as `notebooks/python/11_cross_binding.ipynb` (see there), on the\n", + "shared datasets of `data/`; writes `results/xbind_octave.csv`, compared with\n", + "the Python values by `tools/report.py`. mLibKriging must be on the path\n", + "(`OCTAVE_PATH` or `addpath`)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c001", + "metadata": {}, + "outputs": [], + "source": [ + "s = lkb_suite(\"oct01_cross_binding\");\n", + "out = {};\n", + "function out = put(out, dataset, quantity, values)\n", + " v = double(values(:));\n", + " for i = 1:numel(v)\n", + " out(end + 1, :) = {\"octave\", dataset, quantity, i, sprintf(\"%.17g\", v(i))};\n", + " end\n", + "end" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c002", + "metadata": {}, + "outputs": [], + "source": [ + "for name = {\"branin\", \"hartmann3\", \"borehole\"}\n", + " name = name{1};\n", + " X = read_data(name, \"X\"); y = read_data(name, \"y\"); Xt = read_data(name, \"Xt\")(1:50, :);\n", + " d = columns(X);\n", + " theta0 = 0.3 * ones(1, d);\n", + " kf = Kriging(y, X, \"matern5_2\", \"constant\", false, \"none\", \"LL\", Params(\"theta\", theta0, \"is_theta_estim\", false));\n", + " v = kf.sigma2(); out = put(out, name, \"fixed_sigma2\", v);\n", + " v = kf.beta(); out = put(out, name, \"fixed_beta\", v);\n", + " ll = kf.logLikelihood();\n", + " out = put(out, name, \"fixed_ll\", ll);\n", + " loo = kf.leaveOneOut();\n", + " out = put(out, name, \"fixed_loo\", loo);\n", + " [m, sd] = kf.predict(Xt, true, false, false);\n", + " out = put(out, name, \"fixed_mean\", m);\n", + " out = put(out, name, \"fixed_stdev\", sd);\n", + " [llf, grad] = kf.logLikelihoodFun(theta0', true, false);\n", + " out = put(out, name, \"llfun\", llf);\n", + " out = put(out, name, \"llfun_grad\", grad);\n", + " s = lkb_close(s, [name \": logLikelihood() == logLikelihoodFun(theta0)\"], ll, llf, 1e-10);\n", + " k = Kriging(y, X, \"matern5_2\");\n", + " v = k.theta(); out = put(out, name, \"fit_theta\", v);\n", + " llk = k.logLikelihood();\n", + " out = put(out, name, \"fit_ll\", llk);\n", + " [mk, ~] = k.predict(Xt, false, false, false);\n", + " out = put(out, name, \"fit_mean\", mk);\n", + " S = kf.simulate(int32(3), int32(123), Xt(1:5, :), false);\n", + " out = put(out, name, \"sim\", S);\n", + "end\n", + "d = getenv(\"LKBENCH_RESULTS\"); if isempty(d), d = fullfile(\"..\", \"..\", \"results\"); end\n", + "if ~exist(d, \"dir\"), mkdir(d); end\n", + "fid = fopen(fullfile(d, \"xbind_octave.csv\"), \"w\");\n", + "fprintf(fid, \"lang,dataset,quantity,index,value\\n\");\n", + "for i = 1:size(out, 1)\n", + " fprintf(fid, \"%s,%s,%s,%d,%s\\n\", out{i, :});\n", + "end\n", + "fclose(fid);\n", + "printf(\"%d values written\\n\", size(out, 1));" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c003", + "metadata": {}, + "outputs": [], + "source": [ + "lkb_finalize(s);" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Octave", + "language": "octave", + "name": "octave" + }, + "language_info": { + "name": "octave" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/octave/02_vs_stk.ipynb b/notebooks/octave/02_vs_stk.ipynb new file mode 100644 index 0000000..ee92492 --- /dev/null +++ b/notebooks/octave/02_vs_stk.ipynb @@ -0,0 +1,145 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "c000", + "metadata": {}, + "source": [ + "# Octave 02 — mLibKriging vs STK\n", + "\n", + "[STK](https://github.com/stk-kriging/stk) (Small Toolbox for Kriging) is the\n", + "reference kriging toolbox for Octave/MATLAB.\n", + "\n", + "1. **Same hyperparameters → same numbers** (error-level). STK's anisotropic\n", + " Gaussian covariance is σ² exp(−‖(x−x')/ρ‖²) and libKriging's `gauss` is\n", + " σ² exp(−½ Σ ((xⱼ−x'ⱼ)/θⱼ)²): the same kernel with ρ = θ √2. With a constant\n", + " mean (ordinary/universal kriging, β estimated by GLS in both), the\n", + " predicted mean and variance at fixed (θ, σ²) must coincide. (STK's Matérn\n", + " covariances are radial, libKriging's are product-form: not comparable\n", + " hyperparameter by hyperparameter, hence the Gaussian kernel here.)\n", + "2. **Default fits** on the shared datasets of `data/`: Matérn 5/2, each\n", + " toolbox's own estimation (warn-level comparison of Q²).\n", + "\n", + "STK's location: the `STK_PATH` environment variable (skipped if unset)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c001", + "metadata": {}, + "outputs": [], + "source": [ + "s = lkb_suite(\"oct02_vs_stk\");\n", + "stk_path = getenv(\"STK_PATH\");\n", + "have_stk = ~isempty(stk_path) && exist(fullfile(stk_path, \"stk_init.m\"), \"file\");\n", + "if have_stk\n", + " run(fullfile(stk_path, \"stk_init.m\"));\n", + " printf(\"STK %s\\n\", stk_version());\n", + "else\n", + " s = lkb_check(s, \"STK available\", false, \"absent\", \"STK_PATH\", \"STK not found: comparison skipped\", \"warn\");\n", + "end\n", + "q2 = @(y, m) 1 - sum((y - m).^2) / sum((y - mean(y)).^2);\n", + "datasets = {\"branin\", \"hartmann3\", \"borehole\"};" + ] + }, + { + "cell_type": "markdown", + "id": "c002", + "metadata": {}, + "source": [ + "## 1. Same hyperparameters, same numbers" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c003", + "metadata": {}, + "outputs": [], + "source": [ + "if have_stk\n", + " for i = 1:numel(datasets)\n", + " name = datasets{i};\n", + " X = read_data(name, \"X\"); y = read_data(name, \"y\"); Xt = read_data(name, \"Xt\");\n", + " d = columns(X);\n", + " theta = linspace(0.2, 0.6, d);\n", + " k = Kriging(y, X, \"gauss\", \"constant\", false, \"none\", \"LL\", Params(\"theta\", theta, \"is_theta_estim\", false));\n", + " sigma2 = k.sigma2();\n", + " [m_lk, s_lk] = k.predict(Xt, true, false, false);\n", + " model = stk_model(@stk_gausscov_aniso, d);\n", + " model.param = [log(sigma2); -log(theta(:) * sqrt(2))];\n", + " model.lm = stk_lm_constant();\n", + " zp = stk_predict(model, X, y, Xt);\n", + " s = lkb_close(s, [name \": gauss UK mean == STK\"], m_lk, zp.mean, 1e-6, 1e-8 * std(y));\n", + " s = lkb_close(s, [name \": gauss UK variance == STK\"], s_lk.^2, zp.var, 1e-4, 1e-8 * sigma2);\n", + " end\n", + "end" + ] + }, + { + "cell_type": "markdown", + "id": "c004", + "metadata": {}, + "source": [ + "## 2. Default fits" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c005", + "metadata": {}, + "outputs": [], + "source": [ + "for i = 1:numel(datasets)\n", + " name = datasets{i};\n", + " X = read_data(name, \"X\"); y = read_data(name, \"y\"); Xt = read_data(name, \"Xt\"); yt = read_data(name, \"yt\");\n", + " d = columns(X);\n", + " tic; k = Kriging(y, X, \"matern5_2\"); t = toc;\n", + " [m, ~] = k.predict(Xt, false, false, false);\n", + " q_lk = q2(yt, m);\n", + " s = lkb_metric(s, name, \"mLibKriging\", \"q2\", q_lk); s = lkb_metric(s, name, \"mLibKriging\", \"fit_s\", t);\n", + " tic; k10 = Kriging(y, X, \"matern5_2\", \"constant\", false, \"BFGS10\", \"LL\"); t = toc;\n", + " [m, ~] = k10.predict(Xt, false, false, false);\n", + " q_lk10 = q2(yt, m);\n", + " s = lkb_metric(s, name, \"mLibKriging BFGS10\", \"q2\", q_lk10); s = lkb_metric(s, name, \"mLibKriging BFGS10\", \"fit_s\", t);\n", + " s = lkb_check(s, [name \": Q2(mLibKriging BFGS10) > 0.9\"], q_lk10 > 0.9, q_lk10, \"> 0.9\");\n", + " if have_stk\n", + " tic;\n", + " model = stk_model(@stk_materncov52_aniso, d);\n", + " model = stk_param_estim(model, X, y); % (STK >= 2.8 returns the model)\n", + " zp = stk_predict(model, X, y, Xt);\n", + " t = toc;\n", + " q_stk = q2(yt, zp.mean);\n", + " s = lkb_metric(s, name, \"STK\", \"q2\", q_stk); s = lkb_metric(s, name, \"STK\", \"fit_s\", t);\n", + " s = lkb_check(s, [name \": Q2(mLibKriging) - Q2(STK)\"], q_lk - q_stk > -0.02, q_lk - q_stk, \"> -0.02\", \"\", \"warn\");\n", + " end\n", + "end\n", + "disp(s.metrics(:, 3:6))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c006", + "metadata": {}, + "outputs": [], + "source": [ + "lkb_finalize(s);" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Octave", + "language": "octave", + "name": "octave" + }, + "language_info": { + "name": "octave" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/octave/lkb_check.m b/notebooks/octave/lkb_check.m new file mode 100644 index 0000000..33c30e5 --- /dev/null +++ b/notebooks/octave/lkb_check.m @@ -0,0 +1,14 @@ +function s = lkb_check(s, check, ok, value, expected, detail, severity) + if nargin < 4, value = ""; end + if nargin < 5, expected = ""; end + if nargin < 6, detail = ""; end + if nargin < 7, severity = "error"; end + if ok, status = "PASS"; elseif strcmp(severity, "error"), status = "FAIL"; else, status = "WARN"; end + if ~ok && ~isempty(s.known) + hit = strcmp(s.known(:, 1), s.name) & strcmp(s.known(:, 2), check); + if any(hit), status = "KNOWN"; detail = [s.known{find(hit, 1), 3} " | " detail]; end + end + if isnumeric(value) || islogical(value), value = sprintf("%.6g", double(value)); end + if ~ok, printf("[%s] %s: value=%s expected %s %s\n", status, check, value, expected, detail); end + s.checks(end + 1, :) = {s.name, "octave", check, severity, status, value, expected, detail}; +end diff --git a/notebooks/octave/lkb_close.m b/notebooks/octave/lkb_close.m new file mode 100644 index 0000000..e7172a1 --- /dev/null +++ b/notebooks/octave/lkb_close.m @@ -0,0 +1,15 @@ +function s = lkb_close(s, check, a, b, rtol, atol, detail, severity) + if nargin < 5, rtol = 1e-6; end + if nargin < 6, atol = 0; end + if nargin < 7, detail = ""; end + if nargin < 8, severity = "error"; end + a = double(a(:)); b = double(b(:)); + if numel(a) ~= numel(b) + s = lkb_check(s, check, false, sprintf("length %d", numel(a)), sprintf("length %d", numel(b)), detail, severity); + return; + end + gap = abs(a - b); + ok = all(isfinite(a)) && all(gap <= atol + rtol * abs(b)); + rel = max(gap ./ max(abs(b), 1e-300)); + s = lkb_check(s, check, ok, rel, sprintf("rtol %g, atol %g", rtol, atol), detail, severity); +end diff --git a/notebooks/octave/lkb_finalize.m b/notebooks/octave/lkb_finalize.m new file mode 100644 index 0000000..f878779 --- /dev/null +++ b/notebooks/octave/lkb_finalize.m @@ -0,0 +1,24 @@ +function lkb_finalize(s) + s = lkb_metric(s, "_suite", "_", "elapsed_s", toc(s.t0)); + d = getenv("LKBENCH_RESULTS"); if isempty(d), d = fullfile("..", "..", "results"); end + if ~exist(d, "dir"), mkdir(d); end + write_csv(fullfile(d, [s.name ".checks.csv"]), {"suite", "lang", "check", "severity", "status", "value", "expected", "detail"}, s.checks); + write_csv(fullfile(d, [s.name ".metrics.csv"]), {"suite", "lang", "case", "package", "metric", "value"}, s.metrics); + st = s.checks(:, 5); + printf("%s: PASS %d, FAIL %d, WARN %d, KNOWN %d, SKIP %d\n", s.name, sum(strcmp(st, "PASS")), sum(strcmp(st, "FAIL")), ... + sum(strcmp(st, "WARN")), sum(strcmp(st, "KNOWN")), sum(strcmp(st, "SKIP"))); + failed = s.checks(strcmp(st, "FAIL"), 3); + if ~isempty(failed) + error("%d check(s) failed in %s: %s", numel(failed), s.name, strjoin(failed', "; ")); + end +end + +function write_csv(f, header, rows) + fid = fopen(f, "w"); + fprintf(fid, "%s\n", strjoin(header, ",")); + for i = 1:size(rows, 1) + q = cellfun(@(c) ["\"" strrep(char(c), "\"", "\"\"") "\""], rows(i, :), "UniformOutput", false); + fprintf(fid, "%s\n", strjoin(q, ",")); + end + fclose(fid); +end diff --git a/notebooks/octave/lkb_metric.m b/notebooks/octave/lkb_metric.m new file mode 100644 index 0000000..bd38ed1 --- /dev/null +++ b/notebooks/octave/lkb_metric.m @@ -0,0 +1,3 @@ +function s = lkb_metric(s, case_name, package, metric, value) + s.metrics(end + 1, :) = {s.name, "octave", case_name, package, metric, sprintf("%.6g", double(value))}; +end diff --git a/notebooks/octave/lkb_suite.m b/notebooks/octave/lkb_suite.m new file mode 100644 index 0000000..a26ae2f --- /dev/null +++ b/notebooks/octave/lkb_suite.m @@ -0,0 +1,20 @@ +function s = lkb_suite(name) + % Same CSV outputs as lkbench/checks.py (see there), for the Octave notebooks. + % A suite is a struct passed around by value: s = lkb_close(s, ...). + s.name = name; + s.checks = {}; + s.metrics = {}; + s.t0 = tic(); + s.known = {}; + f = fullfile("..", "..", "known_issues.csv"); + if exist(f, "file") + fid = fopen(f, "r"); fgetl(fid); + while true + l = fgetl(fid); + if ~ischar(l), break; end + p = strsplit(l, ","); + if numel(p) >= 3, s.known(end + 1, :) = {p{1}, p{2}, strjoin(p(3:end), ",")}; end + end + fclose(fid); + end +end diff --git a/notebooks/octave/read_data.m b/notebooks/octave/read_data.m new file mode 100644 index 0000000..1796dc7 --- /dev/null +++ b/notebooks/octave/read_data.m @@ -0,0 +1,3 @@ +function A = read_data(name, stem) + A = dlmread(fullfile("..", "..", "data", [name "_" stem ".csv"]), ","); +end diff --git a/notebooks/octave/src/01_cross_binding.m b/notebooks/octave/src/01_cross_binding.m new file mode 100644 index 0000000..a63524b --- /dev/null +++ b/notebooks/octave/src/01_cross_binding.m @@ -0,0 +1,59 @@ +% %% [markdown] +% # Octave 01 — Cross-binding consistency: the Octave side +% +% Same protocol as `notebooks/python/11_cross_binding.ipynb` (see there), on the +% shared datasets of `data/`; writes `results/xbind_octave.csv`, compared with +% the Python values by `tools/report.py`. mLibKriging must be on the path +% (`OCTAVE_PATH` or `addpath`). + +% %% +s = lkb_suite("oct01_cross_binding"); +out = {}; +function out = put(out, dataset, quantity, values) + v = double(values(:)); + for i = 1:numel(v) + out(end + 1, :) = {"octave", dataset, quantity, i, sprintf("%.17g", v(i))}; + end +end + +% %% +for name = {"branin", "hartmann3", "borehole"} + name = name{1}; + X = read_data(name, "X"); y = read_data(name, "y"); Xt = read_data(name, "Xt")(1:50, :); + d = columns(X); + theta0 = 0.3 * ones(1, d); + kf = Kriging(y, X, "matern5_2", "constant", false, "none", "LL", Params("theta", theta0, "is_theta_estim", false)); + v = kf.sigma2(); out = put(out, name, "fixed_sigma2", v); + v = kf.beta(); out = put(out, name, "fixed_beta", v); + ll = kf.logLikelihood(); + out = put(out, name, "fixed_ll", ll); + loo = kf.leaveOneOut(); + out = put(out, name, "fixed_loo", loo); + [m, sd] = kf.predict(Xt, true, false, false); + out = put(out, name, "fixed_mean", m); + out = put(out, name, "fixed_stdev", sd); + [llf, grad] = kf.logLikelihoodFun(theta0', true, false); + out = put(out, name, "llfun", llf); + out = put(out, name, "llfun_grad", grad); + s = lkb_close(s, [name ": logLikelihood() == logLikelihoodFun(theta0)"], ll, llf, 1e-10); + k = Kriging(y, X, "matern5_2"); + v = k.theta(); out = put(out, name, "fit_theta", v); + llk = k.logLikelihood(); + out = put(out, name, "fit_ll", llk); + [mk, ~] = k.predict(Xt, false, false, false); + out = put(out, name, "fit_mean", mk); + S = kf.simulate(int32(3), int32(123), Xt(1:5, :), false); + out = put(out, name, "sim", S); +end +d = getenv("LKBENCH_RESULTS"); if isempty(d), d = fullfile("..", "..", "results"); end +if ~exist(d, "dir"), mkdir(d); end +fid = fopen(fullfile(d, "xbind_octave.csv"), "w"); +fprintf(fid, "lang,dataset,quantity,index,value\n"); +for i = 1:size(out, 1) + fprintf(fid, "%s,%s,%s,%d,%s\n", out{i, :}); +end +fclose(fid); +printf("%d values written\n", size(out, 1)); + +% %% +lkb_finalize(s); diff --git a/notebooks/octave/src/02_vs_stk.m b/notebooks/octave/src/02_vs_stk.m new file mode 100644 index 0000000..806af40 --- /dev/null +++ b/notebooks/octave/src/02_vs_stk.m @@ -0,0 +1,85 @@ +% %% [markdown] +% # Octave 02 — mLibKriging vs STK +% +% [STK](https://github.com/stk-kriging/stk) (Small Toolbox for Kriging) is the +% reference kriging toolbox for Octave/MATLAB. +% +% 1. **Same hyperparameters → same numbers** (error-level). STK's anisotropic +% Gaussian covariance is σ² exp(−‖(x−x')/ρ‖²) and libKriging's `gauss` is +% σ² exp(−½ Σ ((xⱼ−x'ⱼ)/θⱼ)²): the same kernel with ρ = θ √2. With a constant +% mean (ordinary/universal kriging, β estimated by GLS in both), the +% predicted mean and variance at fixed (θ, σ²) must coincide. (STK's Matérn +% covariances are radial, libKriging's are product-form: not comparable +% hyperparameter by hyperparameter, hence the Gaussian kernel here.) +% 2. **Default fits** on the shared datasets of `data/`: Matérn 5/2, each +% toolbox's own estimation (warn-level comparison of Q²). +% +% STK's location: the `STK_PATH` environment variable (skipped if unset). + +% %% +s = lkb_suite("oct02_vs_stk"); +stk_path = getenv("STK_PATH"); +have_stk = ~isempty(stk_path) && exist(fullfile(stk_path, "stk_init.m"), "file"); +if have_stk + run(fullfile(stk_path, "stk_init.m")); + printf("STK %s\n", stk_version()); +else + s = lkb_check(s, "STK available", false, "absent", "STK_PATH", "STK not found: comparison skipped", "warn"); +end +q2 = @(y, m) 1 - sum((y - m).^2) / sum((y - mean(y)).^2); +datasets = {"branin", "hartmann3", "borehole"}; + +% %% [markdown] +% ## 1. Same hyperparameters, same numbers + +% %% +if have_stk + for i = 1:numel(datasets) + name = datasets{i}; + X = read_data(name, "X"); y = read_data(name, "y"); Xt = read_data(name, "Xt"); + d = columns(X); + theta = linspace(0.2, 0.6, d); + k = Kriging(y, X, "gauss", "constant", false, "none", "LL", Params("theta", theta, "is_theta_estim", false)); + sigma2 = k.sigma2(); + [m_lk, s_lk] = k.predict(Xt, true, false, false); + model = stk_model(@stk_gausscov_aniso, d); + model.param = [log(sigma2); -log(theta(:) * sqrt(2))]; + model.lm = stk_lm_constant(); + zp = stk_predict(model, X, y, Xt); + s = lkb_close(s, [name ": gauss UK mean == STK"], m_lk, zp.mean, 1e-6, 1e-8 * std(y)); + s = lkb_close(s, [name ": gauss UK variance == STK"], s_lk.^2, zp.var, 1e-4, 1e-8 * sigma2); + end +end + +% %% [markdown] +% ## 2. Default fits + +% %% +for i = 1:numel(datasets) + name = datasets{i}; + X = read_data(name, "X"); y = read_data(name, "y"); Xt = read_data(name, "Xt"); yt = read_data(name, "yt"); + d = columns(X); + tic; k = Kriging(y, X, "matern5_2"); t = toc; + [m, ~] = k.predict(Xt, false, false, false); + q_lk = q2(yt, m); + s = lkb_metric(s, name, "mLibKriging", "q2", q_lk); s = lkb_metric(s, name, "mLibKriging", "fit_s", t); + tic; k10 = Kriging(y, X, "matern5_2", "constant", false, "BFGS10", "LL"); t = toc; + [m, ~] = k10.predict(Xt, false, false, false); + q_lk10 = q2(yt, m); + s = lkb_metric(s, name, "mLibKriging BFGS10", "q2", q_lk10); s = lkb_metric(s, name, "mLibKriging BFGS10", "fit_s", t); + s = lkb_check(s, [name ": Q2(mLibKriging BFGS10) > 0.9"], q_lk10 > 0.9, q_lk10, "> 0.9"); + if have_stk + tic; + model = stk_model(@stk_materncov52_aniso, d); + model = stk_param_estim(model, X, y); % (STK >= 2.8 returns the model) + zp = stk_predict(model, X, y, Xt); + t = toc; + q_stk = q2(yt, zp.mean); + s = lkb_metric(s, name, "STK", "q2", q_stk); s = lkb_metric(s, name, "STK", "fit_s", t); + s = lkb_check(s, [name ": Q2(mLibKriging) - Q2(STK)"], q_lk - q_stk > -0.02, q_lk - q_stk, "> -0.02", "", "warn"); + end +end +disp(s.metrics(:, 3:6)) + +% %% +lkb_finalize(s); diff --git a/notebooks/python/00_test_functions.ipynb b/notebooks/python/00_test_functions.ipynb new file mode 100644 index 0000000..1ee5720 --- /dev/null +++ b/notebooks/python/00_test_functions.ipynb @@ -0,0 +1,168 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "c000", + "metadata": {}, + "source": [ + "# 00 — Test functions\n", + "\n", + "Every benchmark of this repository relies on the test functions of\n", + "`lkbench.functions` (after the [Virtual Library of Simulation Experiments](https://www.sfu.ca/~ssurjano/)).\n", + "A typo in one of them would silently bias every result built on it, so they are\n", + "checked first, against the reference values published on the library pages\n", + "(global optima, nominal points) and against the known output ranges." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c001", + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "import pandas as pd\n", + "import matplotlib.pyplot as plt\n", + "\n", + "import lkbench\n", + "from lkbench import TESTS, design\n", + "\n", + "suite = lkbench.Suite(\"py00_test_functions\")" + ] + }, + { + "cell_type": "markdown", + "id": "c002", + "metadata": {}, + "source": [ + "## Reference values" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c003", + "metadata": {}, + "outputs": [], + "source": [ + "rows = []\n", + "for name, tf in TESTS.items():\n", + " for x, v in tf.known:\n", + " fx = float(tf(x)[0])\n", + " suite.close(f\"{name}: f({np.round(x, 4).tolist()})\", fx, v, rtol=1e-4, atol=1e-4)\n", + " rows.append((name, tf.d, np.round(x, 4).tolist(), v, fx))\n", + "pd.DataFrame(rows, columns=[\"function\", \"d\", \"x\", \"reference\", \"computed\"])" + ] + }, + { + "cell_type": "markdown", + "id": "c004", + "metadata": {}, + "source": [ + "## Output ranges\n", + "\n", + "For the engineering functions without a closed-form optimum, the range over the\n", + "domain is compared with the one reported in the literature (loose bounds: a\n", + "random sample does not reach the extremes)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c005", + "metadata": {}, + "outputs": [], + "source": [ + "EXPECTED_RANGES = { # (lower bound of min, upper bound of max) over the domain\n", + " \"otl_circuit\": (2.5, 10.0),\n", + " \"piston\": (0.1, 1.3),\n", + " \"borehole\": (2.0, 320.0),\n", + " \"wing_weight\": (120.0, 480.0),\n", + " \"currin_exp\": (1.0, 14.0),\n", + " \"dette_pepelyshev\": (0.0, 100.0),\n", + "}\n", + "summary = []\n", + "for name, tf in TESTS.items():\n", + " U = design.uniform(20_000, tf.d, seed=1)\n", + " y = tf.unit(U)\n", + " suite.finite(f\"{name}: finite on the domain\", y)\n", + " if name in EXPECTED_RANGES:\n", + " lo, hi = EXPECTED_RANGES[name]\n", + " suite.true(f\"{name}: range within [{lo}, {hi}]\", lo <= y.min() and y.max() <= hi,\n", + " detail=f\"sampled range [{y.min():.4g}, {y.max():.4g}]\")\n", + " summary.append((name, tf.d, \",\".join(tf.tags), y.min(), y.max(), y.std()))\n", + "pd.DataFrame(summary, columns=[\"function\", \"d\", \"tags\", \"min\", \"max\", \"std\"])" + ] + }, + { + "cell_type": "markdown", + "id": "c006", + "metadata": {}, + "source": [ + "## Designs\n", + "\n", + "The designs are seeded: the same seed gives the same points (reproducibility of\n", + "every benchmark), and the optimized LHS is a Latin hypercube (one point per\n", + "stratum in each dimension)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c007", + "metadata": {}, + "outputs": [], + "source": [ + "A, B = design.lhs(20, 3, seed=7), design.lhs(20, 3, seed=7)\n", + "suite.close(\"lhs: reproducible with a seed\", A, B, rtol=0, atol=0)\n", + "strata = np.floor(A * 20).astype(int)\n", + "suite.true(\"lhs: one point per stratum\", all(len(set(strata[:, j])) == 20 for j in range(3)))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c008", + "metadata": {}, + "outputs": [], + "source": [ + "fig, axes = plt.subplots(1, 4, figsize=(16, 3.5))\n", + "for ax, name in zip(axes, [\"forrester\", \"gramacy_lee_2012\"]):\n", + " tf = TESTS[name]\n", + " u = np.linspace(0, 1, 500)[:, None]\n", + " ax.plot(tf.scale(u)[:, 0], tf.unit(u))\n", + " ax.set_title(name)\n", + "for ax, name in zip(axes[2:], [\"branin\", \"gramacy_lee_2008\"]):\n", + " tf = TESTS[name]\n", + " g = np.linspace(0, 1, 100)\n", + " G = np.array(np.meshgrid(g, g)).reshape(2, -1).T\n", + " ax.contourf(g, g, tf.unit(G).reshape(100, 100), 30)\n", + " ax.set_title(name)\n", + "plt.tight_layout()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c009", + "metadata": {}, + "outputs": [], + "source": [ + "suite.finalize()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/python/01_kriging_kernels_trends.ipynb b/notebooks/python/01_kriging_kernels_trends.ipynb new file mode 100644 index 0000000..5517780 --- /dev/null +++ b/notebooks/python/01_kriging_kernels_trends.ipynb @@ -0,0 +1,297 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "c000", + "metadata": {}, + "source": [ + "# 01 — Kriging: kernels × trends on 1D/2D/3D functions\n", + "\n", + "Plain noise-free `Kriging`, every kernel (`gauss`, `exp`, `matern3_2`,\n", + "`matern5_2`) crossed with every trend (`constant`, `linear`, `interactive`,\n", + "`quadratic`) on functions of different regularity:\n", + "\n", + "| function | d | character |\n", + "|---|---|---|\n", + "| Forrester | 1 | smooth, one oscillation |\n", + "| Gramacy & Lee (2012) | 1 | fast oscillation then polynomial: non-stationary |\n", + "| Branin | 2 | smooth, three global minima |\n", + "| Six-hump camel | 2 | polynomial, steep borders |\n", + "| log-Goldstein-Price | 2 | strongly varying |\n", + "| Lim polynomial | 2 | low-degree polynomial: the trend should help |\n", + "| Ishigami | 3 | strongly non-linear, interactions |\n", + "| Hartmann 3 | 3 | multimodal, localized |\n", + "\n", + "Checks:\n", + "* **interpolation**: a noise-free kriging mean reproduces the observations, with\n", + " a (near) zero predictive standard deviation at the design points. Error-level\n", + " for every kernel but `gauss`: its correlation matrices are notoriously\n", + " ill-conditioned, and the numerical jitter libKriging adds to factorize them\n", + " breaks exact interpolation (warn-level, with the size of the gap);\n", + "* **accuracy**: Q² on a test set above a per-function floor for\n", + " `matern5_2` / `constant` with a multistart optimizer (`BFGS10`): a\n", + " regression shows up here first. The same floor for the default single-start\n", + " `BFGS` is warn-level: a single start can end in a poor local optimum of the\n", + " likelihood (see notebook 02), which is worth seeing in the report but is not\n", + " a defect of the model;\n", + "* **sanity of the fitted hyperparameters**: finite, positive θ and σ²;\n", + "* **consistency**: `predict` on a batch equals `predict` point by point, the\n", + " returned covariance has the predicted variances on its diagonal, and is\n", + " symmetric positive semi-definite." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c001", + "metadata": {}, + "outputs": [], + "source": [ + "import time\n", + "\n", + "import numpy as np\n", + "import pandas as pd\n", + "import matplotlib.pyplot as plt\n", + "import pylibkriging as lk\n", + "\n", + "import lkbench\n", + "from lkbench import TESTS, design, metrics, compat\n", + "\n", + "suite = lkbench.Suite(\"py01_kernels_trends\")\n", + "print(\"pylibkriging\", compat.version())\n", + "\n", + "KERNELS = [\"gauss\", \"exp\", \"matern3_2\", \"matern5_2\"]\n", + "TRENDS = [\"constant\", \"linear\", \"interactive\", \"quadratic\"]\n", + "CASES = { # function: (n, Q2 floor for matern5_2/constant)\n", + " \"forrester\": (12, 0.95),\n", + " \"gramacy_lee_2012\": (20, 0.60),\n", + " \"branin\": (30, 0.97),\n", + " \"six_hump_camel\": (30, 0.95),\n", + " \"log_goldstein_price\": (40, 0.85),\n", + " \"lim_polynomial\": (15, 0.99),\n", + " \"ishigami\": (80, 0.75),\n", + " \"hartmann3\": (40, 0.85),\n", + "}\n", + "\n", + "\n", + "def predict(model, X, **kw):\n", + " m, s, cov, *_ = model.predict(X, return_stdev=True, **kw)\n", + " return np.asarray(m).ravel(), np.asarray(s).ravel(), cov" + ] + }, + { + "cell_type": "markdown", + "id": "c002", + "metadata": {}, + "source": [ + "## Kernel × trend grid" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c003", + "metadata": {}, + "outputs": [], + "source": [ + "for fname, (n, q2_floor) in CASES.items():\n", + " tf = TESTS[fname]\n", + " X, y, Xt, yt = design.dataset(tf, n, n_test=1000, seed=1)\n", + " for kernel in KERNELS:\n", + " for trend in TRENDS:\n", + " n_beta = {\"constant\": 1, \"linear\": 1 + tf.d, \"interactive\": 1 + tf.d + tf.d * (tf.d - 1) // 2,\n", + " \"quadratic\": 1 + 2 * tf.d + tf.d * (tf.d - 1) // 2}[trend]\n", + " case = f\"{fname}/{kernel}/{trend}\"\n", + " if n_beta >= n - 1:\n", + " suite.skip(case, f\"{n_beta} trend coefficients for n={n}\")\n", + " continue\n", + " t0 = time.time()\n", + " try:\n", + " k = lk.Kriging(y, X, kernel, regmodel=trend, optim=\"BFGS\")\n", + " except Exception as exc: # noqa: BLE001\n", + " suite.true(f\"{case}: fit\", False, detail=str(exc)[:200])\n", + " continue\n", + " fit_s = time.time() - t0\n", + " m, s, _ = predict(k, Xt)\n", + " res = metrics.all_metrics(yt, m, s)\n", + " suite.metric(case, \"pylibkriging\", fit_s=fit_s, **res)\n", + " theta, sigma2 = np.asarray(k.theta()).ravel(), float(k.sigma2())\n", + " suite.true(f\"{case}: theta, sigma2 finite and > 0\",\n", + " np.all(np.isfinite(theta)) and np.all(theta > 0) and np.isfinite(sigma2) and sigma2 > 0,\n", + " detail=f\"theta={theta}, sigma2={sigma2:.3g}\")\n", + " # interpolation at the design points (relative to the spread of y)\n", + " md, sd, _ = predict(k, X)\n", + " scale = np.std(y)\n", + " sev = \"warn\" if kernel == \"gauss\" else \"error\"\n", + " suite.less(f\"{case}: interpolates y (max |m(X)-y| / sd(y))\", np.max(np.abs(md - y)) / scale, 1e-4,\n", + " severity=sev)\n", + " suite.less(f\"{case}: stdev(X) ~ 0 (max sd / sd(y))\", np.max(sd) / scale, 1e-2, severity=sev)\n", + " if kernel == \"matern5_2\" and trend == \"constant\":\n", + " suite.greater(f\"{case}: Q2 (single-start BFGS)\", res[\"q2\"], q2_floor, severity=\"warn\")\n", + " k10 = lk.Kriging(y, X, kernel, regmodel=trend, optim=\"BFGS10\")\n", + " res10 = metrics.all_metrics(yt, *predict(k10, Xt)[:2])\n", + " suite.metric(case, \"pylibkriging BFGS10\", **res10)\n", + " suite.greater(f\"{case}: Q2 (BFGS10)\", res10[\"q2\"], q2_floor)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c004", + "metadata": {}, + "outputs": [], + "source": [ + "tab = suite.table().reset_index()\n", + "tab = tab[tab[\"package\"] == \"pylibkriging\"]\n", + "tab[[\"function\", \"kernel\", \"trend\"]] = tab[\"case\"].str.split(\"/\", expand=True)\n", + "pivot = tab.pivot_table(index=[\"function\", \"trend\"], columns=\"kernel\", values=\"q2\", sort=False)\n", + "pivot.style.format(\"{:.3f}\").background_gradient(axis=None, vmin=0.5, vmax=1, cmap=\"RdYlGn\")" + ] + }, + { + "cell_type": "markdown", + "id": "c005", + "metadata": {}, + "source": [ + "Expectations that should hold whatever the version (warn-level: they are\n", + "empirical, not mathematical, statements):\n", + "* on the smooth Branin the `exp` kernel (non-differentiable paths) is the least accurate;\n", + "* on the Lim polynomial a `quadratic` trend is at least as good as a `constant` one." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c006", + "metadata": {}, + "outputs": [], + "source": [ + "q = tab.set_index([\"function\", \"kernel\", \"trend\"])[\"q2\"]\n", + "suite.true(\"branin: exp is the worst kernel (constant trend)\",\n", + " q[\"branin\", \"exp\", \"constant\"] <= min(q[\"branin\", k, \"constant\"] for k in KERNELS),\n", + " severity=\"warn\")\n", + "suite.greater(\"lim_polynomial/matern5_2: Q2(quadratic) - Q2(constant)\",\n", + " q[\"lim_polynomial\", \"matern5_2\", \"quadratic\"] - q[\"lim_polynomial\", \"matern5_2\", \"constant\"], -1e-3,\n", + " severity=\"warn\")" + ] + }, + { + "cell_type": "markdown", + "id": "c007", + "metadata": {}, + "source": [ + "## Consistency of `predict`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c008", + "metadata": {}, + "outputs": [], + "source": [ + "tf = TESTS[\"branin\"]\n", + "X, y, Xt, yt = design.dataset(tf, 30, n_test=50, seed=3)\n", + "k = lk.Kriging(y, X, \"matern5_2\")\n", + "m, s, cov = predict(k, Xt, return_cov=True)\n", + "cov = np.asarray(cov)\n", + "suite.close(\"predict: diag(cov) == stdev^2\", np.diag(cov), s ** 2, rtol=1e-6, atol=1e-10)\n", + "suite.close(\"predict: cov symmetric\", cov, cov.T, rtol=1e-10, atol=1e-12)\n", + "suite.greater(\"predict: cov PSD (min eigenvalue / max)\", np.linalg.eigvalsh(cov).min() / np.abs(cov).max(), -1e-8)\n", + "m1 = np.array([predict(k, Xt[i:i + 1])[0][0] for i in range(len(Xt))])\n", + "s1 = np.array([predict(k, Xt[i:i + 1])[1][0] for i in range(len(Xt))])\n", + "suite.close(\"predict: batch == point by point (mean)\", m, m1, rtol=1e-8, atol=1e-10 * np.std(y))\n", + "suite.close(\"predict: batch == point by point (stdev)\", s, s1, rtol=1e-6, atol=1e-10 * np.std(y))\n", + "# the mean is a function of the point only: duplicating/permuting the batch changes nothing\n", + "perm = np.random.default_rng(0).permutation(len(Xt))\n", + "suite.close(\"predict: invariant to the order of the batch\", predict(k, Xt[perm])[0], m[perm], rtol=1e-10, atol=1e-10)" + ] + }, + { + "cell_type": "markdown", + "id": "c009", + "metadata": {}, + "source": [ + "### Derivatives of the predictor\n", + "\n", + "`return_deriv=True` gives the gradients of the mean and of the stdev w.r.t. x;\n", + "compared with central finite differences." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c010", + "metadata": {}, + "outputs": [], + "source": [ + "with suite.runs(\"predict(return_deriv=True)\"):\n", + " out = k.predict(Xt[:5], True, False, True)\n", + " dm, ds = np.asarray(out[3]), np.asarray(out[4])\n", + " h = 1e-6\n", + " fd_m, fd_s = np.zeros_like(dm), np.zeros_like(ds)\n", + " for j in range(2):\n", + " e = np.zeros(2); e[j] = h\n", + " mp, sp, _ = predict(k, Xt[:5] + e)\n", + " mm, sm, _ = predict(k, Xt[:5] - e)\n", + " fd_m[:, j], fd_s[:, j] = (mp - mm) / (2 * h), (sp - sm) / (2 * h)\n", + " suite.close(\"d mean / dx == finite differences\", dm, fd_m, rtol=1e-4, atol=1e-4 * np.abs(fd_m).max())\n", + " suite.close(\"d stdev / dx == finite differences\", ds, fd_s, rtol=1e-3, atol=1e-3 * np.abs(fd_s).max())" + ] + }, + { + "cell_type": "markdown", + "id": "c011", + "metadata": {}, + "source": [ + "## A look at the 1D fits" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c012", + "metadata": {}, + "outputs": [], + "source": [ + "fig, axes = plt.subplots(1, 2, figsize=(13, 4))\n", + "for ax, fname in zip(axes, [\"forrester\", \"gramacy_lee_2012\"]):\n", + " tf = TESTS[fname]\n", + " n = CASES[fname][0]\n", + " X, y, _, _ = design.dataset(tf, n, seed=1)\n", + " u = np.linspace(0, 1, 400)[:, None]\n", + " ax.plot(u, tf.unit(u), \"k--\", label=\"truth\")\n", + " for kernel in [\"gauss\", \"matern5_2\", \"exp\"]:\n", + " mm, ss, _ = predict(lk.Kriging(y, X, kernel), u)\n", + " ax.plot(u, mm, label=kernel)\n", + " if kernel == \"matern5_2\":\n", + " ax.fill_between(u[:, 0], mm - 2 * ss, mm + 2 * ss, alpha=0.2)\n", + " ax.plot(X, y, \"ko\")\n", + " ax.set_title(fname)\n", + " ax.legend()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c013", + "metadata": {}, + "outputs": [], + "source": [ + "suite.finalize()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/python/02_objectives_gradients.ipynb b/notebooks/python/02_objectives_gradients.ipynb new file mode 100644 index 0000000..c2e9912 --- /dev/null +++ b/notebooks/python/02_objectives_gradients.ipynb @@ -0,0 +1,277 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "c000", + "metadata": {}, + "source": [ + "# 02 — Objectives (LL, LOO, LMP), their gradients and optima\n", + "\n", + "The hyperparameters θ of `Kriging` are estimated by maximizing the\n", + "log-likelihood (`LL`), maximizing the log marginal posterior (`LMP`), or\n", + "minimizing the leave-one-out error (`LOO`). This notebook checks the machinery\n", + "behind those fits rather than the predictions:\n", + "\n", + "* **analytic gradients** of `logLikelihoodFun`, `leaveOneOutFun`,\n", + " `logMargPostFun` against central finite differences, at random θ, for\n", + " every kernel;\n", + "* **optimality** of the fitted θ: the objective at θ̂ is not improved by small\n", + " perturbations of θ̂ (a local optimum was reached). A component θ_j > 5 on\n", + " the unit cube is a nearly linear direction whose optimum sits at the upper\n", + " bound of θ: moving it further out is allowed to improve the objective;\n", + "* **global vs local optimum**: the gap between the default single-start fit\n", + " and a 20-start fit (warn-level: it measures how often the default fit ends in\n", + " a poor local optimum);\n", + "* **consistency** between `logLikelihood()` and `logLikelihoodFun(theta())`;\n", + "* the closed-form **leave-one-out** predictions of `leaveOneOutVec` against an\n", + " explicit refit without each point (same θ, σ²);\n", + "* multistart (`BFGS10`) never ends below a single `BFGS` run (warn-level: a\n", + " multistart that does worse points to a starting-point issue).\n", + "\n", + "Functions: Branin (2D), Hartmann 3 (3D), Ishigami (3D), Friedman (5D)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c001", + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "import pandas as pd\n", + "import pylibkriging as lk\n", + "\n", + "import lkbench\n", + "from lkbench import TESTS, design, metrics, compat\n", + "\n", + "suite = lkbench.Suite(\"py02_objectives_gradients\")\n", + "print(\"pylibkriging\", compat.version())\n", + "CASES = {\"branin\": 25, \"hartmann3\": 40, \"ishigami\": 60, \"friedman\": 50}\n", + "KERNELS = [\"gauss\", \"exp\", \"matern3_2\", \"matern5_2\"]\n", + "rng = np.random.default_rng(0)\n", + "\n", + "\n", + "def objective_fun(k, objective):\n", + " \"\"\"(value, gradient) of the objective as a function of theta.\"\"\"\n", + " if objective == \"LL\":\n", + " return lambda t, g=True: k.logLikelihoodFun(np.asarray(t, float), g)[:2]\n", + " if objective == \"LOO\":\n", + " return lambda t, g=True: k.leaveOneOutFun(np.asarray(t, float), g)\n", + " return lambda t, g=True: k.logMargPostFun(np.asarray(t, float), g)" + ] + }, + { + "cell_type": "markdown", + "id": "c002", + "metadata": {}, + "source": [ + "## Analytic gradients vs finite differences" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c003", + "metadata": {}, + "outputs": [], + "source": [ + "rows = []\n", + "for fname, n in CASES.items():\n", + " tf = TESTS[fname]\n", + " X, y, _, _ = design.dataset(tf, n, seed=2)\n", + " for kernel in KERNELS:\n", + " k = lk.Kriging(y, X, kernel)\n", + " for objective in [\"LL\", \"LOO\", \"LMP\"]:\n", + " f = objective_fun(k, objective)\n", + " worst = 0.0\n", + " for _ in range(3):\n", + " theta = np.exp(rng.uniform(np.log(0.05), np.log(1.0), tf.d))\n", + " val, grad = f(theta)\n", + " fd = metrics.finite_diff_grad(lambda t: f(t, False)[0], theta, h=1e-5)\n", + " grad = np.asarray(grad).ravel()\n", + " err = np.max(np.abs(grad - fd)) / max(np.max(np.abs(fd)), 1e-8)\n", + " worst = max(worst, err)\n", + " rows.append((fname, kernel, objective, worst))\n", + " suite.less(f\"{fname}/{kernel}/{objective}: gradient vs finite diff (rel. err)\", worst, 1e-3)\n", + "pd.DataFrame(rows, columns=[\"function\", \"kernel\", \"objective\", \"max rel. error\"]).pivot_table(\n", + " index=[\"function\", \"kernel\"], columns=\"objective\", values=\"max rel. error\").style.format(\"{:.1e}\")" + ] + }, + { + "cell_type": "markdown", + "id": "c004", + "metadata": {}, + "source": [ + "## Optimality of the fitted θ, objective by objective" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c005", + "metadata": {}, + "outputs": [], + "source": [ + "SIGN = {\"LL\": 1, \"LMP\": 1, \"LOO\": -1} # +1: maximized, -1: minimized\n", + "for fname, n in CASES.items():\n", + " tf = TESTS[fname]\n", + " X, y, Xt, yt = design.dataset(tf, n, n_test=1000, seed=2)\n", + " for objective in [\"LL\", \"LOO\", \"LMP\"]:\n", + " k = lk.Kriging(y, X, \"matern5_2\", objective=objective)\n", + " theta = np.asarray(k.theta()).ravel()\n", + " f = objective_fun(k, objective)\n", + " best = SIGN[objective] * f(theta, False)[0]\n", + " improved = []\n", + " for j in range(tf.d):\n", + " for fac in (0.9, 1.1):\n", + " t = theta.copy(); t[j] *= fac\n", + " at_bound = fac > 1 and theta[j] > 5\n", + " # tolerance: the optimizer's stopping criterion (0.01 on a log-density is immaterial)\n", + " tol = 1e-4 * abs(best) + (0.0 if objective == \"LOO\" else 1e-2)\n", + " if SIGN[objective] * f(t, False)[0] > best + tol and not at_bound:\n", + " improved.append(f\"theta{j + 1}*{fac}\")\n", + " suite.true(f\"{fname}/{objective}: theta_hat not improved by +/-10% moves\", not improved,\n", + " detail=f\"theta={np.round(theta, 4)}; improving moves: {improved}\")\n", + " m, s, *_ = k.predict(Xt, return_stdev=True)\n", + " suite.metric(f\"{fname}/{objective}\", \"pylibkriging\", **metrics.all_metrics(yt, m, s))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c006", + "metadata": {}, + "outputs": [], + "source": [ + "tab = suite.table().reset_index()\n", + "tab[[\"function\", \"objective\"]] = tab[\"case\"].str.split(\"/\", expand=True)\n", + "tab.pivot_table(index=\"function\", columns=\"objective\", values=[\"q2\", \"coverage95\"]).round(3)" + ] + }, + { + "cell_type": "markdown", + "id": "c007", + "metadata": {}, + "source": [ + "## `logLikelihood()` == `logLikelihoodFun(theta())` (and the same for LOO, LMP)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c008", + "metadata": {}, + "outputs": [], + "source": [ + "tf = TESTS[\"hartmann3\"]\n", + "X, y, _, _ = design.dataset(tf, 40, seed=5)\n", + "for objective, getter in [(\"LL\", \"logLikelihood\"), (\"LOO\", \"leaveOneOut\"), (\"LMP\", \"logMargPost\")]:\n", + " k = lk.Kriging(y, X, \"matern5_2\", objective=objective)\n", + " v_fun = objective_fun(k, objective)(np.asarray(k.theta()).ravel(), False)[0]\n", + " suite.close(f\"{getter}() == {objective} function at theta()\", getattr(k, getter)(), v_fun, rtol=1e-8)" + ] + }, + { + "cell_type": "markdown", + "id": "c009", + "metadata": {}, + "source": [ + "## Closed-form leave-one-out vs explicit refits\n", + "\n", + "`leaveOneOutVec(θ)` gives the mean and stdev of the prediction of each `y_i`\n", + "from the n−1 others, by the closed-form (Dubrule) formulas. An explicit refit\n", + "without point i, with θ and σ² fixed (β re-estimated, as in the formula), must\n", + "give the same mean." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c010", + "metadata": {}, + "outputs": [], + "source": [ + "for fname in [\"branin\", \"hartmann3\"]:\n", + " tf = TESTS[fname]\n", + " X, y, _, _ = design.dataset(tf, 20, seed=6)\n", + " for trend in [\"constant\", \"linear\"]:\n", + " k = lk.Kriging(y, X, \"matern5_2\", regmodel=trend)\n", + " theta, sigma2 = np.asarray(k.theta()).ravel(), float(k.sigma2())\n", + " m_loo, s_loo = (np.asarray(a).ravel() for a in k.leaveOneOutVec(theta))\n", + " m_ref, s_ref = np.zeros(len(y)), np.zeros(len(y))\n", + " for i in range(len(y)):\n", + " keep = np.arange(len(y)) != i\n", + " ki = lk.Kriging(y[keep], X[keep], \"matern5_2\", regmodel=trend, optim=\"none\",\n", + " parameters={\"theta\": theta.reshape(1, -1), \"is_theta_estim\": False,\n", + " \"sigma2\": sigma2, \"is_sigma2_estim\": False})\n", + " mi, si, *_ = ki.predict(X[i:i + 1], return_stdev=True)\n", + " m_ref[i], s_ref[i] = np.ravel(mi)[0], np.ravel(si)[0]\n", + " suite.close(f\"{fname}/{trend}: leaveOneOutVec mean == explicit refits\", m_loo, m_ref,\n", + " rtol=1e-6, atol=1e-8 * np.std(y))\n", + " suite.close(f\"{fname}/{trend}: leaveOneOutVec stdev == explicit refits\", s_loo, s_ref,\n", + " rtol=1e-4, atol=1e-8 * np.std(y), severity=\"warn\")\n", + " # leaveOneOut() is the mean squared LOO error at the fitted theta\n", + " k_loo = lk.Kriging(y, X, \"matern5_2\", regmodel=trend, objective=\"LOO\")\n", + " m2, _ = k_loo.leaveOneOutVec(np.asarray(k_loo.theta()).ravel())\n", + " suite.close(f\"{fname}/{trend}: leaveOneOut() == mean squared LOO error\", k_loo.leaveOneOut(),\n", + " np.mean((np.ravel(m2) - y) ** 2), rtol=1e-6, severity=\"warn\")" + ] + }, + { + "cell_type": "markdown", + "id": "c011", + "metadata": {}, + "source": [ + "## Multistart\n", + "\n", + "Gap between the default single start and 10 / 20 starts. A large gap means the\n", + "default fit is unreliable on this kind of problem." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c012", + "metadata": {}, + "outputs": [], + "source": [ + "for fname, n in CASES.items():\n", + " tf = TESTS[fname]\n", + " X, y, _, _ = design.dataset(tf, n, seed=7)\n", + " ll1 = lk.Kriging(y, X, \"matern5_2\", optim=\"BFGS\").logLikelihood()\n", + " ll10 = lk.Kriging(y, X, \"matern5_2\", optim=\"BFGS10\").logLikelihood()\n", + " ll20 = lk.Kriging(y, X, \"matern5_2\", optim=\"BFGS20\").logLikelihood()\n", + " suite.metric(fname, \"BFGS\", ll=ll1)\n", + " suite.metric(fname, \"BFGS10\", ll=ll10)\n", + " suite.metric(fname, \"BFGS20\", ll=ll20)\n", + " suite.greater(f\"{fname}: LL(BFGS10) - LL(BFGS)\", ll10 - ll1, -1e-3 * abs(ll1), severity=\"warn\")\n", + " suite.less(f\"{fname}: LL(BFGS20) - LL(BFGS) (default fit far from the best optimum)\", ll20 - ll1, 1.0,\n", + " severity=\"warn\")\n", + "suite.table().loc[list(CASES)].unstack(\"package\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c013", + "metadata": {}, + "outputs": [], + "source": [ + "suite.finalize()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/python/03_noise.ipynb b/notebooks/python/03_noise.ipynb new file mode 100644 index 0000000..c737eb7 --- /dev/null +++ b/notebooks/python/03_noise.ipynb @@ -0,0 +1,235 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "c000", + "metadata": {}, + "source": [ + "# 03 — Noisy observations: `noise=\"nugget\"` and known noise variances\n", + "\n", + "`Kriging` handles noisy responses through its `noise` argument: `\"nugget\"`\n", + "estimates one homogeneous noise variance, a vector gives a known variance per\n", + "observation (heteroscedastic case).\n", + "\n", + "| case | function | noise |\n", + "|---|---|---|\n", + "| 1D homoscedastic | Forrester | N(0, 0.5²) |\n", + "| 5D homoscedastic | Friedman | N(0, 1) (the classical \"Friedman 1\" setup) |\n", + "| 1D heteroscedastic | Gramacy & Lee (2012) | sd growing linearly with x |\n", + "| 2D with replicates | Branin | 4 replicates of 15 points, N(0, 2²) |\n", + "\n", + "Checks:\n", + "* the estimated nugget is close to the true noise variance (factor 3);\n", + "* the noisy fit predicts the noise-free function well (Q² floor), and is\n", + " **smooth**: it does not chase the noisy data between the observations. (At the\n", + " design points themselves the `\"nugget\"` predictor may return y exactly, as\n", + " DiceKriging's nugget does: the nugget is then a micro-scale variation, and\n", + " the predictor is discontinuous there. This is reported, not checked.);\n", + "* prediction intervals are calibrated: 95% coverage of the *noise-free* truth;\n", + "* a zero noise vector gives the noise-free model;\n", + "* with heteroscedastic noise, giving the true variances beats a single nugget\n", + " (NLPD, warn-level);\n", + "* replicates: averaging the replicates with variance σ²/r is equivalent to\n", + " fitting all the replicates (same predictor, up to the optimizer)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c001", + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "import pandas as pd\n", + "import matplotlib.pyplot as plt\n", + "import pylibkriging as lk\n", + "\n", + "import lkbench\n", + "from lkbench import TESTS, design, metrics, compat\n", + "\n", + "suite = lkbench.Suite(\"py03_noise\")\n", + "print(\"pylibkriging\", compat.version())\n", + "\n", + "\n", + "def pred(k, X):\n", + " m, s, *_ = k.predict(X, return_stdev=True)\n", + " return np.ravel(m), np.ravel(s)" + ] + }, + { + "cell_type": "markdown", + "id": "c002", + "metadata": {}, + "source": [ + "## Homoscedastic noise, estimated nugget" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c003", + "metadata": {}, + "outputs": [], + "source": [ + "for fname, n, sd, q2_floor in [(\"forrester\", 40, 0.5, 0.95), (\"friedman\", 200, 1.0, 0.90)]:\n", + " tf = TESTS[fname]\n", + " X, y, Xt, yt = design.dataset(tf, n, n_test=2000, seed=11, noise_sd=sd)\n", + " k = lk.Kriging(y, X, \"matern5_2\", noise=\"nugget\", optim=\"BFGS10\")\n", + " nugget = float(np.ravel(k.nugget())[0])\n", + " m, s = pred(k, Xt)\n", + " res = metrics.all_metrics(yt, m, s)\n", + " suite.metric(fname, \"nugget\", nugget=nugget, true_var=sd ** 2, **res)\n", + " suite.between(f\"{fname}: nugget / true noise variance\", nugget / sd ** 2, 1 / 3, 3)\n", + " suite.greater(f\"{fname}: Q2 vs noise-free truth\", res[\"q2\"], q2_floor)\n", + " mX, _ = pred(k, X)\n", + " mX_near, _ = pred(k, np.clip(X + 1e-7, 0, 1))\n", + " suite.metric(fname, \"nugget\", rms_resid_at_X=metrics.rmse(y, mX) / sd,\n", + " rms_resid_near_X=metrics.rmse(y, mX_near) / sd)\n", + " suite.greater(f\"{fname}: smooths the noisy data next to the design points (rms residual / sd)\",\n", + " metrics.rmse(y, mX_near) / sd, 0.3)\n", + " suite.between(f\"{fname}: coverage of the 95% intervals (noise-free truth)\", res[\"coverage95\"], 0.80, 1.0,\n", + " severity=\"warn\")\n", + " # the noise-free model on the same noisy data interpolates, and generalizes worse\n", + " k0 = lk.Kriging(y, X, \"matern5_2\")\n", + " m0, s0 = pred(k0, Xt)\n", + " suite.metric(fname, \"noise-free fit\", **metrics.all_metrics(yt, m0, s0))\n", + " suite.less(f\"{fname}: RMSE(nugget) < RMSE(interpolating fit)\", res[\"rmse\"], metrics.rmse(yt, m0),\n", + " severity=\"warn\")\n", + "suite.table()" + ] + }, + { + "cell_type": "markdown", + "id": "c004", + "metadata": {}, + "source": [ + "## A zero noise vector is the noise-free model" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c005", + "metadata": {}, + "outputs": [], + "source": [ + "tf = TESTS[\"branin\"]\n", + "X, y, Xt, yt = design.dataset(tf, 25, n_test=200, seed=12)\n", + "k0 = lk.Kriging(y, X, \"matern5_2\")\n", + "theta = np.asarray(k0.theta()).reshape(1, -1)\n", + "fixed = {\"theta\": theta, \"is_theta_estim\": False, \"sigma2\": float(k0.sigma2()), \"is_sigma2_estim\": False}\n", + "kf = lk.Kriging(y, X, \"matern5_2\", optim=\"none\", parameters=fixed)\n", + "with suite.runs(\"noise=zeros, theta and sigma2 fixed: fit\"):\n", + " kz = lk.Kriging(y, X, \"matern5_2\", noise=np.zeros(len(y)), optim=\"none\", parameters=fixed)\n", + " suite.close(\"noise=zeros: same mean as the noise-free model\", pred(kz, Xt)[0], pred(kf, Xt)[0], rtol=1e-6,\n", + " atol=1e-8 * np.std(y))\n", + " suite.close(\"noise=zeros: same stdev as the noise-free model\", pred(kz, Xt)[1], pred(kf, Xt)[1], rtol=1e-4,\n", + " atol=1e-6 * np.std(y))\n", + "with suite.runs(\"noise=zeros, sigma2 estimated: fit\", severity=\"warn\"):\n", + " lk.Kriging(y, X, \"matern5_2\", noise=np.zeros(len(y)), optim=\"none\",\n", + " parameters={\"theta\": theta, \"is_theta_estim\": False})" + ] + }, + { + "cell_type": "markdown", + "id": "c006", + "metadata": {}, + "source": [ + "## Heteroscedastic noise: known variances vs a single nugget" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c007", + "metadata": {}, + "outputs": [], + "source": [ + "tf = TESTS[\"gramacy_lee_2012\"]\n", + "n = 60\n", + "X, _, Xt, yt = design.dataset(tf, n, n_test=2000, seed=13)\n", + "sd_x = 0.02 + 0.4 * X[:, 0] # noise sd grows with x (in unit coordinates)\n", + "y = tf.unit(X) + np.random.default_rng(13).normal(0, sd_x)\n", + "k_known = lk.Kriging(y, X, \"matern5_2\", noise=sd_x ** 2, optim=\"BFGS10\")\n", + "k_nug = lk.Kriging(y, X, \"matern5_2\", noise=\"nugget\", optim=\"BFGS10\")\n", + "res_known = metrics.all_metrics(yt, *pred(k_known, Xt))\n", + "res_nug = metrics.all_metrics(yt, *pred(k_nug, Xt))\n", + "suite.metric(\"gramacy_lee_2012/hetero\", \"known variances\", **res_known)\n", + "suite.metric(\"gramacy_lee_2012/hetero\", \"nugget\", **res_nug)\n", + "suite.finite(\"hetero: predictions finite\", pred(k_known, Xt)[0])\n", + "suite.less(\"hetero: NLPD(known variances) <= NLPD(nugget)\", res_known[\"nlpd\"] - res_nug[\"nlpd\"], 0.05,\n", + " severity=\"warn\")\n", + "suite.close(\"noise(): returns the given variances\", np.ravel(k_known.noise()), sd_x ** 2, rtol=1e-12)\n", + "\n", + "u = np.linspace(0, 1, 400)[:, None]\n", + "fig, ax = plt.subplots(figsize=(10, 4))\n", + "ax.plot(u, tf.unit(u), \"k--\", label=\"truth\")\n", + "ax.errorbar(X[:, 0], y, yerr=2 * sd_x, fmt=\"k.\", alpha=0.4)\n", + "for k, lab in [(k_known, \"known variances\"), (k_nug, \"nugget\")]:\n", + " m, s = pred(k, u)\n", + " ax.plot(u, m, label=lab)\n", + " ax.fill_between(u[:, 0], m - 2 * s, m + 2 * s, alpha=0.15)\n", + "ax.legend();" + ] + }, + { + "cell_type": "markdown", + "id": "c008", + "metadata": {}, + "source": [ + "## Replicates\n", + "\n", + "r noisy replicates at the same point carry the same information as their mean\n", + "with variance σ²/r: fitting the replicates with `noise=σ²` or their means with\n", + "`noise=σ²/r` must give the same predictor at a fixed θ." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c009", + "metadata": {}, + "outputs": [], + "source": [ + "tf = TESTS[\"branin\"]\n", + "r, sd = 4, 2.0\n", + "X1, _, Xt, yt = design.dataset(tf, 15, n_test=300, seed=14)\n", + "Xr = np.repeat(X1, r, axis=0)\n", + "yr = tf.unit(Xr) + np.random.default_rng(14).normal(0, sd, len(Xr))\n", + "ybar = yr.reshape(-1, r).mean(axis=1)\n", + "k_rep = lk.Kriging(yr, Xr, \"matern5_2\", noise=np.full(len(yr), sd ** 2))\n", + "fixed = {\"theta\": np.asarray(k_rep.theta()).reshape(1, -1), \"is_theta_estim\": False,\n", + " \"sigma2\": float(k_rep.sigma2()), \"is_sigma2_estim\": False}\n", + "k_rep = lk.Kriging(yr, Xr, \"matern5_2\", noise=np.full(len(yr), sd ** 2), optim=\"none\", parameters=fixed)\n", + "k_avg = lk.Kriging(ybar, X1, \"matern5_2\", noise=np.full(len(ybar), sd ** 2 / r), optim=\"none\", parameters=fixed)\n", + "suite.close(\"replicates == averages with variance/r (mean)\", pred(k_rep, Xt)[0], pred(k_avg, Xt)[0],\n", + " rtol=1e-6, atol=1e-6 * np.std(yr))\n", + "suite.close(\"replicates == averages with variance/r (stdev)\", pred(k_rep, Xt)[1], pred(k_avg, Xt)[1],\n", + " rtol=1e-4, atol=1e-6 * np.std(yr))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c010", + "metadata": {}, + "outputs": [], + "source": [ + "suite.finalize()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/python/04_update_simulate.ipynb b/notebooks/python/04_update_simulate.ipynb new file mode 100644 index 0000000..44b13ff --- /dev/null +++ b/notebooks/python/04_update_simulate.ipynb @@ -0,0 +1,220 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "c000", + "metadata": {}, + "source": [ + "# 04 — `update`, `simulate`, `update_simulate`\n", + "\n", + "Sequential use of a fitted model (sequential design, Bayesian optimization):\n", + "\n", + "* `update(y_u, X_u, refit=False)` adds observations through a Cholesky\n", + " (Schur complement) update, keeping θ; β and σ² are re-estimated (profiled)\n", + " on the enlarged data. It must match a model built from scratch on all the\n", + " data with the same θ (`optim=\"none\"`, σ² and β estimated).\n", + "* `update(..., refit=True)` re-optimizes: it must match a fresh fit (up to the\n", + " optimizer: warn-level).\n", + "* `simulate(nsim, seed, X)` draws conditional paths: their empirical mean and\n", + " covariance must converge to `predict`'s mean and covariance; a seed makes\n", + " them reproducible; at the design points they reproduce y.\n", + "* `update_simulate(y_u, X_u)` conditions paths already drawn on new data: the\n", + " corrected paths must be distributed as fresh paths of the updated model.\n", + "\n", + "Functions: Branin (2D), Hartmann 3 (3D), Hartmann 6 (6D)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c001", + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "import pylibkriging as lk\n", + "\n", + "import lkbench\n", + "from lkbench import TESTS, design, compat\n", + "\n", + "suite = lkbench.Suite(\"py04_update_simulate\")\n", + "print(\"pylibkriging\", compat.version())\n", + "\n", + "\n", + "def pred(k, X, cov=False):\n", + " m, s, c, *_ = k.predict(X, return_stdev=True, return_cov=cov)\n", + " return np.ravel(m), np.ravel(s), np.asarray(c)\n", + "\n", + "\n", + "def fixed_params(k):\n", + " return {\"theta\": np.asarray(k.theta()).reshape(1, -1), \"is_theta_estim\": False,\n", + " \"sigma2\": float(k.sigma2()), \"is_sigma2_estim\": False}" + ] + }, + { + "cell_type": "markdown", + "id": "c002", + "metadata": {}, + "source": [ + "## `update` vs a fit from scratch" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c003", + "metadata": {}, + "outputs": [], + "source": [ + "for fname, n0, nu in [(\"branin\", 20, 5), (\"hartmann3\", 30, 10), (\"hartmann6\", 60, 20)]:\n", + " tf = TESTS[fname]\n", + " X, y, Xt, yt = design.dataset(tf, n0 + nu, n_test=300, seed=21)\n", + " for trend in [\"constant\", \"linear\"]:\n", + " case = f\"{fname}/{trend}\"\n", + " k = lk.Kriging(y[:n0], X[:n0], \"matern5_2\", regmodel=trend)\n", + " p = fixed_params(k)\n", + " k.update(y[n0:], X[n0:], refit=False)\n", + " ref = lk.Kriging(y, X, \"matern5_2\", regmodel=trend, optim=\"none\",\n", + " parameters={\"theta\": p[\"theta\"], \"is_theta_estim\": False})\n", + " m, s, _ = pred(k, Xt)\n", + " mr, sr, _ = pred(ref, Xt)\n", + " suite.close(f\"{case}: update(refit=False) mean == scratch fit, same theta\", m, mr, rtol=1e-6,\n", + " atol=1e-8 * np.std(y))\n", + " suite.close(f\"{case}: update(refit=False) stdev == scratch fit, same theta\", s, sr, rtol=1e-5,\n", + " atol=1e-8 * np.std(y))\n", + " suite.close(f\"{case}: update(refit=False) sigma2 == scratch fit sigma2\", k.sigma2(), ref.sigma2(),\n", + " rtol=1e-8)\n", + " suite.close(f\"{case}: update(refit=False) keeps theta\", np.ravel(k.theta()), np.ravel(p[\"theta\"]),\n", + " rtol=0, atol=0)\n", + " suite.true(f\"{case}: model holds n0+nu points\", np.asarray(k.X()).shape[0] == n0 + nu)\n", + " # refit=True vs a fresh fit (same optimizer from the same data, possibly another start)\n", + " k2 = lk.Kriging(y[:n0], X[:n0], \"matern5_2\", regmodel=trend)\n", + " k2.update(y[n0:], X[n0:], refit=True)\n", + " fresh = lk.Kriging(y, X, \"matern5_2\", regmodel=trend)\n", + " suite.close(f\"{case}: update(refit=True) LL == fresh fit LL\", k2.logLikelihood(), fresh.logLikelihood(),\n", + " rtol=1e-3, severity=\"warn\")" + ] + }, + { + "cell_type": "markdown", + "id": "c004", + "metadata": {}, + "source": [ + "Several small updates in a row equal one update with all the points." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c005", + "metadata": {}, + "outputs": [], + "source": [ + "tf = TESTS[\"hartmann3\"]\n", + "X, y, Xt, _ = design.dataset(tf, 40, n_test=200, seed=22)\n", + "ka = lk.Kriging(y[:20], X[:20], \"matern5_2\")\n", + "kb = lk.Kriging(y[:20], X[:20], \"matern5_2\")\n", + "for i in range(20, 40, 5):\n", + " ka.update(y[i:i + 5], X[i:i + 5], refit=False)\n", + "kb.update(y[20:], X[20:], refit=False)\n", + "suite.close(\"4 updates of 5 == 1 update of 20 (mean)\", pred(ka, Xt)[0], pred(kb, Xt)[0], rtol=1e-8, atol=1e-10)\n", + "suite.close(\"4 updates of 5 == 1 update of 20 (stdev)\", pred(ka, Xt)[1], pred(kb, Xt)[1], rtol=1e-6, atol=1e-10)" + ] + }, + { + "cell_type": "markdown", + "id": "c006", + "metadata": {}, + "source": [ + "## `simulate`: moments, seeds, conditioning" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c007", + "metadata": {}, + "outputs": [], + "source": [ + "for fname, n in [(\"branin\", 20), (\"hartmann3\", 30)]:\n", + " tf = TESTS[fname]\n", + " X, y, Xt, _ = design.dataset(tf, n, n_test=15, seed=23)\n", + " k = lk.Kriging(y, X, \"matern5_2\")\n", + " nsim = 5000\n", + " S = np.asarray(k.simulate(nsim, 123, Xt))\n", + " suite.true(f\"{fname}: simulate shape (q, nsim)\", S.shape == (len(Xt), nsim), detail=str(S.shape))\n", + " m, s, c = pred(k, Xt, cov=True)\n", + " z = (S.mean(axis=1) - m) / (s / np.sqrt(nsim))\n", + " suite.less(f\"{fname}: empirical mean of paths vs predict (max |z|)\", np.max(np.abs(z)), 5.0)\n", + " rel = np.linalg.norm(np.cov(S) - c) / np.linalg.norm(c)\n", + " suite.less(f\"{fname}: empirical covariance of paths vs predict (rel. Frobenius)\", rel, 0.1)\n", + " S2 = np.asarray(k.simulate(nsim, 123, Xt))\n", + " suite.close(f\"{fname}: same seed, same paths\", S2, S, rtol=0, atol=0)\n", + " S3 = np.asarray(k.simulate(nsim, 124, Xt))\n", + " suite.true(f\"{fname}: another seed, other paths\", not np.allclose(S3, S))\n", + " Sx = np.asarray(k.simulate(20, 1, X[:5]))\n", + " # up to the numerical jitter added to the conditional covariance (relative to the prior sd)\n", + " suite.close(f\"{fname}: paths reproduce y at the design points\", Sx, np.tile(y[:5, None], (1, 20)),\n", + " rtol=0, atol=1e-4 * np.sqrt(k.sigma2()))" + ] + }, + { + "cell_type": "markdown", + "id": "c008", + "metadata": {}, + "source": [ + "## `update_simulate`: paths conditioned on new data\n", + "\n", + "Paths drawn with `will_update=True`, then corrected by `update_simulate(y_u,\n", + "X_u)`, must have the moments of the model updated with (X_u, y_u)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c009", + "metadata": {}, + "outputs": [], + "source": [ + "tf = TESTS[\"branin\"]\n", + "X, y, Xt, _ = design.dataset(tf, 25, n_test=12, seed=24)\n", + "n0 = 20\n", + "k = lk.Kriging(y[:n0], X[:n0], \"matern5_2\")\n", + "p = fixed_params(k)\n", + "nsim = 5000\n", + "with suite.runs(\"update_simulate runs\"):\n", + " k.simulate(nsim, 7, Xt, will_update=True)\n", + " U = np.asarray(k.update_simulate(y[n0:], X[n0:]))\n", + " # paths drawn under (theta, sigma2) of the first model, then conditioned on all the data\n", + " ref = lk.Kriging(y, X, \"matern5_2\", optim=\"none\", parameters=p)\n", + " m, s, c = pred(ref, Xt, cov=True)\n", + " z = (U.mean(axis=1) - m) / (s / np.sqrt(nsim))\n", + " suite.less(\"update_simulate: mean of corrected paths vs updated model (max |z|)\", np.max(np.abs(z)), 5.0)\n", + " rel = np.linalg.norm(np.cov(U) - c) / np.linalg.norm(c)\n", + " suite.less(\"update_simulate: covariance vs updated model (rel. Frobenius)\", rel, 0.1)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c010", + "metadata": {}, + "outputs": [], + "source": [ + "suite.finalize()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/python/05_warpkriging.ipynb b/notebooks/python/05_warpkriging.ipynb new file mode 100644 index 0000000..89c0ecf --- /dev/null +++ b/notebooks/python/05_warpkriging.ipynb @@ -0,0 +1,286 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "c000", + "metadata": {}, + "source": [ + "# 05 — `WarpKriging`: non-stationarity, categorical and ordinal inputs\n", + "\n", + "`WarpKriging` learns a transform per input variable jointly with the GP\n", + "hyperparameters. Problems where a plain stationary GP is the wrong model:\n", + "\n", + "| case | function | warp |\n", + "|---|---|---|\n", + "| 1D non-stationary | Gramacy & Lee (2012): fast oscillation on the left, flat on the right | `kumaraswamy`, `knots(5)`, `neural_mono(8)`, `boxcox`, `affine` |\n", + "| 2D non-stationary | Gramacy & Lee (2008): a bump in a corner of [−2, 6]² | `kumaraswamy` × 2 |\n", + "| mixed continuous / categorical | Branin + a level-dependent shift and scale | `categorical(5,2)`, `none` |\n", + "| mixed continuous / ordinal | Hartmann 3 with x₁ observed on 6 ordered levels | `ordinal(6)` |\n", + "\n", + "Checks: all-`none` warping reproduces plain `Kriging`; a noise-free\n", + "`WarpKriging` interpolates; predictions are finite; and (warn-level) the warps\n", + "beat plain `Kriging` where they are meant to." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c001", + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "import pandas as pd\n", + "import matplotlib.pyplot as plt\n", + "import pylibkriging as lk\n", + "\n", + "import lkbench\n", + "from lkbench import TESTS, design, metrics, compat\n", + "\n", + "suite = lkbench.Suite(\"py05_warpkriging\")\n", + "print(\"pylibkriging\", compat.version())\n", + "if not compat.has(\"WarpKriging\"):\n", + " suite.skip(\"WarpKriging\", \"not available in this pylibkriging\")\n", + "\n", + "\n", + "def pred(k, X):\n", + " m, s, *_ = k.predict(X, return_stdev=True)\n", + " return np.ravel(m), np.ravel(s)\n", + "\n", + "\n", + "BASE_NLPD = {}\n", + "\n", + "\n", + "def evaluate(case, label, k, X, y, Xt, yt):\n", + " m, s = pred(k, Xt)\n", + " res = metrics.all_metrics(yt, m, s)\n", + " # calibration: a warp must not make the stdev collapse where the error is not small\n", + " if label.startswith(\"Kriging\"):\n", + " BASE_NLPD[case] = res[\"nlpd\"]\n", + " else:\n", + " suite.less(f\"{case}/{label}: NLPD - NLPD(Kriging) (overconfident stdev?)\", res[\"nlpd\"] - BASE_NLPD[case],\n", + " 2.0, severity=\"warn\", detail=f\"min stdev {np.min(s):.3g}, max |error|/stdev \"\n", + " f\"{np.max(np.abs(yt - m) / np.maximum(s, 1e-300)):.3g}\")\n", + " suite.metric(case, label, ll=float(k.logLikelihood()), **res)\n", + " suite.finite(f\"{case}/{label}: predictions finite\", m)\n", + " mX, _ = pred(k, X)\n", + " suite.less(f\"{case}/{label}: interpolates (max |m(X)-y| / sd(y))\", np.max(np.abs(mX - y)) / np.std(y), 1e-3,\n", + " severity=\"warn\")\n", + " return res" + ] + }, + { + "cell_type": "markdown", + "id": "c002", + "metadata": {}, + "source": [ + "## `none` warping == plain Kriging" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c003", + "metadata": {}, + "outputs": [], + "source": [ + "if compat.has(\"WarpKriging\"):\n", + " tf = TESTS[\"branin\"]\n", + " X, y, Xt, yt = design.dataset(tf, 30, n_test=500, seed=31)\n", + " for kernel in [\"gauss\", \"matern5_2\"]:\n", + " kp = lk.Kriging(y, X, kernel)\n", + " kw = lk.WarpKriging(y, X, [\"none\", \"none\"], kernel)\n", + " suite.close(f\"{kernel}: WarpKriging(none) LL == Kriging LL\", kw.logLikelihood(), kp.logLikelihood(),\n", + " rtol=1e-3, severity=\"warn\")\n", + " suite.close(f\"{kernel}: WarpKriging(none) mean == Kriging mean\", pred(kw, Xt)[0], pred(kp, Xt)[0],\n", + " rtol=1e-2, atol=1e-2 * np.std(y), severity=\"warn\")" + ] + }, + { + "cell_type": "markdown", + "id": "c004", + "metadata": {}, + "source": [ + "## 1D non-stationary: Gramacy & Lee (2012)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c005", + "metadata": {}, + "outputs": [], + "source": [ + "WARPS_1D = [\"affine\", \"boxcox\", \"kumaraswamy\", \"knots(5)\", \"neural_mono(8)\"]\n", + "if compat.has(\"WarpKriging\"):\n", + " tf = TESTS[\"gramacy_lee_2012\"]\n", + " X, y, Xt, yt = design.dataset(tf, 25, n_test=1000, seed=32)\n", + " X = X * 0.98 + 0.01; Xt = Xt * 0.98 + 0.01 # strictly inside (0,1) for boxcox/kumaraswamy\n", + " y, yt = tf.unit(X), tf.unit(Xt)\n", + " base = evaluate(\"gl2012\", \"Kriging\", lk.Kriging(y, X, \"matern5_2\"), X, y, Xt, yt)\n", + " fits = {}\n", + " for w in WARPS_1D:\n", + " with suite.runs(f\"gl2012/{w}: fit\"):\n", + " fits[w] = lk.WarpKriging(y, X, [w], \"matern5_2\")\n", + " evaluate(\"gl2012\", w, fits[w], X, y, Xt, yt)\n", + " t = suite.table().loc[\"gl2012\"]\n", + " best = t.drop(index=\"Kriging\")[\"rmse\"].min()\n", + " suite.less(\"gl2012: best warp RMSE / Kriging RMSE\", best / base[\"rmse\"], 1.0, severity=\"warn\")\n", + " u = np.linspace(0.01, 0.99, 400)[:, None]\n", + " fig, ax = plt.subplots(figsize=(11, 4))\n", + " ax.plot(u, tf.unit(u), \"k--\", lw=2, label=\"truth\")\n", + " ax.plot(u, pred(lk.Kriging(y, X, \"matern5_2\"), u)[0], label=\"Kriging\")\n", + " for w in [\"kumaraswamy\", \"knots(5)\", \"neural_mono(8)\"]:\n", + " if w in fits:\n", + " ax.plot(u, pred(fits[w], u)[0], label=w)\n", + " ax.plot(X, y, \"ko\"); ax.legend(); ax.set_title(\"Gramacy & Lee (2012), n=25\")\n", + " display(t.round(4))" + ] + }, + { + "cell_type": "markdown", + "id": "c006", + "metadata": {}, + "source": [ + "## 2D non-stationary: Gramacy & Lee (2008)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c007", + "metadata": {}, + "outputs": [], + "source": [ + "if compat.has(\"WarpKriging\"):\n", + " tf = TESTS[\"gramacy_lee_2008\"]\n", + " X, y, Xt, yt = design.dataset(tf, 40, n_test=2000, seed=33)\n", + " X = X * 0.98 + 0.01; Xt = Xt * 0.98 + 0.01\n", + " y, yt = tf.unit(X), tf.unit(Xt)\n", + " base = evaluate(\"gl2008\", \"Kriging\", lk.Kriging(y, X, \"matern5_2\"), X, y, Xt, yt)\n", + " with suite.runs(\"gl2008/kumaraswamy: fit\"):\n", + " res = evaluate(\"gl2008\", \"kumaraswamy\", lk.WarpKriging(y, X, [\"kumaraswamy\"] * 2, \"matern5_2\"),\n", + " X, y, Xt, yt)\n", + " suite.less(\"gl2008: RMSE(kumaraswamy) / RMSE(Kriging)\", res[\"rmse\"] / base[\"rmse\"], 1.0, severity=\"warn\")" + ] + }, + { + "cell_type": "markdown", + "id": "c008", + "metadata": {}, + "source": [ + "## Mixed continuous / categorical\n", + "\n", + "y = a_c · Branin(x₁, x₂) + b_c for a level c ∈ {0..4} with level-specific\n", + "(a_c, b_c): two pairs of levels are close, the fifth one is different. The\n", + "categorical variable is passed as its level index 0..L−1." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c009", + "metadata": {}, + "outputs": [], + "source": [ + "if compat.has(\"WarpKriging\"):\n", + " tf = TESTS[\"branin\"]\n", + " L = 5\n", + " a = np.array([1.0, 1.05, 0.6, 0.65, -0.5])\n", + " b = np.array([0.0, 5.0, 40.0, 45.0, 100.0])\n", + "\n", + " def mixed(Xc):\n", + " c = Xc[:, 0].astype(int)\n", + " return a[c] * tf.unit(Xc[:, 1:]) + b[c]\n", + "\n", + " def mixed_design(n, seed):\n", + " rng = np.random.default_rng(seed)\n", + " Xc = np.column_stack([np.arange(n) % L, design.lhs(n, 2, seed=seed)])\n", + " return Xc[rng.permutation(n)]\n", + "\n", + " X = mixed_design(60, 34); Xt = mixed_design(1000, 35)\n", + " y, yt = mixed(X), mixed(Xt)\n", + " base = evaluate(\"mixed\", \"Kriging (level as number)\", lk.Kriging(y, X, \"matern5_2\"), X, y, Xt, yt)\n", + " with suite.runs(\"mixed/categorical: fit\"):\n", + " kc = lk.WarpKriging(y, X, [f\"categorical({L},2)\", \"none\", \"none\"], \"matern5_2\")\n", + " res = evaluate(\"mixed\", \"categorical(5,2)\", kc, X, y, Xt, yt)\n", + " suite.less(\"mixed: RMSE(categorical) / RMSE(level as number)\", res[\"rmse\"] / base[\"rmse\"], 1.0,\n", + " severity=\"warn\")\n", + " suite.greater(\"mixed/categorical: Q2\", res[\"q2\"], 0.8, severity=\"warn\")" + ] + }, + { + "cell_type": "markdown", + "id": "c010", + "metadata": {}, + "source": [ + "## Mixed continuous / ordinal\n", + "\n", + "Hartmann 3 with x₁ only observed on 6 non-equally spaced ordered levels; the\n", + "model sees the level rank 0..5." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c011", + "metadata": {}, + "outputs": [], + "source": [ + "if compat.has(\"WarpKriging\"):\n", + " tf = TESTS[\"hartmann3\"]\n", + " levels = np.array([0.0, 0.05, 0.15, 0.4, 0.8, 1.0])\n", + "\n", + " def ordinal_design(n, seed):\n", + " U = design.lhs(n, 3, seed=seed)\n", + " r = np.minimum((U[:, 0] * 6).astype(int), 5)\n", + " return np.column_stack([r, U[:, 1:]]).astype(float)\n", + "\n", + " def f_ord(Xo):\n", + " return tf.unit(np.column_stack([levels[Xo[:, 0].astype(int)], Xo[:, 1:]]))\n", + "\n", + " X = ordinal_design(60, 36); Xt = ordinal_design(1000, 37)\n", + " y, yt = f_ord(X), f_ord(Xt)\n", + " base = evaluate(\"ordinal\", \"Kriging (rank as number)\", lk.Kriging(y, X / [5, 1, 1], \"matern5_2\"),\n", + " X / [5, 1, 1], y, Xt / [5, 1, 1], yt)\n", + " with suite.runs(\"ordinal/ordinal(6): fit\"):\n", + " ko = lk.WarpKriging(y, X, [\"ordinal(6)\", \"none\", \"none\"], \"matern5_2\")\n", + " res = evaluate(\"ordinal\", \"ordinal(6)\", ko, X, y, Xt, yt)\n", + " suite.less(\"ordinal: RMSE(ordinal) / RMSE(rank as number)\", res[\"rmse\"] / base[\"rmse\"], 1.1,\n", + " severity=\"warn\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c012", + "metadata": {}, + "outputs": [], + "source": [ + "suite.table().round(4)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c013", + "metadata": {}, + "outputs": [], + "source": [ + "suite.finalize()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/python/06_high_dimension_mlp.ipynb b/notebooks/python/06_high_dimension_mlp.ipynb new file mode 100644 index 0000000..0142185 --- /dev/null +++ b/notebooks/python/06_high_dimension_mlp.ipynb @@ -0,0 +1,212 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "c000", + "metadata": {}, + "source": [ + "# 06 — Moderate and high dimension: engineering functions, `normalize`, `MLPKriging`\n", + "\n", + "| function | d | character |\n", + "|---|---|---|\n", + "| OTL circuit | 6 | smooth, inputs on very different scales |\n", + "| Piston | 7 | smooth, strongly non-linear in a few inputs |\n", + "| Borehole | 8 | smooth, 2–3 dominant inputs, raw scales from 0.05 to 10⁵ |\n", + "| Robot arm | 8 | oscillating (angles): hard for a stationary GP |\n", + "| Dette & Pepelyshev | 8 | curved, logarithmic terms |\n", + "| Wing weight | 10 | smooth, nearly additive |\n", + "| Welch et al. (1992) | 20 | screening problem: few active inputs, interactions |\n", + "\n", + "Checks:\n", + "* **`normalize=True` on raw inputs** gives the same accuracy as an explicit\n", + " rescaling to [0, 1] (they are the same model up to the optimizer);\n", + " inputs in native units (Borehole spans 0.05 to 50 000) must not break a fit;\n", + "* Q² floors per function (regression detection);\n", + "* the fitted θ of Welch's **inactive** inputs are larger than those of the\n", + " active ones (screening ability, warn-level);\n", + "* `MLPKriging` (deep kernel) on a function with a low-dimensional structure:\n", + " finite predictions; with a multistart optimizer, a Q² close to plain\n", + " `Kriging`'s (error-level floor); the default single-start fit is compared\n", + " with the multistart one (warn-level)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c001", + "metadata": {}, + "outputs": [], + "source": [ + "import time\n", + "\n", + "import numpy as np\n", + "import pandas as pd\n", + "import pylibkriging as lk\n", + "\n", + "import lkbench\n", + "from lkbench import TESTS, design, metrics, compat\n", + "\n", + "suite = lkbench.Suite(\"py06_high_dimension_mlp\")\n", + "print(\"pylibkriging\", compat.version())\n", + "\n", + "\n", + "def pred(k, X):\n", + " m, s, *_ = k.predict(X, return_stdev=True)\n", + " return np.ravel(m), np.ravel(s)" + ] + }, + { + "cell_type": "markdown", + "id": "c002", + "metadata": {}, + "source": [ + "## Accuracy, unit-cube inputs vs native inputs with `normalize=True`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c003", + "metadata": {}, + "outputs": [], + "source": [ + "CASES = { # function: (n, Q2 floor on the unit cube)\n", + " \"otl_circuit\": (60, 0.999),\n", + " \"piston\": (70, 0.98),\n", + " \"borehole\": (80, 0.995),\n", + " \"robot_arm\": (160, 0.45),\n", + " \"dette_pepelyshev\": (80, 0.99),\n", + " \"wing_weight\": (100, 0.999),\n", + " \"welch\": (200, 0.98),\n", + "}\n", + "for fname, (n, floor) in CASES.items():\n", + " tf = TESTS[fname]\n", + " U, y, Ut, yt = design.dataset(tf, n, n_test=2000, seed=41)\n", + " t0 = time.time()\n", + " k = lk.Kriging(y, U, \"matern5_2\")\n", + " fit_s = time.time() - t0\n", + " res = metrics.all_metrics(yt, *pred(k, Ut))\n", + " suite.metric(fname, \"unit cube\", fit_s=fit_s, **res)\n", + " suite.greater(f\"{fname}: Q2 (unit cube, matern5_2)\", res[\"q2\"], floor)\n", + " # same problem in native units, rescaled internally\n", + " Xn, Xtn = tf.scale(U), tf.scale(Ut)\n", + " with suite.runs(f\"{fname}: fit on native inputs with normalize=True\"):\n", + " kn = lk.Kriging(y, Xn, \"matern5_2\", normalize=True)\n", + " res_n = metrics.all_metrics(yt, *pred(kn, Xtn))\n", + " suite.metric(fname, \"native + normalize\", **res_n)\n", + " suite.greater(f\"{fname}: Q2(normalize) - Q2(unit cube)\", res_n[\"q2\"] - res[\"q2\"], -0.02)\n", + "tab = suite.table()\n", + "tab.round(4)" + ] + }, + { + "cell_type": "markdown", + "id": "c004", + "metadata": {}, + "source": [ + "## Screening: Welch et al. (1992)\n", + "\n", + "Inputs 8 and 16 do not appear in the function; inputs 1, 4, 5, 12, 19, 20 are\n", + "the most influential ones. A small θ_j (short correlation length) means an\n", + "influential input, so the inactive inputs should get the largest θ." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c005", + "metadata": {}, + "outputs": [], + "source": [ + "tf = TESTS[\"welch\"]\n", + "U, y, _, _ = design.dataset(tf, 200, seed=42)\n", + "k = lk.Kriging(y, U, \"matern5_2\")\n", + "theta = np.ravel(k.theta())\n", + "inactive = [7, 15] # 0-based indices of x8, x16\n", + "active = [0, 3, 4, 11, 18, 19] # x1, x4, x5, x12, x19, x20\n", + "suite.greater(\"welch: min theta(inactive) / max theta(active)\", theta[inactive].min() / theta[active].max(), 1.0,\n", + " severity=\"warn\")\n", + "pd.Series(theta, index=[f\"x{i + 1}\" for i in range(20)]).round(3).to_frame(\"theta\").T" + ] + }, + { + "cell_type": "markdown", + "id": "c006", + "metadata": {}, + "source": [ + "## `MLPKriging`: deep kernel on a low-dimensional structure\n", + "\n", + "f(x) = Branin(Wx) for x ∈ [0,1]¹⁰ and a fixed 2 × 10 projection W: the\n", + "function only varies in a 2D subspace, which a joint feature map can find\n", + "whereas a per-axis kernel cannot." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c007", + "metadata": {}, + "outputs": [], + "source": [ + "if compat.has(\"MLPKriging\"):\n", + " rng = np.random.default_rng(43)\n", + " W = rng.normal(size=(2, 10))\n", + " W /= np.abs(W).sum(axis=1, keepdims=True)\n", + " tfb = TESTS[\"branin\"]\n", + "\n", + " def projected(X):\n", + " Z = X @ W.T\n", + " Z = (Z - (W.clip(max=0).sum(axis=1))) / np.abs(W).sum(axis=1) # back to [0,1]^2\n", + " return tfb.unit(Z)\n", + "\n", + " X = design.lhs(120, 10, seed=44); Xt = design.uniform(2000, 10, seed=45)\n", + " y, yt = projected(X), projected(Xt)\n", + " kp = lk.Kriging(y, X, \"matern5_2\")\n", + " res_k = metrics.all_metrics(yt, *pred(kp, Xt))\n", + " suite.metric(\"projected_branin_10d\", \"Kriging\", **res_k)\n", + " for act in [\"tanh\", \"selu\"]:\n", + " for optim in [\"BFGS\", \"BFGS10\"]:\n", + " case = f\"MLPKriging({act}, {optim})\"\n", + " with suite.runs(f\"{case}: fit\"):\n", + " t0 = time.time()\n", + " km = lk.MLPKriging(y, X, hidden_dims=[16, 8], d_out=2, activation=act, kernel=\"matern5_2\",\n", + " optim=optim)\n", + " fit_s = time.time() - t0\n", + " m, s = pred(km, Xt)\n", + " suite.finite(f\"{case}: predictions finite\", m)\n", + " res = metrics.all_metrics(yt, m, s)\n", + " suite.metric(\"projected_branin_10d\", f\"MLPKriging {act} {optim}\", fit_s=fit_s,\n", + " ll=float(km.logLikelihood()), **res)\n", + " if optim == \"BFGS10\":\n", + " suite.greater(f\"{case}: Q2\", res[\"q2\"], 0.8)\n", + " else:\n", + " suite.greater(f\"{case}: Q2 - Q2(Kriging)\", res[\"q2\"] - res_k[\"q2\"], -0.1, severity=\"warn\")\n", + " display(suite.table().loc[\"projected_branin_10d\"].round(4))\n", + "else:\n", + " suite.skip(\"MLPKriging\", \"not available in this pylibkriging\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c008", + "metadata": {}, + "outputs": [], + "source": [ + "suite.finalize()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/python/07_scalability.ipynb b/notebooks/python/07_scalability.ipynb new file mode 100644 index 0000000..822d335 --- /dev/null +++ b/notebooks/python/07_scalability.ipynb @@ -0,0 +1,219 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "c000", + "metadata": {}, + "source": [ + "# 07 — Large n: Vecchia, Nyström, NestedKriging, subset of data\n", + "\n", + "The exact fit costs O(n³). libKriging offers:\n", + "\n", + "* `method=\"vecchia(m)\"`: Vecchia likelihood with m conditioning neighbours\n", + " (local, recommended in low dimension);\n", + "* `method=\"nystrom(k)\"`: low-rank Nyström approximation with k landmarks\n", + " (global);\n", + "* `NestedKriging`: a partition into groups, one sub-model per group,\n", + " aggregated predictions (`NK`, `PoE`, `gPoE`, `BCM`, `rBCM`);\n", + "* `Kriging.subsetOfData`: a representative subset before an exact fit.\n", + "\n", + "Problems: Hartmann 3 (3D, local structure: Vecchia's ground) and the OTL\n", + "circuit (6D, smooth: Nyström's ground), n = 1500 (exact fit still affordable,\n", + "for reference) and n = 5000 (approximations only).\n", + "\n", + "Checks:\n", + "* **limit cases**: Vecchia with m = n − 1 neighbours *is* the exact likelihood;\n", + "* the method is explicit at prediction time: `predict(method=\"exact\")` on a\n", + " Nyström/Vecchia fit raises, `method=` gives finite predictions;\n", + "* accuracy of each approximation close to the exact fit (warn-level gaps,\n", + " error-level floors) and faster than the exact fit at n = 1500 (warn-level);\n", + "* `NestedKriging` `NK` interpolates the data (it is a proper kriging\n", + " predictor); the PoE family does not need to." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c001", + "metadata": {}, + "outputs": [], + "source": [ + "import time\n", + "\n", + "import numpy as np\n", + "import pandas as pd\n", + "import pylibkriging as lk\n", + "\n", + "import lkbench\n", + "from lkbench import TESTS, design, metrics, compat\n", + "\n", + "suite = lkbench.Suite(\"py07_scalability\")\n", + "print(\"pylibkriging\", compat.version())\n", + "HAS_METHOD = compat.has(\"method\")\n", + "if not HAS_METHOD:\n", + " suite.skip(\"method=vecchia/nystrom\", \"pylibkriging without the method= argument\")\n", + "\n", + "\n", + "def pred(k, X, **kw):\n", + " m, s, *_ = k.predict(X, return_stdev=True, **kw)\n", + " return np.ravel(m), np.ravel(s)\n", + "\n", + "\n", + "def timed(f):\n", + " t0 = time.time()\n", + " out = f()\n", + " return out, time.time() - t0" + ] + }, + { + "cell_type": "markdown", + "id": "c002", + "metadata": {}, + "source": [ + "## Limit case: Vecchia with all the previous points as neighbours is exact" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c003", + "metadata": {}, + "outputs": [], + "source": [ + "if HAS_METHOD:\n", + " tf = TESTS[\"branin\"]\n", + " X, y, Xt, _ = design.dataset(tf, 40, n_test=100, seed=51)\n", + " ke = lk.Kriging(y, X, \"matern5_2\")\n", + " fixed = {\"theta\": np.asarray(ke.theta()).reshape(1, -1), \"is_theta_estim\": False}\n", + " kv = lk.Kriging(y, X, \"matern5_2\", optim=\"none\", parameters=fixed, method=f\"vecchia({len(y) - 1})\")\n", + " suite.close(\"vecchia(n-1) LL == exact LL (same theta)\", kv.logLikelihood(method=\"vecchia\"),\n", + " ke.logLikelihood(), rtol=1e-6)\n", + " suite.close(\"vecchia(n-1) predict == exact predict\", pred(kv, Xt, method=\"vecchia\")[0], pred(ke, Xt)[0],\n", + " rtol=1e-5, atol=1e-6 * np.std(y), severity=\"warn\")\n", + " kn = lk.Kriging(y, X, \"matern5_2\", optim=\"none\", parameters=fixed, method=f\"nystrom({len(y)})\")\n", + " suite.close(\"nystrom(n) LL == exact LL (same theta)\", kn.logLikelihood(method=\"nystrom\"), ke.logLikelihood(),\n", + " rtol=1e-3, severity=\"warn\")\n", + " # the method must be explicit after an approximate fit\n", + " suite.raises(\"nystrom fit: predict(method='exact') raises\", lambda: kn.predict(Xt, return_stdev=True,\n", + " method=\"exact\"))\n", + " suite.true(\"method() reports the fit method\", kv.method().startswith(\"vecchia\") and\n", + " kn.method().startswith(\"nystrom\"), detail=f\"{kv.method()}, {kn.method()}\")" + ] + }, + { + "cell_type": "markdown", + "id": "c004", + "metadata": {}, + "source": [ + "## Accuracy and cost" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c005", + "metadata": {}, + "outputs": [], + "source": [ + "CASES = {\"hartmann3\": (1500, 5000), \"otl_circuit\": (1500, 5000)}\n", + "FLOORS = {\"hartmann3\": 0.98, \"otl_circuit\": 0.99}\n", + "for fname, (n_mid, n_big) in CASES.items():\n", + " tf = TESTS[fname]\n", + " X, y, Xt, yt = design.dataset(tf, n_big, n_test=2000, seed=52)\n", + " Xm, ym = X[:n_mid], y[:n_mid]\n", + " ke, t_exact = timed(lambda: lk.Kriging(ym, Xm, \"matern5_2\"))\n", + " res_e = metrics.all_metrics(yt, *pred(ke, Xt))\n", + " suite.metric(f\"{fname}/n={n_mid}\", \"exact\", fit_s=t_exact, **res_e)\n", + " suite.greater(f\"{fname}/n={n_mid}: Q2 exact\", res_e[\"q2\"], FLOORS[fname])\n", + " variants = {}\n", + " if HAS_METHOD:\n", + " variants.update({\n", + " \"vecchia(30)\": lambda Xa, ya: (lk.Kriging(ya, Xa, \"matern5_2\", method=\"vecchia(30)\"), \"vecchia\"),\n", + " \"nystrom(100)\": lambda Xa, ya: (lk.Kriging(ya, Xa, \"matern5_2\", method=\"nystrom(100)\"), \"nystrom\"),\n", + " })\n", + " if compat.has(\"NestedKriging\"):\n", + " variants[\"nested NK (20 groups)\"] = lambda Xa, ya: (\n", + " lk.NestedKriging(ya, Xa, \"matern5_2\", nb_groups=20, aggregation=\"NK\", seed=1), None)\n", + " variants[\"nested rBCM (20 groups)\"] = lambda Xa, ya: (\n", + " lk.NestedKriging(ya, Xa, \"matern5_2\", nb_groups=20, aggregation=\"rBCM\", seed=1), None)\n", + " if compat.has(\"subsetOfData\"):\n", + " def sod(Xa, ya):\n", + " idx = np.ravel(lk.Kriging.subsetOfData(Xa, n_max=500, method=\"kmeans\", seed=1)).astype(int)\n", + " return lk.Kriging(ya[idx], Xa[idx], \"matern5_2\"), None\n", + " variants[\"subset of data (500)\"] = sod\n", + " for label, make in variants.items():\n", + " for n in (n_mid, n_big):\n", + " case = f\"{fname}/n={n}\"\n", + " with suite.runs(f\"{case}/{label}: fit + predict\"):\n", + " (k, meth), t = timed(lambda: make(X[:n], y[:n]))\n", + " m, s = pred(k, Xt, method=meth) if meth else (\n", + " lambda o: (np.ravel(o[0]), np.ravel(o[1])))(k.predict(Xt, return_stdev=True))\n", + " suite.finite(f\"{case}/{label}: finite predictions\", m)\n", + " res = metrics.all_metrics(yt, m, s)\n", + " suite.metric(case, label, fit_s=t, **res)\n", + " suite.greater(f\"{case}/{label}: Q2\", res[\"q2\"], FLOORS[fname] - 0.05)\n", + " if n == n_mid:\n", + " suite.greater(f\"{case}/{label}: Q2 - Q2(exact)\", res[\"q2\"] - res_e[\"q2\"], -0.02,\n", + " severity=\"warn\")\n", + " suite.less(f\"{case}/{label}: fit time / exact fit time\", t / t_exact, 1.0, severity=\"warn\")\n", + "suite.table().round(4)" + ] + }, + { + "cell_type": "markdown", + "id": "c006", + "metadata": {}, + "source": [ + "## `NestedKriging` aggregations: interpolation and variance" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c007", + "metadata": {}, + "outputs": [], + "source": [ + "if compat.has(\"NestedKriging\"):\n", + " tf = TESTS[\"branin\"]\n", + " X, y, Xt, yt = design.dataset(tf, 200, n_test=500, seed=53)\n", + " for agg in [\"NK\", \"PoE\", \"gPoE\", \"BCM\", \"rBCM\"]:\n", + " with suite.runs(f\"nested {agg}: fit\"):\n", + " k = lk.NestedKriging(y, X, \"matern5_2\", nb_groups=8, aggregation=agg, seed=2)\n", + " m, s = (np.ravel(a) for a in k.predict(Xt, return_stdev=True)[:2])\n", + " suite.true(f\"nested {agg}: stdev >= 0\", np.all(s >= 0))\n", + " suite.metric(\"branin/nested\", agg, **metrics.all_metrics(yt, m, s))\n", + " if agg == \"NK\":\n", + " mX = np.ravel(k.predict(X, return_stdev=True)[0])\n", + " suite.less(\"nested NK: interpolates (max |m(X)-y| / sd(y))\", np.max(np.abs(mX - y)) / np.std(y),\n", + " 1e-4)\n", + " display(suite.table().loc[\"branin/nested\"].round(4))\n", + " suite.raises(\"nested NK with a linear trend is rejected\",\n", + " lambda: lk.NestedKriging(y, X, \"matern5_2\", nb_groups=4, aggregation=\"NK\", regmodel=\"linear\"),\n", + " severity=\"warn\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c008", + "metadata": {}, + "outputs": [], + "source": [ + "suite.finalize()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/python/08_api_persistence.ipynb b/notebooks/python/08_api_persistence.ipynb new file mode 100644 index 0000000..f7ae830 --- /dev/null +++ b/notebooks/python/08_api_persistence.ipynb @@ -0,0 +1,262 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "c000", + "metadata": {}, + "source": [ + "# 08 — API: save/load, pickle, copy, accessors, scikit-learn estimators\n", + "\n", + "Persistence and introspection, for every model class:\n", + "\n", + "* `save` → `load` (class-specific and generic `pylibkriging.load`) gives a model\n", + " with **identical** predictions and hyperparameters;\n", + "* `pickle` / `copy` round trips, and a copy is independent of the original\n", + " (updating one leaves the other untouched);\n", + "* accessors mirror the constructor arguments (`kernel()`, `regmodel()`,\n", + " `optim()`, `objective()`, `X()`, `y()`, ...);\n", + "* scikit-learn estimators (`pylibkriging.sklearn`): `fit`/`predict`,\n", + " `get_params`/`set_params`/`clone`, use inside `Pipeline` and `GridSearchCV`,\n", + " same predictions as the native class." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c001", + "metadata": {}, + "outputs": [], + "source": [ + "import copy\n", + "import pickle\n", + "import tempfile\n", + "from pathlib import Path\n", + "\n", + "import numpy as np\n", + "import pylibkriging as lk\n", + "\n", + "import lkbench\n", + "from lkbench import TESTS, design, compat\n", + "\n", + "suite = lkbench.Suite(\"py08_api_persistence\")\n", + "print(\"pylibkriging\", compat.version())\n", + "tmp = Path(tempfile.mkdtemp())\n", + "\n", + "tf = TESTS[\"hartmann3\"]\n", + "X, y, Xt, yt = design.dataset(tf, 40, n_test=100, seed=61)\n", + "\n", + "\n", + "def mean_sd(k, Xn):\n", + " out = k.predict(Xn, return_stdev=True)\n", + " return np.ravel(out[0]), np.ravel(out[1])\n", + "\n", + "\n", + "MODELS = {\n", + " \"Kriging\": lambda: lk.Kriging(y, X, \"matern5_2\", regmodel=\"linear\"),\n", + " \"Kriging nugget\": lambda: lk.Kriging(y, X, \"matern3_2\", noise=\"nugget\"),\n", + " \"Kriging noise vector\": lambda: lk.Kriging(y, X, \"gauss\", noise=np.full(len(y), 1e-4)),\n", + " \"Kriging LOO\": lambda: lk.Kriging(y, X, \"matern5_2\", objective=\"LOO\"),\n", + " \"Kriging LMP\": lambda: lk.Kriging(y, X, \"matern5_2\", objective=\"LMP\"),\n", + "}\n", + "if compat.has(\"method\"):\n", + " MODELS[\"Kriging vecchia\"] = lambda: lk.Kriging(y, X, \"matern5_2\", method=\"vecchia(10)\")\n", + " MODELS[\"Kriging nystrom\"] = lambda: lk.Kriging(y, X, \"matern5_2\", method=\"nystrom(20)\")\n", + "if compat.has(\"WarpKriging\"):\n", + " MODELS[\"WarpKriging\"] = lambda: lk.WarpKriging(y, X, [\"kumaraswamy\", \"none\", \"affine\"], \"matern5_2\")\n", + "if compat.has(\"MLPKriging\"):\n", + " MODELS[\"MLPKriging\"] = lambda: lk.MLPKriging(y, X, hidden_dims=[8], d_out=2, activation=\"tanh\",\n", + " kernel=\"matern5_2\")\n", + "if compat.has(\"NestedKriging\"):\n", + " MODELS[\"NestedKriging\"] = lambda: lk.NestedKriging(y, X, \"matern5_2\", nb_groups=4, seed=1)\n", + "\n", + "\n", + "def predict_any(k, Xn):\n", + " meth = k.method() if hasattr(k, \"method\") and isinstance(k, lk.Kriging) else None\n", + " if meth and meth != \"exact\":\n", + " out = k.predict(Xn, return_stdev=True, method=meth.split(\"(\")[0])\n", + " return np.ravel(out[0]), np.ravel(out[1])\n", + " return mean_sd(k, Xn)" + ] + }, + { + "cell_type": "markdown", + "id": "c002", + "metadata": {}, + "source": [ + "## save / load, pickle, copy" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c003", + "metadata": {}, + "outputs": [], + "source": [ + "for name, make in MODELS.items():\n", + " with suite.runs(f\"{name}: fit\"):\n", + " k = make()\n", + " m0, s0 = predict_any(k, Xt)\n", + " cls = type(k)\n", + " path = tmp / f\"{name.replace(' ', '_')}.json\"\n", + " if not hasattr(k, \"save\"):\n", + " suite.skip(f\"{name}: save + load\", \"no save() in this pylibkriging\")\n", + " continue\n", + " with suite.runs(f\"{name}: save + load\"):\n", + " k.save(str(path))\n", + " k1 = cls.load(str(path))\n", + " m1, s1 = predict_any(k1, Xt)\n", + " suite.close(f\"{name}: load(save(k)) predicts the same mean\", m1, m0, rtol=1e-12, atol=1e-12)\n", + " suite.close(f\"{name}: load(save(k)) predicts the same stdev\", s1, s0, rtol=1e-10, atol=1e-12)\n", + " if hasattr(k, \"theta\"):\n", + " suite.close(f\"{name}: same theta after load\", np.ravel(k1.theta()), np.ravel(k.theta()),\n", + " rtol=1e-14, atol=0)\n", + " if hasattr(lk, \"load\") and (compat.has(\"load(noise)\") or \"noise\" not in name and \"nugget\" not in name):\n", + " with suite.runs(f\"{name}: generic pylibkriging.load\"):\n", + " k2 = lk.load(str(path))\n", + " suite.true(f\"{name}: generic load returns {cls.__name__}\", type(k2).__name__ == cls.__name__,\n", + " detail=type(k2).__name__)\n", + " suite.close(f\"{name}: generic load predicts the same\", predict_any(k2, Xt)[0], m0, rtol=1e-12,\n", + " atol=1e-12)\n", + " if not compat.has(\"pickle\"):\n", + " suite.skip(f\"{name}: pickle / deepcopy\", \"models not picklable in this pylibkriging\")\n", + " continue\n", + " with suite.runs(f\"{name}: pickle round trip\"):\n", + " k3 = pickle.loads(pickle.dumps(k))\n", + " suite.close(f\"{name}: pickle predicts the same\", predict_any(k3, Xt)[0], m0, rtol=1e-12, atol=1e-12)\n", + " with suite.runs(f\"{name}: deepcopy\"):\n", + " k4 = copy.deepcopy(k)\n", + " suite.close(f\"{name}: deepcopy predicts the same\", predict_any(k4, Xt)[0], m0, rtol=1e-12, atol=1e-12)" + ] + }, + { + "cell_type": "markdown", + "id": "c004", + "metadata": {}, + "source": [ + "A copy is independent: updating it leaves the original model unchanged." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c005", + "metadata": {}, + "outputs": [], + "source": [ + "k = lk.Kriging(y[:30], X[:30], \"matern5_2\")\n", + "m_before = mean_sd(k, Xt)[0]\n", + "kc = k.copy() if hasattr(k, \"copy\") else copy.deepcopy(k)\n", + "kc.update(y[30:], X[30:], refit=False)\n", + "suite.close(\"update on a copy leaves the original unchanged\", mean_sd(k, Xt)[0], m_before, rtol=0, atol=0)\n", + "suite.true(\"the updated copy did change\", not np.allclose(mean_sd(kc, Xt)[0], m_before))" + ] + }, + { + "cell_type": "markdown", + "id": "c006", + "metadata": {}, + "source": [ + "## Accessors mirror the constructor" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c007", + "metadata": {}, + "outputs": [], + "source": [ + "k = lk.Kriging(y, X, \"matern3_2\", regmodel=\"linear\", normalize=False, optim=\"BFGS\", objective=\"LMP\")\n", + "for acc, expected in [(\"kernel\", \"matern3_2\"), (\"regmodel\", \"linear\"), (\"optim\", \"BFGS\"), (\"objective\", \"LMP\"),\n", + " (\"normalize\", False)]:\n", + " suite.true(f\"Kriging.{acc}() == {expected!r}\", getattr(k, acc)() == expected, detail=str(getattr(k, acc)()))\n", + "suite.close(\"Kriging.X() is the design\", np.asarray(k.X()), X, rtol=0, atol=0)\n", + "suite.close(\"Kriging.y() is the response\", np.ravel(k.y()), y, rtol=0, atol=0)\n", + "# with normalize=True: do the accessors return the data as given, or as rescaled internally?\n", + "kn = lk.Kriging(y, X * 10, \"matern3_2\", normalize=True)\n", + "suite.true(\"Kriging.normalize() == True\", kn.normalize() is True or kn.normalize() == 1)\n", + "suite.close(\"normalize=True: X() is the design as given\", np.asarray(kn.X()), X * 10, rtol=1e-12, atol=0,\n", + " severity=\"warn\")\n", + "suite.close(\"normalize=True: y() is the response as given\", np.ravel(kn.y()), y, rtol=1e-12, atol=0,\n", + " severity=\"warn\")\n", + "suite.true(\"Kriging.beta() has 1 + d coefficients (linear trend)\", np.ravel(k.beta()).size == 1 + X.shape[1])\n", + "suite.true(\"summary() is a non-empty string\", isinstance(k.summary(), str) and len(k.summary()) > 20)\n", + "suite.true(\"is_theta_estim() after an optimized fit\", bool(k.is_theta_estim()))" + ] + }, + { + "cell_type": "markdown", + "id": "c008", + "metadata": {}, + "source": [ + "## scikit-learn estimators" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c009", + "metadata": {}, + "outputs": [], + "source": [ + "if compat.has(\"sklearn\"):\n", + " from sklearn.base import clone\n", + " from sklearn.model_selection import GridSearchCV, cross_val_score\n", + " from sklearn.pipeline import make_pipeline\n", + " from sklearn.preprocessing import MinMaxScaler\n", + " from pylibkriging.sklearn import KrigingRegressor\n", + "\n", + " est = KrigingRegressor(kernel=\"matern5_2\").fit(X, y)\n", + " m_sk, s_sk = est.predict(Xt, return_std=True)\n", + " suite.true(\"KrigingRegressor.predict returns 1-D arrays (scikit-learn convention)\",\n", + " np.ndim(m_sk) == 1 and np.ndim(s_sk) == 1, detail=f\"shapes {np.shape(m_sk)}, {np.shape(s_sk)}\",\n", + " severity=\"warn\")\n", + " m_sk, s_sk = np.ravel(m_sk), np.ravel(s_sk)\n", + " m_nat, s_nat = mean_sd(lk.Kriging(y, X, \"matern5_2\"), Xt)\n", + " suite.close(\"KrigingRegressor == Kriging (mean)\", m_sk, m_nat, rtol=1e-8, atol=1e-10)\n", + " suite.close(\"KrigingRegressor == Kriging (stdev)\", s_sk, s_nat, rtol=1e-6, atol=1e-10)\n", + " est2 = clone(est).set_params(kernel=\"gauss\")\n", + " suite.true(\"clone + set_params\", est2.get_params()[\"kernel\"] == \"gauss\" and est.get_params()[\"kernel\"] ==\n", + " \"matern5_2\")\n", + " with suite.runs(\"Pipeline(MinMaxScaler, KrigingRegressor)\"):\n", + " pipe = make_pipeline(MinMaxScaler(), KrigingRegressor(kernel=\"matern5_2\")).fit(tf.scale(X), y)\n", + " suite.greater(\"pipeline Q2 on native inputs\", pipe.score(tf.scale(Xt), yt), 0.8)\n", + " with suite.runs(\"GridSearchCV over kernels\"):\n", + " gs = GridSearchCV(KrigingRegressor(), {\"kernel\": [\"gauss\", \"matern3_2\", \"matern5_2\"]}, cv=3).fit(X, y)\n", + " suite.true(\"GridSearchCV picked a kernel\", gs.best_params_[\"kernel\"] in (\"gauss\", \"matern3_2\", \"matern5_2\"),\n", + " detail=str(gs.best_params_))\n", + " with suite.runs(\"cross_val_score\"):\n", + " sc = cross_val_score(KrigingRegressor(kernel=\"matern5_2\"), X, y, cv=4)\n", + " suite.finite(\"cross_val_score finite\", sc)\n", + " with suite.runs(\"sample_y\"):\n", + " S = est.sample_y(Xt[:10], n_samples=5, random_state=1)\n", + " suite.true(\"sample_y shape (q, n_samples)\", np.shape(S) == (10, 5), detail=str(np.shape(S)))\n", + "else:\n", + " suite.skip(\"sklearn estimators\", \"pylibkriging.sklearn or scikit-learn not available\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c010", + "metadata": {}, + "outputs": [], + "source": [ + "suite.finalize()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/python/09_robustness.ipynb b/notebooks/python/09_robustness.ipynb new file mode 100644 index 0000000..337faf6 --- /dev/null +++ b/notebooks/python/09_robustness.ipynb @@ -0,0 +1,260 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "c000", + "metadata": {}, + "source": [ + "# 09 — Robustness: degenerate and adversarial inputs\n", + "\n", + "What a user eventually feeds a surrogate model, and what must happen:\n", + "\n", + "| input | expected behaviour |\n", + "|---|---|\n", + "| NaN / Inf in X or y | rejected with an exception (never a silent NaN model) |\n", + "| X and y with inconsistent sizes | rejected |\n", + "| a wrong number of warping specs | rejected |\n", + "| duplicated design points, noise-free | either rejected or a finite model (no NaN, no crash) |\n", + "| duplicated design points with `noise=\"nugget\"` | a finite model |\n", + "| constant response y | a finite model predicting the constant |\n", + "| very few points (n = 2, 3) | a finite model |\n", + "| very clustered points (1e-8 apart) with the `gauss` kernel | finite predictions (ill-conditioning) |\n", + "| inputs scaled by 1e6 or 1e-6 with `normalize=True` | the same predictions as on [0,1] |\n", + "| y scaled by 1e8 / shifted by 1e6 | predictions scaled / shifted accordingly (equivariance) |\n", + "| predicting far outside the design | the mean reverts to the trend, the stdev to (slightly above) √σ² |\n", + "| many prediction points | cost linear in their number (mean and stdev only need O(n·q), not O(q²)) |" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c001", + "metadata": {}, + "outputs": [], + "source": [ + "import time\n", + "\n", + "import numpy as np\n", + "import pylibkriging as lk\n", + "\n", + "import lkbench\n", + "from lkbench import TESTS, design, compat\n", + "\n", + "suite = lkbench.Suite(\"py09_robustness\")\n", + "print(\"pylibkriging\", compat.version())\n", + "tf = TESTS[\"branin\"]\n", + "X, y, Xt, yt = design.dataset(tf, 20, n_test=200, seed=71)\n", + "\n", + "\n", + "def pred(k, Xn):\n", + " m, s, *_ = k.predict(Xn, return_stdev=True)\n", + " return np.ravel(m), np.ravel(s)\n", + "\n", + "\n", + "def finite_model(check, make, Xn=Xt, severity=\"error\"):\n", + " \"\"\"The model either raises (acceptable) or predicts finite values.\"\"\"\n", + " try:\n", + " k = make()\n", + " except Exception as exc: # noqa: BLE001\n", + " suite.true(f\"{check}: rejected or finite\", True, detail=f\"rejected: {type(exc).__name__}\")\n", + " return None\n", + " m, s = pred(k, Xn)\n", + " suite.true(f\"{check}: rejected or finite\", np.all(np.isfinite(m)) and np.all(np.isfinite(s)),\n", + " detail=f\"non-finite: {np.sum(~np.isfinite(m))} mean, {np.sum(~np.isfinite(s))} stdev\",\n", + " severity=severity)\n", + " return k" + ] + }, + { + "cell_type": "markdown", + "id": "c002", + "metadata": {}, + "source": [ + "## Invalid inputs are rejected" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c003", + "metadata": {}, + "outputs": [], + "source": [ + "Xn = X.copy(); Xn[3, 1] = np.nan\n", + "yn = y.copy(); yn[5] = np.nan\n", + "yi = y.copy(); yi[5] = np.inf\n", + "suite.raises(\"NaN in X is rejected\", lambda: lk.Kriging(y, Xn, \"matern5_2\"))\n", + "suite.raises(\"NaN in y is rejected\", lambda: lk.Kriging(yn, X, \"matern5_2\"))\n", + "suite.raises(\"Inf in y is rejected\", lambda: lk.Kriging(yi, X, \"matern5_2\"))\n", + "suite.raises(\"len(y) != nrow(X) is rejected\", lambda: lk.Kriging(y[:-1], X, \"matern5_2\"))\n", + "suite.raises(\"unknown kernel is rejected\", lambda: lk.Kriging(y, X, \"matern7_2\"))\n", + "suite.raises(\"unknown trend is rejected\", lambda: lk.Kriging(y, X, \"matern5_2\", regmodel=\"cubic\"))\n", + "suite.raises(\"unknown objective is rejected\", lambda: lk.Kriging(y, X, \"matern5_2\", objective=\"XYZ\"))\n", + "suite.raises(\"noise vector of the wrong length is rejected\",\n", + " lambda: lk.Kriging(y, X, \"matern5_2\", noise=np.ones(len(y) - 1)))\n", + "suite.raises(\"negative noise variance is rejected\",\n", + " lambda: lk.Kriging(y, X, \"matern5_2\", noise=-np.ones(len(y))), severity=\"warn\")\n", + "k = lk.Kriging(y, X, \"matern5_2\")\n", + "suite.raises(\"predict with the wrong number of columns is rejected\", lambda: k.predict(np.ones((3, 3)), True))\n", + "if compat.has(\"WarpKriging\"):\n", + " suite.raises(\"WarpKriging with 1 spec for 2 columns is rejected\", lambda: lk.WarpKriging(y, X, [\"none\"], \"gauss\"))\n", + " suite.raises(\"WarpKriging with an unknown spec is rejected\",\n", + " lambda: lk.WarpKriging(y, X, [\"none\", \"spline\"], \"gauss\"))" + ] + }, + { + "cell_type": "markdown", + "id": "c004", + "metadata": {}, + "source": [ + "## Degenerate designs and responses" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c005", + "metadata": {}, + "outputs": [], + "source": [ + "Xd = np.vstack([X, X[:3]]); yd = np.concatenate([y, y[:3]])\n", + "finite_model(\"duplicated points, noise-free\", lambda: lk.Kriging(yd, Xd, \"matern5_2\"))\n", + "if compat.has(\"noise\"):\n", + " k = finite_model(\"duplicated points, nugget\", lambda: lk.Kriging(yd, Xd, \"matern5_2\", noise=\"nugget\"))\n", + " suite.true(\"duplicated points, nugget: a model is built\", k is not None)\n", + "kc = finite_model(\"constant y\", lambda: lk.Kriging(np.full(20, 3.7), X, \"matern5_2\"))\n", + "if kc is not None:\n", + " suite.close(\"constant y: predicts the constant\", pred(kc, Xt)[0], np.full(len(Xt), 3.7), rtol=1e-6, atol=1e-6)\n", + "for n in (2, 3):\n", + " finite_model(f\"n = {n}\", lambda: lk.Kriging(y[:n], X[:n], \"matern5_2\"))\n", + "Xclose = np.vstack([X[:10], X[:10] + 1e-8])\n", + "yclose = tf.unit(Xclose)\n", + "for kernel in (\"gauss\", \"matern5_2\"):\n", + " finite_model(f\"points 1e-8 apart, {kernel}\", lambda: lk.Kriging(yclose, Xclose, kernel))\n", + "# a single input dimension constant over the design\n", + "Xconst = np.column_stack([X[:, 0], np.full(20, 0.5)])\n", + "finite_model(\"one input constant over the design\", lambda: lk.Kriging(tf.unit(Xconst), Xconst, \"matern5_2\"))" + ] + }, + { + "cell_type": "markdown", + "id": "c006", + "metadata": {}, + "source": [ + "## Scale invariance and equivariance" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c007", + "metadata": {}, + "outputs": [], + "source": [ + "k0 = lk.Kriging(y, X, \"matern5_2\", normalize=True)\n", + "m0, s0 = pred(k0, Xt)\n", + "for scale in (1e-6, 1e6):\n", + " with suite.runs(f\"X * {scale:g} with normalize=True: fit\"):\n", + " ks = lk.Kriging(y, X * scale, \"matern5_2\", normalize=True)\n", + " suite.close(f\"X * {scale:g}, normalize=True: same mean as on [0,1]\", pred(ks, Xt * scale)[0], m0,\n", + " rtol=1e-3, atol=1e-3 * np.std(y))\n", + "for a, b in [(1e8, 0.0), (1.0, 1e6), (1e-8, 0.0)]:\n", + " with suite.runs(f\"y * {a:g} + {b:g}: fit\"):\n", + " ka = lk.Kriging(a * y + b, X, \"matern5_2\", normalize=True)\n", + " ma, sa = pred(ka, Xt)\n", + " suite.close(f\"y * {a:g} + {b:g}: mean equivariant\", ma, a * m0 + b, rtol=1e-3, atol=1e-3 * a * np.std(y))\n", + " suite.close(f\"y * {a:g} + {b:g}: stdev scales by |a|\", sa, abs(a) * s0, rtol=1e-2,\n", + " atol=1e-3 * a * np.std(y))\n", + "# the same without normalize: the optimizer starts from a scale-dependent point, warn-level only\n", + "for a, b in [(1e6, 0.0), (1.0, 1e6)]:\n", + " ka = lk.Kriging(a * y + b, X, \"matern5_2\")\n", + " suite.close(f\"y * {a:g} + {b:g} (normalize=False): mean equivariant\", pred(ka, Xt)[0],\n", + " a * pred(lk.Kriging(y, X, \"matern5_2\"), Xt)[0] + b, rtol=1e-3, atol=1e-3 * a * np.std(y),\n", + " severity=\"warn\")" + ] + }, + { + "cell_type": "markdown", + "id": "c008", + "metadata": {}, + "source": [ + "## Far from the data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c009", + "metadata": {}, + "outputs": [], + "source": [ + "k = lk.Kriging(y, X, \"matern5_2\")\n", + "far = np.array([[50.0, 50.0], [-30.0, 80.0]])\n", + "m, s = pred(k, far)\n", + "suite.close(\"far away: mean -> beta (constant trend)\", m, np.full(2, np.ravel(k.beta())[0]), rtol=1e-6,\n", + " atol=1e-8)\n", + "# universal kriging: sigma2 * (1 + 1 / (1' R^-1 1)) with the trend uncertainty, slightly above sigma2\n", + "suite.between(\"far away: stdev / sqrt(sigma2)\", np.max(s) / np.sqrt(k.sigma2()), 1.0 - 1e-6, 1.5)\n", + "suite.close(\"far away: same stdev at both points\", s[0], s[1], rtol=1e-6)" + ] + }, + { + "cell_type": "markdown", + "id": "c010", + "metadata": {}, + "source": [ + "## Many prediction points\n", + "\n", + "Mean and stdev at q points need O(n·q) operations and memory; only the full\n", + "covariance (`return_cov=True`) is O(q²). The time for q = 1000, 2000, 4000,\n", + "8000 points must grow linearly (log2 of the ratio per doubling ~ 1). An\n", + "exponent close to 2 means a q × q matrix is formed: a million points then needs\n", + "terabytes. (Not run at 10⁶ points directly: it would exhaust the runner.)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c011", + "metadata": {}, + "outputs": [], + "source": [ + "times = {}\n", + "for q in (1000, 2000, 4000, 8000):\n", + " Xq = design.uniform(q, 2, seed=72)\n", + " best = np.inf\n", + " for _ in range(3):\n", + " t0 = time.perf_counter()\n", + " pred(k, Xq)\n", + " best = min(best, time.perf_counter() - t0)\n", + " times[q] = best\n", + " suite.metric(f\"predict_q={q}\", \"pylibkriging\", seconds=best)\n", + "expo = np.log2(times[8000] / times[4000])\n", + "suite.less(\"predict(stdev) time growth exponent in q (8000 vs 4000)\", expo, 1.5,\n", + " detail=\", \".join(f\"q={q}: {t:.3f}s\" for q, t in times.items()))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c012", + "metadata": {}, + "outputs": [], + "source": [ + "suite.finalize()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/python/10_compare_python_packages.ipynb b/notebooks/python/10_compare_python_packages.ipynb new file mode 100644 index 0000000..2181f55 --- /dev/null +++ b/notebooks/python/10_compare_python_packages.ipynb @@ -0,0 +1,296 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "c000", + "metadata": {}, + "source": [ + "# 10 — Comparison with Python GP packages\n", + "\n", + "Two kinds of comparisons, which should not be confused:\n", + "\n", + "1. **Same model, same hyperparameters → same numbers.** With θ, σ², β fixed,\n", + " libKriging's predictor and likelihood are textbook formulas: another\n", + " implementation must give the same values to rounding error. Done against\n", + " scikit-learn (`RBF` = libKriging's `gauss`, both product-form) and\n", + " OpenTURNS (product of 1D Matérn 5/2 = libKriging's `matern5_2`). A gap\n", + " here is a bug in one of the packages (error-level).\n", + "2. **Same problem, each package's default fit → accuracy and cost.** Each\n", + " package estimates its hyperparameters its own way (optimizer, restarts,\n", + " bounds, mean model), so differences are expected; the table is a\n", + " benchmark, and the checks are warn-level: libKriging should not be\n", + " markedly less accurate than the best package.\n", + "\n", + "Packages (each optional: skipped if not installed): scikit-learn, SMT,\n", + "OpenTURNS, GPy." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c001", + "metadata": {}, + "outputs": [], + "source": [ + "import time\n", + "import warnings\n", + "\n", + "import numpy as np\n", + "import pandas as pd\n", + "import pylibkriging as lk\n", + "\n", + "import lkbench\n", + "from lkbench import TESTS, design, metrics, compat\n", + "\n", + "warnings.filterwarnings(\"ignore\")\n", + "suite = lkbench.Suite(\"py10_compare_python_packages\")\n", + "sk = compat.optional(\"sklearn.gaussian_process\")\n", + "smt = compat.optional(\"smt.surrogate_models\")\n", + "ot = compat.optional(\"openturns\")\n", + "GPy = compat.optional(\"GPy\")\n", + "print(\"pylibkriging\", compat.version(), \"| sklearn\", bool(sk), \"| SMT\", bool(smt), \"| OpenTURNS\", bool(ot),\n", + " \"| GPy\", bool(GPy))\n", + "\n", + "\n", + "def lk_fixed(y, X, kernel, theta, sigma2, beta):\n", + " return lk.Kriging(y, X, kernel, optim=\"none\",\n", + " parameters={\"theta\": np.reshape(theta, (1, -1)), \"is_theta_estim\": False,\n", + " \"sigma2\": float(sigma2), \"is_sigma2_estim\": False,\n", + " \"beta\": np.reshape(beta, (-1, 1)), \"is_beta_estim\": False})" + ] + }, + { + "cell_type": "markdown", + "id": "c002", + "metadata": {}, + "source": [ + "## 1. Same hyperparameters, same numbers" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c003", + "metadata": {}, + "outputs": [], + "source": [ + "tf = TESTS[\"hartmann3\"]\n", + "X, y, Xt, yt = design.dataset(tf, 40, n_test=300, seed=81)\n", + "kg = lk.Kriging(y, X, \"gauss\")\n", + "theta, sigma2, beta = np.ravel(kg.theta()), float(kg.sigma2()), float(np.ravel(kg.beta())[0])\n", + "k_simple = lk_fixed(y, X, \"gauss\", theta, sigma2, beta) # simple kriging, known beta\n", + "m_lk, s_lk, *_ = (np.ravel(a) for a in k_simple.predict(Xt, return_stdev=True)[:2])\n", + "\n", + "if sk is not None:\n", + " from sklearn.gaussian_process import GaussianProcessRegressor\n", + " from sklearn.gaussian_process.kernels import RBF, ConstantKernel\n", + "\n", + " kern = ConstantKernel(sigma2, \"fixed\") * RBF(theta, \"fixed\")\n", + " gp = GaussianProcessRegressor(kern, alpha=1e-12, optimizer=None, normalize_y=False).fit(X, y - beta)\n", + " m_sk, s_sk = gp.predict(Xt, return_std=True)\n", + " suite.close(\"sklearn RBF == libKriging gauss: simple kriging mean\", m_sk + beta, m_lk, rtol=1e-6,\n", + " atol=1e-7 * np.std(y))\n", + " suite.close(\"sklearn RBF == libKriging gauss: simple kriging stdev (beta fixed)\", s_sk, s_lk, rtol=1e-4,\n", + " atol=1e-6 * np.sqrt(sigma2))\n", + " # with beta estimated, libKriging's stdev is the universal kriging one: the simple kriging variance\n", + " # plus the trend-estimation term (f - F'C^-1 c)^2 / (F'C^-1 F), computed here by hand\n", + " C = kern(X); c = kern(X, Xt)\n", + " w = np.linalg.solve(C, c)\n", + " one = np.ones((len(y), 1))\n", + " v_uk = s_sk ** 2 + ((1 - one.T @ w) ** 2).ravel() / (one.T @ np.linalg.solve(C, one)).item()\n", + " s_uk = np.ravel(kg.predict(Xt, return_stdev=True)[1])\n", + " suite.close(\"libKriging gauss (beta estimated) stdev == universal kriging formula\", s_uk, np.sqrt(v_uk),\n", + " rtol=1e-4, atol=1e-6 * np.sqrt(sigma2))\n", + " # the (concentrated) log-likelihood at the MLE equals the full Gaussian log-density at (theta, sigma2, beta)\n", + " lml = gp.log_marginal_likelihood_value_\n", + " suite.close(\"sklearn log-marginal-likelihood == libKriging logLikelihood()\", lml, kg.logLikelihood(),\n", + " rtol=1e-6)\n", + "else:\n", + " suite.skip(\"sklearn exact comparison\", \"scikit-learn not installed\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c004", + "metadata": {}, + "outputs": [], + "source": [ + "if ot is not None:\n", + " km = lk.Kriging(y, X, \"matern5_2\")\n", + " th, s2 = np.ravel(km.theta()), float(km.sigma2())\n", + " cov = ot.ProductCovarianceModel([ot.MaternModel([t], [1.0], 2.5) for t in th])\n", + " cov.setAmplitude([np.sqrt(s2)])\n", + " basis = ot.ConstantBasisFactory(3).build()\n", + " algo = ot.KrigingAlgorithm(ot.Sample(X), ot.Sample(y.reshape(-1, 1)), cov, basis)\n", + " algo.setOptimizeParameters(False)\n", + " ot.ResourceMap.SetAsScalar(\"GeneralLinearModelAlgorithm-DefaultOptimizationLowerBound\", 1e-6)\n", + " algo.run()\n", + " res = algo.getResult()\n", + " m_ot = np.ravel(res.getConditionalMean(ot.Sample(Xt)))\n", + " v_ot = np.ravel(res.getConditionalMarginalVariance(ot.Sample(Xt)))\n", + " # OpenTURNS re-estimates beta by GLS: compare with libKriging's universal kriging at the same theta, sigma2\n", + " ku = lk.Kriging(y, X, \"matern5_2\", optim=\"none\",\n", + " parameters={\"theta\": th.reshape(1, -1), \"is_theta_estim\": False,\n", + " \"sigma2\": s2, \"is_sigma2_estim\": False})\n", + " m_u, s_u = (np.ravel(a) for a in ku.predict(Xt, return_stdev=True)[:2])\n", + " suite.close(\"OpenTURNS product Matern 5/2 == libKriging matern5_2: mean\", m_ot, m_u, rtol=1e-6,\n", + " atol=1e-6 * np.std(y))\n", + " suite.close(\"OpenTURNS product Matern 5/2 == libKriging matern5_2: variance (universal kriging)\", v_ot, s_u ** 2,\n", + " rtol=1e-4, atol=1e-8 * s2)\n", + "else:\n", + " suite.skip(\"OpenTURNS exact comparison\", \"openturns not installed\")" + ] + }, + { + "cell_type": "markdown", + "id": "c005", + "metadata": {}, + "source": [ + "## 2. Default fits on a set of test functions\n", + "\n", + "Matérn 5/2 (or the closest the package offers), constant mean, maximum\n", + "likelihood with each package's default optimizer, 3 designs per case." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c006", + "metadata": {}, + "outputs": [], + "source": [ + "def fit_libkriging(X, y):\n", + " k = lk.Kriging(y, X, \"matern5_2\")\n", + " return lambda Z: tuple(np.ravel(a) for a in k.predict(Z, return_stdev=True)[:2])\n", + "\n", + "\n", + "def fit_libkriging_ms(X, y):\n", + " k = lk.Kriging(y, X, \"matern5_2\", optim=\"BFGS10\")\n", + " return lambda Z: tuple(np.ravel(a) for a in k.predict(Z, return_stdev=True)[:2])\n", + "\n", + "\n", + "PACKAGES = {\"pylibkriging\": fit_libkriging, \"pylibkriging BFGS10\": fit_libkriging_ms}\n", + "\n", + "if sk is not None:\n", + " from sklearn.gaussian_process.kernels import Matern\n", + "\n", + " def fit_sklearn(X, y):\n", + " d = X.shape[1]\n", + " kern = ConstantKernel(1.0, (1e-3, 1e3)) * Matern(np.full(d, 0.5), (1e-3, 1e3), nu=2.5)\n", + " gp = GaussianProcessRegressor(kern, alpha=1e-10, normalize_y=True, n_restarts_optimizer=5,\n", + " random_state=0).fit(X, y)\n", + " return lambda Z: gp.predict(Z, return_std=True)\n", + " PACKAGES[\"scikit-learn\"] = fit_sklearn\n", + "\n", + "if smt is not None:\n", + " def fit_smt(X, y):\n", + " m = smt.KRG(corr=\"matern52\", poly=\"constant\", print_global=False)\n", + " m.set_training_values(X, y)\n", + " m.train()\n", + " return lambda Z: (np.ravel(m.predict_values(Z)), np.sqrt(np.maximum(np.ravel(m.predict_variances(Z)), 0)))\n", + " PACKAGES[\"SMT\"] = fit_smt\n", + "\n", + "if ot is not None:\n", + " def fit_openturns(X, y):\n", + " d = X.shape[1]\n", + " cov = ot.MaternModel([0.5] * d, [float(np.std(y))], 2.5)\n", + " algo = ot.KrigingAlgorithm(ot.Sample(X), ot.Sample(y.reshape(-1, 1)), cov, ot.ConstantBasisFactory(d).build())\n", + " algo.run()\n", + " r = algo.getResult()\n", + " return lambda Z: (np.ravel(r.getConditionalMean(ot.Sample(Z))),\n", + " np.sqrt(np.maximum(np.ravel(r.getConditionalMarginalVariance(ot.Sample(Z))), 0)))\n", + " PACKAGES[\"OpenTURNS\"] = fit_openturns\n", + "\n", + "if GPy is not None:\n", + " def fit_gpy(X, y):\n", + " kern = GPy.kern.Matern52(X.shape[1], ARD=True)\n", + " m = GPy.models.GPRegression(X, y.reshape(-1, 1), kern, mean_function=None)\n", + " m.Gaussian_noise.variance.fix(1e-8)\n", + " m.optimize_restarts(3, verbose=False)\n", + " def p(Z):\n", + " mu, var = m.predict(Z)\n", + " return np.ravel(mu), np.sqrt(np.maximum(np.ravel(var) - 1e-8, 0))\n", + " return p\n", + " PACKAGES[\"GPy\"] = fit_gpy\n", + "\n", + "CASES = {\"branin\": 30, \"six_hump_camel\": 40, \"hartmann3\": 50, \"ishigami\": 100, \"friedman\": 80,\n", + " \"otl_circuit\": 80, \"piston\": 100, \"borehole\": 80, \"wing_weight\": 100, \"hartmann6\": 120}\n", + "rows = []\n", + "for fname, n in CASES.items():\n", + " tf = TESTS[fname]\n", + " for rep in range(3):\n", + " X, y, Xt, yt = design.dataset(tf, n, n_test=1000, seed=100 + rep)\n", + " for pkg, fit in PACKAGES.items():\n", + " try:\n", + " t0 = time.time()\n", + " predictor = fit(X, y)\n", + " t_fit = time.time() - t0\n", + " t0 = time.time()\n", + " m, s = predictor(Xt)\n", + " t_pred = time.time() - t0\n", + " rows.append(dict(function=fname, n=n, rep=rep, package=pkg, fit_s=t_fit, predict_s=t_pred,\n", + " **metrics.all_metrics(yt, m, s)))\n", + " except Exception as exc: # noqa: BLE001\n", + " rows.append(dict(function=fname, n=n, rep=rep, package=pkg, error=str(exc)[:100]))\n", + " if pkg.startswith(\"pylibkriging\"):\n", + " suite.true(f\"{fname}/rep{rep}/{pkg}: fit\", False, detail=str(exc)[:200])\n", + "res = pd.DataFrame(rows)\n", + "summary = res.groupby([\"function\", \"package\"], sort=False)[[\"q2\", \"nlpd\", \"coverage95\", \"fit_s\"]].median()\n", + "for (fname, pkg), r in summary.iterrows():\n", + " suite.metric(fname, pkg, **r.to_dict())\n", + "summary.unstack(\"package\")[\"q2\"].round(4)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c007", + "metadata": {}, + "outputs": [], + "source": [ + "summary.unstack(\"package\")[[\"fit_s\"]].round(3)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c008", + "metadata": {}, + "outputs": [], + "source": [ + "for fname in CASES:\n", + " s = summary.loc[fname][\"q2\"]\n", + " others = s.drop(index=[p for p in s.index if p.startswith(\"pylibkriging\")])\n", + " if len(others):\n", + " suite.greater(f\"{fname}: Q2(pylibkriging) - best other package\", s[\"pylibkriging\"] - others.max(), -0.02,\n", + " severity=\"warn\", detail=f\"best: {others.idxmax()}\")\n", + " suite.greater(f\"{fname}: Q2(pylibkriging BFGS10) - best other package\", s[\"pylibkriging BFGS10\"] - others.max(),\n", + " -0.05, detail=f\"best: {others.idxmax()}\") if len(others) else None\n", + " suite.greater(f\"{fname}: Q2(pylibkriging BFGS10) > 0.5\", s[\"pylibkriging BFGS10\"], 0.5)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c009", + "metadata": {}, + "outputs": [], + "source": [ + "suite.finalize()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/python/11_cross_binding.ipynb b/notebooks/python/11_cross_binding.ipynb new file mode 100644 index 0000000..945bb71 --- /dev/null +++ b/notebooks/python/11_cross_binding.ipynb @@ -0,0 +1,123 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "c000", + "metadata": {}, + "source": [ + "# 11 — Cross-binding consistency: the Python side\n", + "\n", + "All bindings (Python, R, Octave, Julia) wrap the same C++ library: on the\n", + "same data with the same options they must return the same numbers. Each\n", + "language runs the protocol below on the shared datasets of `data/` and writes\n", + "`results/xbind_.csv` (`lang,dataset,quantity,index,value`);\n", + "`tools/report.py` then compares every language to Python.\n", + "\n", + "Protocol, for each dataset (Branin n=25, Hartmann 3 n=40, Borehole n=80, on\n", + "the unit cube), kernel `matern5_2`, trend `constant`, θ₀ = 0.3 in every\n", + "dimension:\n", + "\n", + "| quantity | how | tolerance in the report |\n", + "|---|---|---|\n", + "| `fixed_sigma2`, `fixed_beta`, `fixed_ll`, `fixed_loo` | `optim=\"none\"`, θ = θ₀ fixed | rel. 1e-8 (error) |\n", + "| `fixed_mean`, `fixed_stdev` | predict at the first 50 test points | rel. 1e-8 (error) |\n", + "| `llfun`, `llfun_grad` | `logLikelihoodFun(θ₀)` with gradient | rel. 1e-8 (error) |\n", + "| `fit_theta`, `fit_ll`, `fit_mean` | default fit (`BFGS`) | rel. 1e-4 (warn: optimizer paths may differ) |\n", + "| `sim` | `simulate(nsim=3, seed=123)` at the first 5 test points (5 × 3, column-major) | rel. 1e-8 (warn: RNG may be binding-specific) |" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c001", + "metadata": {}, + "outputs": [], + "source": [ + "import csv\n", + "from pathlib import Path\n", + "\n", + "import numpy as np\n", + "import pylibkriging as lk\n", + "\n", + "import lkbench\n", + "from lkbench import compat\n", + "from lkbench.checks import RESULTS_DIR\n", + "\n", + "suite = lkbench.Suite(\"py11_cross_binding\")\n", + "print(\"pylibkriging\", compat.version())\n", + "DATA = Path(\"../../data\")\n", + "DATASETS = [\"branin\", \"hartmann3\", \"borehole\"]\n", + "rows = []\n", + "\n", + "\n", + "def put(dataset, quantity, values):\n", + " for i, v in enumerate(np.ravel(values), start=1):\n", + " rows.append((\"python\", dataset, quantity, i, f\"{float(v):.17g}\"))\n", + "\n", + "\n", + "def load(name, stem):\n", + " return np.loadtxt(DATA / f\"{name}_{stem}.csv\", delimiter=\",\", ndmin=2)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c002", + "metadata": {}, + "outputs": [], + "source": [ + "for name in DATASETS:\n", + " X, y, Xt = load(name, \"X\"), load(name, \"y\").ravel(), load(name, \"Xt\")[:50]\n", + " d = X.shape[1]\n", + " theta0 = np.full((1, d), 0.3)\n", + " kf = lk.Kriging(y, X, \"matern5_2\", regmodel=\"constant\", optim=\"none\",\n", + " parameters={\"theta\": theta0, \"is_theta_estim\": False})\n", + " put(name, \"fixed_sigma2\", kf.sigma2())\n", + " put(name, \"fixed_beta\", kf.beta())\n", + " put(name, \"fixed_ll\", kf.logLikelihood())\n", + " put(name, \"fixed_loo\", kf.leaveOneOut())\n", + " m, s, *_ = kf.predict(Xt, return_stdev=True)\n", + " put(name, \"fixed_mean\", m)\n", + " put(name, \"fixed_stdev\", s)\n", + " ll, grad, _ = kf.logLikelihoodFun(theta0.ravel(), True)\n", + " put(name, \"llfun\", ll)\n", + " put(name, \"llfun_grad\", grad)\n", + " suite.close(f\"{name}: logLikelihood() == logLikelihoodFun(theta0)\", kf.logLikelihood(), ll, rtol=1e-10)\n", + " k = lk.Kriging(y, X, \"matern5_2\")\n", + " put(name, \"fit_theta\", k.theta())\n", + " put(name, \"fit_ll\", k.logLikelihood())\n", + " put(name, \"fit_mean\", k.predict(Xt, return_stdev=False)[0])\n", + " put(name, \"sim\", np.ravel(np.asarray(kf.simulate(3, 123, Xt[:5])), order=\"F\")) # column-major, as R/Octave/Julia\n", + "\n", + "RESULTS_DIR.mkdir(exist_ok=True)\n", + "with open(RESULTS_DIR / \"xbind_python.csv\", \"w\", newline=\"\") as fh:\n", + " w = csv.writer(fh)\n", + " w.writerow([\"lang\", \"dataset\", \"quantity\", \"index\", \"value\"])\n", + " w.writerows(rows)\n", + "print(len(rows), \"values written\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c003", + "metadata": {}, + "outputs": [], + "source": [ + "suite.finalize()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/python/src/00_test_functions.py b/notebooks/python/src/00_test_functions.py new file mode 100644 index 0000000..d6a133c --- /dev/null +++ b/notebooks/python/src/00_test_functions.py @@ -0,0 +1,89 @@ +# %% [markdown] +# # 00 — Test functions +# +# Every benchmark of this repository relies on the test functions of +# `lkbench.functions` (after the [Virtual Library of Simulation Experiments](https://www.sfu.ca/~ssurjano/)). +# A typo in one of them would silently bias every result built on it, so they are +# checked first, against the reference values published on the library pages +# (global optima, nominal points) and against the known output ranges. + +# %% +import numpy as np +import pandas as pd +import matplotlib.pyplot as plt + +import lkbench +from lkbench import TESTS, design + +suite = lkbench.Suite("py00_test_functions") + +# %% [markdown] +# ## Reference values + +# %% +rows = [] +for name, tf in TESTS.items(): + for x, v in tf.known: + fx = float(tf(x)[0]) + suite.close(f"{name}: f({np.round(x, 4).tolist()})", fx, v, rtol=1e-4, atol=1e-4) + rows.append((name, tf.d, np.round(x, 4).tolist(), v, fx)) +pd.DataFrame(rows, columns=["function", "d", "x", "reference", "computed"]) + +# %% [markdown] +# ## Output ranges +# +# For the engineering functions without a closed-form optimum, the range over the +# domain is compared with the one reported in the literature (loose bounds: a +# random sample does not reach the extremes). + +# %% +EXPECTED_RANGES = { # (lower bound of min, upper bound of max) over the domain + "otl_circuit": (2.5, 10.0), + "piston": (0.1, 1.3), + "borehole": (2.0, 320.0), + "wing_weight": (120.0, 480.0), + "currin_exp": (1.0, 14.0), + "dette_pepelyshev": (0.0, 100.0), +} +summary = [] +for name, tf in TESTS.items(): + U = design.uniform(20_000, tf.d, seed=1) + y = tf.unit(U) + suite.finite(f"{name}: finite on the domain", y) + if name in EXPECTED_RANGES: + lo, hi = EXPECTED_RANGES[name] + suite.true(f"{name}: range within [{lo}, {hi}]", lo <= y.min() and y.max() <= hi, + detail=f"sampled range [{y.min():.4g}, {y.max():.4g}]") + summary.append((name, tf.d, ",".join(tf.tags), y.min(), y.max(), y.std())) +pd.DataFrame(summary, columns=["function", "d", "tags", "min", "max", "std"]) + +# %% [markdown] +# ## Designs +# +# The designs are seeded: the same seed gives the same points (reproducibility of +# every benchmark), and the optimized LHS is a Latin hypercube (one point per +# stratum in each dimension). + +# %% +A, B = design.lhs(20, 3, seed=7), design.lhs(20, 3, seed=7) +suite.close("lhs: reproducible with a seed", A, B, rtol=0, atol=0) +strata = np.floor(A * 20).astype(int) +suite.true("lhs: one point per stratum", all(len(set(strata[:, j])) == 20 for j in range(3))) + +# %% +fig, axes = plt.subplots(1, 4, figsize=(16, 3.5)) +for ax, name in zip(axes, ["forrester", "gramacy_lee_2012"]): + tf = TESTS[name] + u = np.linspace(0, 1, 500)[:, None] + ax.plot(tf.scale(u)[:, 0], tf.unit(u)) + ax.set_title(name) +for ax, name in zip(axes[2:], ["branin", "gramacy_lee_2008"]): + tf = TESTS[name] + g = np.linspace(0, 1, 100) + G = np.array(np.meshgrid(g, g)).reshape(2, -1).T + ax.contourf(g, g, tf.unit(G).reshape(100, 100), 30) + ax.set_title(name) +plt.tight_layout() + +# %% +suite.finalize() diff --git a/notebooks/python/src/01_kriging_kernels_trends.py b/notebooks/python/src/01_kriging_kernels_trends.py new file mode 100644 index 0000000..eed51aa --- /dev/null +++ b/notebooks/python/src/01_kriging_kernels_trends.py @@ -0,0 +1,194 @@ +# %% [markdown] +# # 01 — Kriging: kernels × trends on 1D/2D/3D functions +# +# Plain noise-free `Kriging`, every kernel (`gauss`, `exp`, `matern3_2`, +# `matern5_2`) crossed with every trend (`constant`, `linear`, `interactive`, +# `quadratic`) on functions of different regularity: +# +# | function | d | character | +# |---|---|---| +# | Forrester | 1 | smooth, one oscillation | +# | Gramacy & Lee (2012) | 1 | fast oscillation then polynomial: non-stationary | +# | Branin | 2 | smooth, three global minima | +# | Six-hump camel | 2 | polynomial, steep borders | +# | log-Goldstein-Price | 2 | strongly varying | +# | Lim polynomial | 2 | low-degree polynomial: the trend should help | +# | Ishigami | 3 | strongly non-linear, interactions | +# | Hartmann 3 | 3 | multimodal, localized | +# +# Checks: +# * **interpolation**: a noise-free kriging mean reproduces the observations, with +# a (near) zero predictive standard deviation at the design points. Error-level +# for every kernel but `gauss`: its correlation matrices are notoriously +# ill-conditioned, and the numerical jitter libKriging adds to factorize them +# breaks exact interpolation (warn-level, with the size of the gap); +# * **accuracy**: Q² on a test set above a per-function floor for +# `matern5_2` / `constant` with a multistart optimizer (`BFGS10`): a +# regression shows up here first. The same floor for the default single-start +# `BFGS` is warn-level: a single start can end in a poor local optimum of the +# likelihood (see notebook 02), which is worth seeing in the report but is not +# a defect of the model; +# * **sanity of the fitted hyperparameters**: finite, positive θ and σ²; +# * **consistency**: `predict` on a batch equals `predict` point by point, the +# returned covariance has the predicted variances on its diagonal, and is +# symmetric positive semi-definite. + +# %% +import time + +import numpy as np +import pandas as pd +import matplotlib.pyplot as plt +import pylibkriging as lk + +import lkbench +from lkbench import TESTS, design, metrics, compat + +suite = lkbench.Suite("py01_kernels_trends") +print("pylibkriging", compat.version()) + +KERNELS = ["gauss", "exp", "matern3_2", "matern5_2"] +TRENDS = ["constant", "linear", "interactive", "quadratic"] +CASES = { # function: (n, Q2 floor for matern5_2/constant) + "forrester": (12, 0.95), + "gramacy_lee_2012": (20, 0.60), + "branin": (30, 0.97), + "six_hump_camel": (30, 0.95), + "log_goldstein_price": (40, 0.85), + "lim_polynomial": (15, 0.99), + "ishigami": (80, 0.75), + "hartmann3": (40, 0.85), +} + + +def predict(model, X, **kw): + m, s, cov, *_ = model.predict(X, return_stdev=True, **kw) + return np.asarray(m).ravel(), np.asarray(s).ravel(), cov + +# %% [markdown] +# ## Kernel × trend grid + +# %% +for fname, (n, q2_floor) in CASES.items(): + tf = TESTS[fname] + X, y, Xt, yt = design.dataset(tf, n, n_test=1000, seed=1) + for kernel in KERNELS: + for trend in TRENDS: + n_beta = {"constant": 1, "linear": 1 + tf.d, "interactive": 1 + tf.d + tf.d * (tf.d - 1) // 2, + "quadratic": 1 + 2 * tf.d + tf.d * (tf.d - 1) // 2}[trend] + case = f"{fname}/{kernel}/{trend}" + if n_beta >= n - 1: + suite.skip(case, f"{n_beta} trend coefficients for n={n}") + continue + t0 = time.time() + try: + k = lk.Kriging(y, X, kernel, regmodel=trend, optim="BFGS") + except Exception as exc: # noqa: BLE001 + suite.true(f"{case}: fit", False, detail=str(exc)[:200]) + continue + fit_s = time.time() - t0 + m, s, _ = predict(k, Xt) + res = metrics.all_metrics(yt, m, s) + suite.metric(case, "pylibkriging", fit_s=fit_s, **res) + theta, sigma2 = np.asarray(k.theta()).ravel(), float(k.sigma2()) + suite.true(f"{case}: theta, sigma2 finite and > 0", + np.all(np.isfinite(theta)) and np.all(theta > 0) and np.isfinite(sigma2) and sigma2 > 0, + detail=f"theta={theta}, sigma2={sigma2:.3g}") + # interpolation at the design points (relative to the spread of y) + md, sd, _ = predict(k, X) + scale = np.std(y) + sev = "warn" if kernel == "gauss" else "error" + suite.less(f"{case}: interpolates y (max |m(X)-y| / sd(y))", np.max(np.abs(md - y)) / scale, 1e-4, + severity=sev) + suite.less(f"{case}: stdev(X) ~ 0 (max sd / sd(y))", np.max(sd) / scale, 1e-2, severity=sev) + if kernel == "matern5_2" and trend == "constant": + suite.greater(f"{case}: Q2 (single-start BFGS)", res["q2"], q2_floor, severity="warn") + k10 = lk.Kriging(y, X, kernel, regmodel=trend, optim="BFGS10") + res10 = metrics.all_metrics(yt, *predict(k10, Xt)[:2]) + suite.metric(case, "pylibkriging BFGS10", **res10) + suite.greater(f"{case}: Q2 (BFGS10)", res10["q2"], q2_floor) + +# %% +tab = suite.table().reset_index() +tab = tab[tab["package"] == "pylibkriging"] +tab[["function", "kernel", "trend"]] = tab["case"].str.split("/", expand=True) +pivot = tab.pivot_table(index=["function", "trend"], columns="kernel", values="q2", sort=False) +pivot.style.format("{:.3f}").background_gradient(axis=None, vmin=0.5, vmax=1, cmap="RdYlGn") + +# %% [markdown] +# Expectations that should hold whatever the version (warn-level: they are +# empirical, not mathematical, statements): +# * on the smooth Branin the `exp` kernel (non-differentiable paths) is the least accurate; +# * on the Lim polynomial a `quadratic` trend is at least as good as a `constant` one. + +# %% +q = tab.set_index(["function", "kernel", "trend"])["q2"] +suite.true("branin: exp is the worst kernel (constant trend)", + q["branin", "exp", "constant"] <= min(q["branin", k, "constant"] for k in KERNELS), + severity="warn") +suite.greater("lim_polynomial/matern5_2: Q2(quadratic) - Q2(constant)", + q["lim_polynomial", "matern5_2", "quadratic"] - q["lim_polynomial", "matern5_2", "constant"], -1e-3, + severity="warn") + +# %% [markdown] +# ## Consistency of `predict` + +# %% +tf = TESTS["branin"] +X, y, Xt, yt = design.dataset(tf, 30, n_test=50, seed=3) +k = lk.Kriging(y, X, "matern5_2") +m, s, cov = predict(k, Xt, return_cov=True) +cov = np.asarray(cov) +suite.close("predict: diag(cov) == stdev^2", np.diag(cov), s ** 2, rtol=1e-6, atol=1e-10) +suite.close("predict: cov symmetric", cov, cov.T, rtol=1e-10, atol=1e-12) +suite.greater("predict: cov PSD (min eigenvalue / max)", np.linalg.eigvalsh(cov).min() / np.abs(cov).max(), -1e-8) +m1 = np.array([predict(k, Xt[i:i + 1])[0][0] for i in range(len(Xt))]) +s1 = np.array([predict(k, Xt[i:i + 1])[1][0] for i in range(len(Xt))]) +suite.close("predict: batch == point by point (mean)", m, m1, rtol=1e-8, atol=1e-10 * np.std(y)) +suite.close("predict: batch == point by point (stdev)", s, s1, rtol=1e-6, atol=1e-10 * np.std(y)) +# the mean is a function of the point only: duplicating/permuting the batch changes nothing +perm = np.random.default_rng(0).permutation(len(Xt)) +suite.close("predict: invariant to the order of the batch", predict(k, Xt[perm])[0], m[perm], rtol=1e-10, atol=1e-10) + +# %% [markdown] +# ### Derivatives of the predictor +# +# `return_deriv=True` gives the gradients of the mean and of the stdev w.r.t. x; +# compared with central finite differences. + +# %% +with suite.runs("predict(return_deriv=True)"): + out = k.predict(Xt[:5], True, False, True) + dm, ds = np.asarray(out[3]), np.asarray(out[4]) + h = 1e-6 + fd_m, fd_s = np.zeros_like(dm), np.zeros_like(ds) + for j in range(2): + e = np.zeros(2); e[j] = h + mp, sp, _ = predict(k, Xt[:5] + e) + mm, sm, _ = predict(k, Xt[:5] - e) + fd_m[:, j], fd_s[:, j] = (mp - mm) / (2 * h), (sp - sm) / (2 * h) + suite.close("d mean / dx == finite differences", dm, fd_m, rtol=1e-4, atol=1e-4 * np.abs(fd_m).max()) + suite.close("d stdev / dx == finite differences", ds, fd_s, rtol=1e-3, atol=1e-3 * np.abs(fd_s).max()) + +# %% [markdown] +# ## A look at the 1D fits + +# %% +fig, axes = plt.subplots(1, 2, figsize=(13, 4)) +for ax, fname in zip(axes, ["forrester", "gramacy_lee_2012"]): + tf = TESTS[fname] + n = CASES[fname][0] + X, y, _, _ = design.dataset(tf, n, seed=1) + u = np.linspace(0, 1, 400)[:, None] + ax.plot(u, tf.unit(u), "k--", label="truth") + for kernel in ["gauss", "matern5_2", "exp"]: + mm, ss, _ = predict(lk.Kriging(y, X, kernel), u) + ax.plot(u, mm, label=kernel) + if kernel == "matern5_2": + ax.fill_between(u[:, 0], mm - 2 * ss, mm + 2 * ss, alpha=0.2) + ax.plot(X, y, "ko") + ax.set_title(fname) + ax.legend() + +# %% +suite.finalize() diff --git a/notebooks/python/src/02_objectives_gradients.py b/notebooks/python/src/02_objectives_gradients.py new file mode 100644 index 0000000..812e97f --- /dev/null +++ b/notebooks/python/src/02_objectives_gradients.py @@ -0,0 +1,174 @@ +# %% [markdown] +# # 02 — Objectives (LL, LOO, LMP), their gradients and optima +# +# The hyperparameters θ of `Kriging` are estimated by maximizing the +# log-likelihood (`LL`), maximizing the log marginal posterior (`LMP`), or +# minimizing the leave-one-out error (`LOO`). This notebook checks the machinery +# behind those fits rather than the predictions: +# +# * **analytic gradients** of `logLikelihoodFun`, `leaveOneOutFun`, +# `logMargPostFun` against central finite differences, at random θ, for +# every kernel; +# * **optimality** of the fitted θ: the objective at θ̂ is not improved by small +# perturbations of θ̂ (a local optimum was reached). A component θ_j > 5 on +# the unit cube is a nearly linear direction whose optimum sits at the upper +# bound of θ: moving it further out is allowed to improve the objective; +# * **global vs local optimum**: the gap between the default single-start fit +# and a 20-start fit (warn-level: it measures how often the default fit ends in +# a poor local optimum); +# * **consistency** between `logLikelihood()` and `logLikelihoodFun(theta())`; +# * the closed-form **leave-one-out** predictions of `leaveOneOutVec` against an +# explicit refit without each point (same θ, σ²); +# * multistart (`BFGS10`) never ends below a single `BFGS` run (warn-level: a +# multistart that does worse points to a starting-point issue). +# +# Functions: Branin (2D), Hartmann 3 (3D), Ishigami (3D), Friedman (5D). + +# %% +import numpy as np +import pandas as pd +import pylibkriging as lk + +import lkbench +from lkbench import TESTS, design, metrics, compat + +suite = lkbench.Suite("py02_objectives_gradients") +print("pylibkriging", compat.version()) +CASES = {"branin": 25, "hartmann3": 40, "ishigami": 60, "friedman": 50} +KERNELS = ["gauss", "exp", "matern3_2", "matern5_2"] +rng = np.random.default_rng(0) + + +def objective_fun(k, objective): + """(value, gradient) of the objective as a function of theta.""" + if objective == "LL": + return lambda t, g=True: k.logLikelihoodFun(np.asarray(t, float), g)[:2] + if objective == "LOO": + return lambda t, g=True: k.leaveOneOutFun(np.asarray(t, float), g) + return lambda t, g=True: k.logMargPostFun(np.asarray(t, float), g) + +# %% [markdown] +# ## Analytic gradients vs finite differences + +# %% +rows = [] +for fname, n in CASES.items(): + tf = TESTS[fname] + X, y, _, _ = design.dataset(tf, n, seed=2) + for kernel in KERNELS: + k = lk.Kriging(y, X, kernel) + for objective in ["LL", "LOO", "LMP"]: + f = objective_fun(k, objective) + worst = 0.0 + for _ in range(3): + theta = np.exp(rng.uniform(np.log(0.05), np.log(1.0), tf.d)) + val, grad = f(theta) + fd = metrics.finite_diff_grad(lambda t: f(t, False)[0], theta, h=1e-5) + grad = np.asarray(grad).ravel() + err = np.max(np.abs(grad - fd)) / max(np.max(np.abs(fd)), 1e-8) + worst = max(worst, err) + rows.append((fname, kernel, objective, worst)) + suite.less(f"{fname}/{kernel}/{objective}: gradient vs finite diff (rel. err)", worst, 1e-3) +pd.DataFrame(rows, columns=["function", "kernel", "objective", "max rel. error"]).pivot_table( + index=["function", "kernel"], columns="objective", values="max rel. error").style.format("{:.1e}") + +# %% [markdown] +# ## Optimality of the fitted θ, objective by objective + +# %% +SIGN = {"LL": 1, "LMP": 1, "LOO": -1} # +1: maximized, -1: minimized +for fname, n in CASES.items(): + tf = TESTS[fname] + X, y, Xt, yt = design.dataset(tf, n, n_test=1000, seed=2) + for objective in ["LL", "LOO", "LMP"]: + k = lk.Kriging(y, X, "matern5_2", objective=objective) + theta = np.asarray(k.theta()).ravel() + f = objective_fun(k, objective) + best = SIGN[objective] * f(theta, False)[0] + improved = [] + for j in range(tf.d): + for fac in (0.9, 1.1): + t = theta.copy(); t[j] *= fac + at_bound = fac > 1 and theta[j] > 5 + # tolerance: the optimizer's stopping criterion (0.01 on a log-density is immaterial) + tol = 1e-4 * abs(best) + (0.0 if objective == "LOO" else 1e-2) + if SIGN[objective] * f(t, False)[0] > best + tol and not at_bound: + improved.append(f"theta{j + 1}*{fac}") + suite.true(f"{fname}/{objective}: theta_hat not improved by +/-10% moves", not improved, + detail=f"theta={np.round(theta, 4)}; improving moves: {improved}") + m, s, *_ = k.predict(Xt, return_stdev=True) + suite.metric(f"{fname}/{objective}", "pylibkriging", **metrics.all_metrics(yt, m, s)) + +# %% +tab = suite.table().reset_index() +tab[["function", "objective"]] = tab["case"].str.split("/", expand=True) +tab.pivot_table(index="function", columns="objective", values=["q2", "coverage95"]).round(3) + +# %% [markdown] +# ## `logLikelihood()` == `logLikelihoodFun(theta())` (and the same for LOO, LMP) + +# %% +tf = TESTS["hartmann3"] +X, y, _, _ = design.dataset(tf, 40, seed=5) +for objective, getter in [("LL", "logLikelihood"), ("LOO", "leaveOneOut"), ("LMP", "logMargPost")]: + k = lk.Kriging(y, X, "matern5_2", objective=objective) + v_fun = objective_fun(k, objective)(np.asarray(k.theta()).ravel(), False)[0] + suite.close(f"{getter}() == {objective} function at theta()", getattr(k, getter)(), v_fun, rtol=1e-8) + +# %% [markdown] +# ## Closed-form leave-one-out vs explicit refits +# +# `leaveOneOutVec(θ)` gives the mean and stdev of the prediction of each `y_i` +# from the n−1 others, by the closed-form (Dubrule) formulas. An explicit refit +# without point i, with θ and σ² fixed (β re-estimated, as in the formula), must +# give the same mean. + +# %% +for fname in ["branin", "hartmann3"]: + tf = TESTS[fname] + X, y, _, _ = design.dataset(tf, 20, seed=6) + for trend in ["constant", "linear"]: + k = lk.Kriging(y, X, "matern5_2", regmodel=trend) + theta, sigma2 = np.asarray(k.theta()).ravel(), float(k.sigma2()) + m_loo, s_loo = (np.asarray(a).ravel() for a in k.leaveOneOutVec(theta)) + m_ref, s_ref = np.zeros(len(y)), np.zeros(len(y)) + for i in range(len(y)): + keep = np.arange(len(y)) != i + ki = lk.Kriging(y[keep], X[keep], "matern5_2", regmodel=trend, optim="none", + parameters={"theta": theta.reshape(1, -1), "is_theta_estim": False, + "sigma2": sigma2, "is_sigma2_estim": False}) + mi, si, *_ = ki.predict(X[i:i + 1], return_stdev=True) + m_ref[i], s_ref[i] = np.ravel(mi)[0], np.ravel(si)[0] + suite.close(f"{fname}/{trend}: leaveOneOutVec mean == explicit refits", m_loo, m_ref, + rtol=1e-6, atol=1e-8 * np.std(y)) + suite.close(f"{fname}/{trend}: leaveOneOutVec stdev == explicit refits", s_loo, s_ref, + rtol=1e-4, atol=1e-8 * np.std(y), severity="warn") + # leaveOneOut() is the mean squared LOO error at the fitted theta + k_loo = lk.Kriging(y, X, "matern5_2", regmodel=trend, objective="LOO") + m2, _ = k_loo.leaveOneOutVec(np.asarray(k_loo.theta()).ravel()) + suite.close(f"{fname}/{trend}: leaveOneOut() == mean squared LOO error", k_loo.leaveOneOut(), + np.mean((np.ravel(m2) - y) ** 2), rtol=1e-6, severity="warn") + +# %% [markdown] +# ## Multistart +# +# Gap between the default single start and 10 / 20 starts. A large gap means the +# default fit is unreliable on this kind of problem. + +# %% +for fname, n in CASES.items(): + tf = TESTS[fname] + X, y, _, _ = design.dataset(tf, n, seed=7) + ll1 = lk.Kriging(y, X, "matern5_2", optim="BFGS").logLikelihood() + ll10 = lk.Kriging(y, X, "matern5_2", optim="BFGS10").logLikelihood() + ll20 = lk.Kriging(y, X, "matern5_2", optim="BFGS20").logLikelihood() + suite.metric(fname, "BFGS", ll=ll1) + suite.metric(fname, "BFGS10", ll=ll10) + suite.metric(fname, "BFGS20", ll=ll20) + suite.greater(f"{fname}: LL(BFGS10) - LL(BFGS)", ll10 - ll1, -1e-3 * abs(ll1), severity="warn") + suite.less(f"{fname}: LL(BFGS20) - LL(BFGS) (default fit far from the best optimum)", ll20 - ll1, 1.0, + severity="warn") +suite.table().loc[list(CASES)].unstack("package") + +# %% +suite.finalize() diff --git a/notebooks/python/src/03_noise.py b/notebooks/python/src/03_noise.py new file mode 100644 index 0000000..8097f4f --- /dev/null +++ b/notebooks/python/src/03_noise.py @@ -0,0 +1,151 @@ +# %% [markdown] +# # 03 — Noisy observations: `noise="nugget"` and known noise variances +# +# `Kriging` handles noisy responses through its `noise` argument: `"nugget"` +# estimates one homogeneous noise variance, a vector gives a known variance per +# observation (heteroscedastic case). +# +# | case | function | noise | +# |---|---|---| +# | 1D homoscedastic | Forrester | N(0, 0.5²) | +# | 5D homoscedastic | Friedman | N(0, 1) (the classical "Friedman 1" setup) | +# | 1D heteroscedastic | Gramacy & Lee (2012) | sd growing linearly with x | +# | 2D with replicates | Branin | 4 replicates of 15 points, N(0, 2²) | +# +# Checks: +# * the estimated nugget is close to the true noise variance (factor 3); +# * the noisy fit predicts the noise-free function well (Q² floor), and is +# **smooth**: it does not chase the noisy data between the observations. (At the +# design points themselves the `"nugget"` predictor may return y exactly, as +# DiceKriging's nugget does: the nugget is then a micro-scale variation, and +# the predictor is discontinuous there. This is reported, not checked.); +# * prediction intervals are calibrated: 95% coverage of the *noise-free* truth; +# * a zero noise vector gives the noise-free model; +# * with heteroscedastic noise, giving the true variances beats a single nugget +# (NLPD, warn-level); +# * replicates: averaging the replicates with variance σ²/r is equivalent to +# fitting all the replicates (same predictor, up to the optimizer). + +# %% +import numpy as np +import pandas as pd +import matplotlib.pyplot as plt +import pylibkriging as lk + +import lkbench +from lkbench import TESTS, design, metrics, compat + +suite = lkbench.Suite("py03_noise") +print("pylibkriging", compat.version()) + + +def pred(k, X): + m, s, *_ = k.predict(X, return_stdev=True) + return np.ravel(m), np.ravel(s) + +# %% [markdown] +# ## Homoscedastic noise, estimated nugget + +# %% +for fname, n, sd, q2_floor in [("forrester", 40, 0.5, 0.95), ("friedman", 200, 1.0, 0.90)]: + tf = TESTS[fname] + X, y, Xt, yt = design.dataset(tf, n, n_test=2000, seed=11, noise_sd=sd) + k = lk.Kriging(y, X, "matern5_2", noise="nugget", optim="BFGS10") + nugget = float(np.ravel(k.nugget())[0]) + m, s = pred(k, Xt) + res = metrics.all_metrics(yt, m, s) + suite.metric(fname, "nugget", nugget=nugget, true_var=sd ** 2, **res) + suite.between(f"{fname}: nugget / true noise variance", nugget / sd ** 2, 1 / 3, 3) + suite.greater(f"{fname}: Q2 vs noise-free truth", res["q2"], q2_floor) + mX, _ = pred(k, X) + mX_near, _ = pred(k, np.clip(X + 1e-7, 0, 1)) + suite.metric(fname, "nugget", rms_resid_at_X=metrics.rmse(y, mX) / sd, + rms_resid_near_X=metrics.rmse(y, mX_near) / sd) + suite.greater(f"{fname}: smooths the noisy data next to the design points (rms residual / sd)", + metrics.rmse(y, mX_near) / sd, 0.3) + suite.between(f"{fname}: coverage of the 95% intervals (noise-free truth)", res["coverage95"], 0.80, 1.0, + severity="warn") + # the noise-free model on the same noisy data interpolates, and generalizes worse + k0 = lk.Kriging(y, X, "matern5_2") + m0, s0 = pred(k0, Xt) + suite.metric(fname, "noise-free fit", **metrics.all_metrics(yt, m0, s0)) + suite.less(f"{fname}: RMSE(nugget) < RMSE(interpolating fit)", res["rmse"], metrics.rmse(yt, m0), + severity="warn") +suite.table() + +# %% [markdown] +# ## A zero noise vector is the noise-free model + +# %% +tf = TESTS["branin"] +X, y, Xt, yt = design.dataset(tf, 25, n_test=200, seed=12) +k0 = lk.Kriging(y, X, "matern5_2") +theta = np.asarray(k0.theta()).reshape(1, -1) +fixed = {"theta": theta, "is_theta_estim": False, "sigma2": float(k0.sigma2()), "is_sigma2_estim": False} +kf = lk.Kriging(y, X, "matern5_2", optim="none", parameters=fixed) +with suite.runs("noise=zeros, theta and sigma2 fixed: fit"): + kz = lk.Kriging(y, X, "matern5_2", noise=np.zeros(len(y)), optim="none", parameters=fixed) + suite.close("noise=zeros: same mean as the noise-free model", pred(kz, Xt)[0], pred(kf, Xt)[0], rtol=1e-6, + atol=1e-8 * np.std(y)) + suite.close("noise=zeros: same stdev as the noise-free model", pred(kz, Xt)[1], pred(kf, Xt)[1], rtol=1e-4, + atol=1e-6 * np.std(y)) +with suite.runs("noise=zeros, sigma2 estimated: fit", severity="warn"): + lk.Kriging(y, X, "matern5_2", noise=np.zeros(len(y)), optim="none", + parameters={"theta": theta, "is_theta_estim": False}) + +# %% [markdown] +# ## Heteroscedastic noise: known variances vs a single nugget + +# %% +tf = TESTS["gramacy_lee_2012"] +n = 60 +X, _, Xt, yt = design.dataset(tf, n, n_test=2000, seed=13) +sd_x = 0.02 + 0.4 * X[:, 0] # noise sd grows with x (in unit coordinates) +y = tf.unit(X) + np.random.default_rng(13).normal(0, sd_x) +k_known = lk.Kriging(y, X, "matern5_2", noise=sd_x ** 2, optim="BFGS10") +k_nug = lk.Kriging(y, X, "matern5_2", noise="nugget", optim="BFGS10") +res_known = metrics.all_metrics(yt, *pred(k_known, Xt)) +res_nug = metrics.all_metrics(yt, *pred(k_nug, Xt)) +suite.metric("gramacy_lee_2012/hetero", "known variances", **res_known) +suite.metric("gramacy_lee_2012/hetero", "nugget", **res_nug) +suite.finite("hetero: predictions finite", pred(k_known, Xt)[0]) +suite.less("hetero: NLPD(known variances) <= NLPD(nugget)", res_known["nlpd"] - res_nug["nlpd"], 0.05, + severity="warn") +suite.close("noise(): returns the given variances", np.ravel(k_known.noise()), sd_x ** 2, rtol=1e-12) + +u = np.linspace(0, 1, 400)[:, None] +fig, ax = plt.subplots(figsize=(10, 4)) +ax.plot(u, tf.unit(u), "k--", label="truth") +ax.errorbar(X[:, 0], y, yerr=2 * sd_x, fmt="k.", alpha=0.4) +for k, lab in [(k_known, "known variances"), (k_nug, "nugget")]: + m, s = pred(k, u) + ax.plot(u, m, label=lab) + ax.fill_between(u[:, 0], m - 2 * s, m + 2 * s, alpha=0.15) +ax.legend(); + +# %% [markdown] +# ## Replicates +# +# r noisy replicates at the same point carry the same information as their mean +# with variance σ²/r: fitting the replicates with `noise=σ²` or their means with +# `noise=σ²/r` must give the same predictor at a fixed θ. + +# %% +tf = TESTS["branin"] +r, sd = 4, 2.0 +X1, _, Xt, yt = design.dataset(tf, 15, n_test=300, seed=14) +Xr = np.repeat(X1, r, axis=0) +yr = tf.unit(Xr) + np.random.default_rng(14).normal(0, sd, len(Xr)) +ybar = yr.reshape(-1, r).mean(axis=1) +k_rep = lk.Kriging(yr, Xr, "matern5_2", noise=np.full(len(yr), sd ** 2)) +fixed = {"theta": np.asarray(k_rep.theta()).reshape(1, -1), "is_theta_estim": False, + "sigma2": float(k_rep.sigma2()), "is_sigma2_estim": False} +k_rep = lk.Kriging(yr, Xr, "matern5_2", noise=np.full(len(yr), sd ** 2), optim="none", parameters=fixed) +k_avg = lk.Kriging(ybar, X1, "matern5_2", noise=np.full(len(ybar), sd ** 2 / r), optim="none", parameters=fixed) +suite.close("replicates == averages with variance/r (mean)", pred(k_rep, Xt)[0], pred(k_avg, Xt)[0], + rtol=1e-6, atol=1e-6 * np.std(yr)) +suite.close("replicates == averages with variance/r (stdev)", pred(k_rep, Xt)[1], pred(k_avg, Xt)[1], + rtol=1e-4, atol=1e-6 * np.std(yr)) + +# %% +suite.finalize() diff --git a/notebooks/python/src/04_update_simulate.py b/notebooks/python/src/04_update_simulate.py new file mode 100644 index 0000000..2041635 --- /dev/null +++ b/notebooks/python/src/04_update_simulate.py @@ -0,0 +1,136 @@ +# %% [markdown] +# # 04 — `update`, `simulate`, `update_simulate` +# +# Sequential use of a fitted model (sequential design, Bayesian optimization): +# +# * `update(y_u, X_u, refit=False)` adds observations through a Cholesky +# (Schur complement) update, keeping θ; β and σ² are re-estimated (profiled) +# on the enlarged data. It must match a model built from scratch on all the +# data with the same θ (`optim="none"`, σ² and β estimated). +# * `update(..., refit=True)` re-optimizes: it must match a fresh fit (up to the +# optimizer: warn-level). +# * `simulate(nsim, seed, X)` draws conditional paths: their empirical mean and +# covariance must converge to `predict`'s mean and covariance; a seed makes +# them reproducible; at the design points they reproduce y. +# * `update_simulate(y_u, X_u)` conditions paths already drawn on new data: the +# corrected paths must be distributed as fresh paths of the updated model. +# +# Functions: Branin (2D), Hartmann 3 (3D), Hartmann 6 (6D). + +# %% +import numpy as np +import pylibkriging as lk + +import lkbench +from lkbench import TESTS, design, compat + +suite = lkbench.Suite("py04_update_simulate") +print("pylibkriging", compat.version()) + + +def pred(k, X, cov=False): + m, s, c, *_ = k.predict(X, return_stdev=True, return_cov=cov) + return np.ravel(m), np.ravel(s), np.asarray(c) + + +def fixed_params(k): + return {"theta": np.asarray(k.theta()).reshape(1, -1), "is_theta_estim": False, + "sigma2": float(k.sigma2()), "is_sigma2_estim": False} + +# %% [markdown] +# ## `update` vs a fit from scratch + +# %% +for fname, n0, nu in [("branin", 20, 5), ("hartmann3", 30, 10), ("hartmann6", 60, 20)]: + tf = TESTS[fname] + X, y, Xt, yt = design.dataset(tf, n0 + nu, n_test=300, seed=21) + for trend in ["constant", "linear"]: + case = f"{fname}/{trend}" + k = lk.Kriging(y[:n0], X[:n0], "matern5_2", regmodel=trend) + p = fixed_params(k) + k.update(y[n0:], X[n0:], refit=False) + ref = lk.Kriging(y, X, "matern5_2", regmodel=trend, optim="none", + parameters={"theta": p["theta"], "is_theta_estim": False}) + m, s, _ = pred(k, Xt) + mr, sr, _ = pred(ref, Xt) + suite.close(f"{case}: update(refit=False) mean == scratch fit, same theta", m, mr, rtol=1e-6, + atol=1e-8 * np.std(y)) + suite.close(f"{case}: update(refit=False) stdev == scratch fit, same theta", s, sr, rtol=1e-5, + atol=1e-8 * np.std(y)) + suite.close(f"{case}: update(refit=False) sigma2 == scratch fit sigma2", k.sigma2(), ref.sigma2(), + rtol=1e-8) + suite.close(f"{case}: update(refit=False) keeps theta", np.ravel(k.theta()), np.ravel(p["theta"]), + rtol=0, atol=0) + suite.true(f"{case}: model holds n0+nu points", np.asarray(k.X()).shape[0] == n0 + nu) + # refit=True vs a fresh fit (same optimizer from the same data, possibly another start) + k2 = lk.Kriging(y[:n0], X[:n0], "matern5_2", regmodel=trend) + k2.update(y[n0:], X[n0:], refit=True) + fresh = lk.Kriging(y, X, "matern5_2", regmodel=trend) + suite.close(f"{case}: update(refit=True) LL == fresh fit LL", k2.logLikelihood(), fresh.logLikelihood(), + rtol=1e-3, severity="warn") + +# %% [markdown] +# Several small updates in a row equal one update with all the points. + +# %% +tf = TESTS["hartmann3"] +X, y, Xt, _ = design.dataset(tf, 40, n_test=200, seed=22) +ka = lk.Kriging(y[:20], X[:20], "matern5_2") +kb = lk.Kriging(y[:20], X[:20], "matern5_2") +for i in range(20, 40, 5): + ka.update(y[i:i + 5], X[i:i + 5], refit=False) +kb.update(y[20:], X[20:], refit=False) +suite.close("4 updates of 5 == 1 update of 20 (mean)", pred(ka, Xt)[0], pred(kb, Xt)[0], rtol=1e-8, atol=1e-10) +suite.close("4 updates of 5 == 1 update of 20 (stdev)", pred(ka, Xt)[1], pred(kb, Xt)[1], rtol=1e-6, atol=1e-10) + +# %% [markdown] +# ## `simulate`: moments, seeds, conditioning + +# %% +for fname, n in [("branin", 20), ("hartmann3", 30)]: + tf = TESTS[fname] + X, y, Xt, _ = design.dataset(tf, n, n_test=15, seed=23) + k = lk.Kriging(y, X, "matern5_2") + nsim = 5000 + S = np.asarray(k.simulate(nsim, 123, Xt)) + suite.true(f"{fname}: simulate shape (q, nsim)", S.shape == (len(Xt), nsim), detail=str(S.shape)) + m, s, c = pred(k, Xt, cov=True) + z = (S.mean(axis=1) - m) / (s / np.sqrt(nsim)) + suite.less(f"{fname}: empirical mean of paths vs predict (max |z|)", np.max(np.abs(z)), 5.0) + rel = np.linalg.norm(np.cov(S) - c) / np.linalg.norm(c) + suite.less(f"{fname}: empirical covariance of paths vs predict (rel. Frobenius)", rel, 0.1) + S2 = np.asarray(k.simulate(nsim, 123, Xt)) + suite.close(f"{fname}: same seed, same paths", S2, S, rtol=0, atol=0) + S3 = np.asarray(k.simulate(nsim, 124, Xt)) + suite.true(f"{fname}: another seed, other paths", not np.allclose(S3, S)) + Sx = np.asarray(k.simulate(20, 1, X[:5])) + # up to the numerical jitter added to the conditional covariance (relative to the prior sd) + suite.close(f"{fname}: paths reproduce y at the design points", Sx, np.tile(y[:5, None], (1, 20)), + rtol=0, atol=1e-4 * np.sqrt(k.sigma2())) + +# %% [markdown] +# ## `update_simulate`: paths conditioned on new data +# +# Paths drawn with `will_update=True`, then corrected by `update_simulate(y_u, +# X_u)`, must have the moments of the model updated with (X_u, y_u). + +# %% +tf = TESTS["branin"] +X, y, Xt, _ = design.dataset(tf, 25, n_test=12, seed=24) +n0 = 20 +k = lk.Kriging(y[:n0], X[:n0], "matern5_2") +p = fixed_params(k) +nsim = 5000 +with suite.runs("update_simulate runs"): + k.simulate(nsim, 7, Xt, will_update=True) + U = np.asarray(k.update_simulate(y[n0:], X[n0:])) + # paths drawn under (theta, sigma2) of the first model, then conditioned on all the data + ref = lk.Kriging(y, X, "matern5_2", optim="none", parameters=p) + m, s, c = pred(ref, Xt, cov=True) + z = (U.mean(axis=1) - m) / (s / np.sqrt(nsim)) + suite.less("update_simulate: mean of corrected paths vs updated model (max |z|)", np.max(np.abs(z)), 5.0) + rel = np.linalg.norm(np.cov(U) - c) / np.linalg.norm(c) + suite.less("update_simulate: covariance vs updated model (rel. Frobenius)", rel, 0.1) + +# %% +suite.finalize() diff --git a/notebooks/python/src/05_warpkriging.py b/notebooks/python/src/05_warpkriging.py new file mode 100644 index 0000000..05d4477 --- /dev/null +++ b/notebooks/python/src/05_warpkriging.py @@ -0,0 +1,183 @@ +# %% [markdown] +# # 05 — `WarpKriging`: non-stationarity, categorical and ordinal inputs +# +# `WarpKriging` learns a transform per input variable jointly with the GP +# hyperparameters. Problems where a plain stationary GP is the wrong model: +# +# | case | function | warp | +# |---|---|---| +# | 1D non-stationary | Gramacy & Lee (2012): fast oscillation on the left, flat on the right | `kumaraswamy`, `knots(5)`, `neural_mono(8)`, `boxcox`, `affine` | +# | 2D non-stationary | Gramacy & Lee (2008): a bump in a corner of [−2, 6]² | `kumaraswamy` × 2 | +# | mixed continuous / categorical | Branin + a level-dependent shift and scale | `categorical(5,2)`, `none` | +# | mixed continuous / ordinal | Hartmann 3 with x₁ observed on 6 ordered levels | `ordinal(6)` | +# +# Checks: all-`none` warping reproduces plain `Kriging`; a noise-free +# `WarpKriging` interpolates; predictions are finite; and (warn-level) the warps +# beat plain `Kriging` where they are meant to. + +# %% +import numpy as np +import pandas as pd +import matplotlib.pyplot as plt +import pylibkriging as lk + +import lkbench +from lkbench import TESTS, design, metrics, compat + +suite = lkbench.Suite("py05_warpkriging") +print("pylibkriging", compat.version()) +if not compat.has("WarpKriging"): + suite.skip("WarpKriging", "not available in this pylibkriging") + + +def pred(k, X): + m, s, *_ = k.predict(X, return_stdev=True) + return np.ravel(m), np.ravel(s) + + +BASE_NLPD = {} + + +def evaluate(case, label, k, X, y, Xt, yt): + m, s = pred(k, Xt) + res = metrics.all_metrics(yt, m, s) + # calibration: a warp must not make the stdev collapse where the error is not small + if label.startswith("Kriging"): + BASE_NLPD[case] = res["nlpd"] + else: + suite.less(f"{case}/{label}: NLPD - NLPD(Kriging) (overconfident stdev?)", res["nlpd"] - BASE_NLPD[case], + 2.0, severity="warn", detail=f"min stdev {np.min(s):.3g}, max |error|/stdev " + f"{np.max(np.abs(yt - m) / np.maximum(s, 1e-300)):.3g}") + suite.metric(case, label, ll=float(k.logLikelihood()), **res) + suite.finite(f"{case}/{label}: predictions finite", m) + mX, _ = pred(k, X) + suite.less(f"{case}/{label}: interpolates (max |m(X)-y| / sd(y))", np.max(np.abs(mX - y)) / np.std(y), 1e-3, + severity="warn") + return res + +# %% [markdown] +# ## `none` warping == plain Kriging + +# %% +if compat.has("WarpKriging"): + tf = TESTS["branin"] + X, y, Xt, yt = design.dataset(tf, 30, n_test=500, seed=31) + for kernel in ["gauss", "matern5_2"]: + kp = lk.Kriging(y, X, kernel) + kw = lk.WarpKriging(y, X, ["none", "none"], kernel) + suite.close(f"{kernel}: WarpKriging(none) LL == Kriging LL", kw.logLikelihood(), kp.logLikelihood(), + rtol=1e-3, severity="warn") + suite.close(f"{kernel}: WarpKriging(none) mean == Kriging mean", pred(kw, Xt)[0], pred(kp, Xt)[0], + rtol=1e-2, atol=1e-2 * np.std(y), severity="warn") + +# %% [markdown] +# ## 1D non-stationary: Gramacy & Lee (2012) + +# %% +WARPS_1D = ["affine", "boxcox", "kumaraswamy", "knots(5)", "neural_mono(8)"] +if compat.has("WarpKriging"): + tf = TESTS["gramacy_lee_2012"] + X, y, Xt, yt = design.dataset(tf, 25, n_test=1000, seed=32) + X = X * 0.98 + 0.01; Xt = Xt * 0.98 + 0.01 # strictly inside (0,1) for boxcox/kumaraswamy + y, yt = tf.unit(X), tf.unit(Xt) + base = evaluate("gl2012", "Kriging", lk.Kriging(y, X, "matern5_2"), X, y, Xt, yt) + fits = {} + for w in WARPS_1D: + with suite.runs(f"gl2012/{w}: fit"): + fits[w] = lk.WarpKriging(y, X, [w], "matern5_2") + evaluate("gl2012", w, fits[w], X, y, Xt, yt) + t = suite.table().loc["gl2012"] + best = t.drop(index="Kriging")["rmse"].min() + suite.less("gl2012: best warp RMSE / Kriging RMSE", best / base["rmse"], 1.0, severity="warn") + u = np.linspace(0.01, 0.99, 400)[:, None] + fig, ax = plt.subplots(figsize=(11, 4)) + ax.plot(u, tf.unit(u), "k--", lw=2, label="truth") + ax.plot(u, pred(lk.Kriging(y, X, "matern5_2"), u)[0], label="Kriging") + for w in ["kumaraswamy", "knots(5)", "neural_mono(8)"]: + if w in fits: + ax.plot(u, pred(fits[w], u)[0], label=w) + ax.plot(X, y, "ko"); ax.legend(); ax.set_title("Gramacy & Lee (2012), n=25") + display(t.round(4)) + +# %% [markdown] +# ## 2D non-stationary: Gramacy & Lee (2008) + +# %% +if compat.has("WarpKriging"): + tf = TESTS["gramacy_lee_2008"] + X, y, Xt, yt = design.dataset(tf, 40, n_test=2000, seed=33) + X = X * 0.98 + 0.01; Xt = Xt * 0.98 + 0.01 + y, yt = tf.unit(X), tf.unit(Xt) + base = evaluate("gl2008", "Kriging", lk.Kriging(y, X, "matern5_2"), X, y, Xt, yt) + with suite.runs("gl2008/kumaraswamy: fit"): + res = evaluate("gl2008", "kumaraswamy", lk.WarpKriging(y, X, ["kumaraswamy"] * 2, "matern5_2"), + X, y, Xt, yt) + suite.less("gl2008: RMSE(kumaraswamy) / RMSE(Kriging)", res["rmse"] / base["rmse"], 1.0, severity="warn") + +# %% [markdown] +# ## Mixed continuous / categorical +# +# y = a_c · Branin(x₁, x₂) + b_c for a level c ∈ {0..4} with level-specific +# (a_c, b_c): two pairs of levels are close, the fifth one is different. The +# categorical variable is passed as its level index 0..L−1. + +# %% +if compat.has("WarpKriging"): + tf = TESTS["branin"] + L = 5 + a = np.array([1.0, 1.05, 0.6, 0.65, -0.5]) + b = np.array([0.0, 5.0, 40.0, 45.0, 100.0]) + + def mixed(Xc): + c = Xc[:, 0].astype(int) + return a[c] * tf.unit(Xc[:, 1:]) + b[c] + + def mixed_design(n, seed): + rng = np.random.default_rng(seed) + Xc = np.column_stack([np.arange(n) % L, design.lhs(n, 2, seed=seed)]) + return Xc[rng.permutation(n)] + + X = mixed_design(60, 34); Xt = mixed_design(1000, 35) + y, yt = mixed(X), mixed(Xt) + base = evaluate("mixed", "Kriging (level as number)", lk.Kriging(y, X, "matern5_2"), X, y, Xt, yt) + with suite.runs("mixed/categorical: fit"): + kc = lk.WarpKriging(y, X, [f"categorical({L},2)", "none", "none"], "matern5_2") + res = evaluate("mixed", "categorical(5,2)", kc, X, y, Xt, yt) + suite.less("mixed: RMSE(categorical) / RMSE(level as number)", res["rmse"] / base["rmse"], 1.0, + severity="warn") + suite.greater("mixed/categorical: Q2", res["q2"], 0.8, severity="warn") + +# %% [markdown] +# ## Mixed continuous / ordinal +# +# Hartmann 3 with x₁ only observed on 6 non-equally spaced ordered levels; the +# model sees the level rank 0..5. + +# %% +if compat.has("WarpKriging"): + tf = TESTS["hartmann3"] + levels = np.array([0.0, 0.05, 0.15, 0.4, 0.8, 1.0]) + + def ordinal_design(n, seed): + U = design.lhs(n, 3, seed=seed) + r = np.minimum((U[:, 0] * 6).astype(int), 5) + return np.column_stack([r, U[:, 1:]]).astype(float) + + def f_ord(Xo): + return tf.unit(np.column_stack([levels[Xo[:, 0].astype(int)], Xo[:, 1:]])) + + X = ordinal_design(60, 36); Xt = ordinal_design(1000, 37) + y, yt = f_ord(X), f_ord(Xt) + base = evaluate("ordinal", "Kriging (rank as number)", lk.Kriging(y, X / [5, 1, 1], "matern5_2"), + X / [5, 1, 1], y, Xt / [5, 1, 1], yt) + with suite.runs("ordinal/ordinal(6): fit"): + ko = lk.WarpKriging(y, X, ["ordinal(6)", "none", "none"], "matern5_2") + res = evaluate("ordinal", "ordinal(6)", ko, X, y, Xt, yt) + suite.less("ordinal: RMSE(ordinal) / RMSE(rank as number)", res["rmse"] / base["rmse"], 1.1, + severity="warn") + +# %% +suite.table().round(4) + +# %% +suite.finalize() diff --git a/notebooks/python/src/06_high_dimension_mlp.py b/notebooks/python/src/06_high_dimension_mlp.py new file mode 100644 index 0000000..7888da7 --- /dev/null +++ b/notebooks/python/src/06_high_dimension_mlp.py @@ -0,0 +1,140 @@ +# %% [markdown] +# # 06 — Moderate and high dimension: engineering functions, `normalize`, `MLPKriging` +# +# | function | d | character | +# |---|---|---| +# | OTL circuit | 6 | smooth, inputs on very different scales | +# | Piston | 7 | smooth, strongly non-linear in a few inputs | +# | Borehole | 8 | smooth, 2–3 dominant inputs, raw scales from 0.05 to 10⁵ | +# | Robot arm | 8 | oscillating (angles): hard for a stationary GP | +# | Dette & Pepelyshev | 8 | curved, logarithmic terms | +# | Wing weight | 10 | smooth, nearly additive | +# | Welch et al. (1992) | 20 | screening problem: few active inputs, interactions | +# +# Checks: +# * **`normalize=True` on raw inputs** gives the same accuracy as an explicit +# rescaling to [0, 1] (they are the same model up to the optimizer); +# inputs in native units (Borehole spans 0.05 to 50 000) must not break a fit; +# * Q² floors per function (regression detection); +# * the fitted θ of Welch's **inactive** inputs are larger than those of the +# active ones (screening ability, warn-level); +# * `MLPKriging` (deep kernel) on a function with a low-dimensional structure: +# finite predictions; with a multistart optimizer, a Q² close to plain +# `Kriging`'s (error-level floor); the default single-start fit is compared +# with the multistart one (warn-level). + +# %% +import time + +import numpy as np +import pandas as pd +import pylibkriging as lk + +import lkbench +from lkbench import TESTS, design, metrics, compat + +suite = lkbench.Suite("py06_high_dimension_mlp") +print("pylibkriging", compat.version()) + + +def pred(k, X): + m, s, *_ = k.predict(X, return_stdev=True) + return np.ravel(m), np.ravel(s) + +# %% [markdown] +# ## Accuracy, unit-cube inputs vs native inputs with `normalize=True` + +# %% +CASES = { # function: (n, Q2 floor on the unit cube) + "otl_circuit": (60, 0.999), + "piston": (70, 0.98), + "borehole": (80, 0.995), + "robot_arm": (160, 0.45), + "dette_pepelyshev": (80, 0.99), + "wing_weight": (100, 0.999), + "welch": (200, 0.98), +} +for fname, (n, floor) in CASES.items(): + tf = TESTS[fname] + U, y, Ut, yt = design.dataset(tf, n, n_test=2000, seed=41) + t0 = time.time() + k = lk.Kriging(y, U, "matern5_2") + fit_s = time.time() - t0 + res = metrics.all_metrics(yt, *pred(k, Ut)) + suite.metric(fname, "unit cube", fit_s=fit_s, **res) + suite.greater(f"{fname}: Q2 (unit cube, matern5_2)", res["q2"], floor) + # same problem in native units, rescaled internally + Xn, Xtn = tf.scale(U), tf.scale(Ut) + with suite.runs(f"{fname}: fit on native inputs with normalize=True"): + kn = lk.Kriging(y, Xn, "matern5_2", normalize=True) + res_n = metrics.all_metrics(yt, *pred(kn, Xtn)) + suite.metric(fname, "native + normalize", **res_n) + suite.greater(f"{fname}: Q2(normalize) - Q2(unit cube)", res_n["q2"] - res["q2"], -0.02) +tab = suite.table() +tab.round(4) + +# %% [markdown] +# ## Screening: Welch et al. (1992) +# +# Inputs 8 and 16 do not appear in the function; inputs 1, 4, 5, 12, 19, 20 are +# the most influential ones. A small θ_j (short correlation length) means an +# influential input, so the inactive inputs should get the largest θ. + +# %% +tf = TESTS["welch"] +U, y, _, _ = design.dataset(tf, 200, seed=42) +k = lk.Kriging(y, U, "matern5_2") +theta = np.ravel(k.theta()) +inactive = [7, 15] # 0-based indices of x8, x16 +active = [0, 3, 4, 11, 18, 19] # x1, x4, x5, x12, x19, x20 +suite.greater("welch: min theta(inactive) / max theta(active)", theta[inactive].min() / theta[active].max(), 1.0, + severity="warn") +pd.Series(theta, index=[f"x{i + 1}" for i in range(20)]).round(3).to_frame("theta").T + +# %% [markdown] +# ## `MLPKriging`: deep kernel on a low-dimensional structure +# +# f(x) = Branin(Wx) for x ∈ [0,1]¹⁰ and a fixed 2 × 10 projection W: the +# function only varies in a 2D subspace, which a joint feature map can find +# whereas a per-axis kernel cannot. + +# %% +if compat.has("MLPKriging"): + rng = np.random.default_rng(43) + W = rng.normal(size=(2, 10)) + W /= np.abs(W).sum(axis=1, keepdims=True) + tfb = TESTS["branin"] + + def projected(X): + Z = X @ W.T + Z = (Z - (W.clip(max=0).sum(axis=1))) / np.abs(W).sum(axis=1) # back to [0,1]^2 + return tfb.unit(Z) + + X = design.lhs(120, 10, seed=44); Xt = design.uniform(2000, 10, seed=45) + y, yt = projected(X), projected(Xt) + kp = lk.Kriging(y, X, "matern5_2") + res_k = metrics.all_metrics(yt, *pred(kp, Xt)) + suite.metric("projected_branin_10d", "Kriging", **res_k) + for act in ["tanh", "selu"]: + for optim in ["BFGS", "BFGS10"]: + case = f"MLPKriging({act}, {optim})" + with suite.runs(f"{case}: fit"): + t0 = time.time() + km = lk.MLPKriging(y, X, hidden_dims=[16, 8], d_out=2, activation=act, kernel="matern5_2", + optim=optim) + fit_s = time.time() - t0 + m, s = pred(km, Xt) + suite.finite(f"{case}: predictions finite", m) + res = metrics.all_metrics(yt, m, s) + suite.metric("projected_branin_10d", f"MLPKriging {act} {optim}", fit_s=fit_s, + ll=float(km.logLikelihood()), **res) + if optim == "BFGS10": + suite.greater(f"{case}: Q2", res["q2"], 0.8) + else: + suite.greater(f"{case}: Q2 - Q2(Kriging)", res["q2"] - res_k["q2"], -0.1, severity="warn") + display(suite.table().loc["projected_branin_10d"].round(4)) +else: + suite.skip("MLPKriging", "not available in this pylibkriging") + +# %% +suite.finalize() diff --git a/notebooks/python/src/07_scalability.py b/notebooks/python/src/07_scalability.py new file mode 100644 index 0000000..29ab65a --- /dev/null +++ b/notebooks/python/src/07_scalability.py @@ -0,0 +1,147 @@ +# %% [markdown] +# # 07 — Large n: Vecchia, Nyström, NestedKriging, subset of data +# +# The exact fit costs O(n³). libKriging offers: +# +# * `method="vecchia(m)"`: Vecchia likelihood with m conditioning neighbours +# (local, recommended in low dimension); +# * `method="nystrom(k)"`: low-rank Nyström approximation with k landmarks +# (global); +# * `NestedKriging`: a partition into groups, one sub-model per group, +# aggregated predictions (`NK`, `PoE`, `gPoE`, `BCM`, `rBCM`); +# * `Kriging.subsetOfData`: a representative subset before an exact fit. +# +# Problems: Hartmann 3 (3D, local structure: Vecchia's ground) and the OTL +# circuit (6D, smooth: Nyström's ground), n = 1500 (exact fit still affordable, +# for reference) and n = 5000 (approximations only). +# +# Checks: +# * **limit cases**: Vecchia with m = n − 1 neighbours *is* the exact likelihood; +# * the method is explicit at prediction time: `predict(method="exact")` on a +# Nyström/Vecchia fit raises, `method=` gives finite predictions; +# * accuracy of each approximation close to the exact fit (warn-level gaps, +# error-level floors) and faster than the exact fit at n = 1500 (warn-level); +# * `NestedKriging` `NK` interpolates the data (it is a proper kriging +# predictor); the PoE family does not need to. + +# %% +import time + +import numpy as np +import pandas as pd +import pylibkriging as lk + +import lkbench +from lkbench import TESTS, design, metrics, compat + +suite = lkbench.Suite("py07_scalability") +print("pylibkriging", compat.version()) +HAS_METHOD = compat.has("method") +if not HAS_METHOD: + suite.skip("method=vecchia/nystrom", "pylibkriging without the method= argument") + + +def pred(k, X, **kw): + m, s, *_ = k.predict(X, return_stdev=True, **kw) + return np.ravel(m), np.ravel(s) + + +def timed(f): + t0 = time.time() + out = f() + return out, time.time() - t0 + +# %% [markdown] +# ## Limit case: Vecchia with all the previous points as neighbours is exact + +# %% +if HAS_METHOD: + tf = TESTS["branin"] + X, y, Xt, _ = design.dataset(tf, 40, n_test=100, seed=51) + ke = lk.Kriging(y, X, "matern5_2") + fixed = {"theta": np.asarray(ke.theta()).reshape(1, -1), "is_theta_estim": False} + kv = lk.Kriging(y, X, "matern5_2", optim="none", parameters=fixed, method=f"vecchia({len(y) - 1})") + suite.close("vecchia(n-1) LL == exact LL (same theta)", kv.logLikelihood(method="vecchia"), + ke.logLikelihood(), rtol=1e-6) + suite.close("vecchia(n-1) predict == exact predict", pred(kv, Xt, method="vecchia")[0], pred(ke, Xt)[0], + rtol=1e-5, atol=1e-6 * np.std(y), severity="warn") + kn = lk.Kriging(y, X, "matern5_2", optim="none", parameters=fixed, method=f"nystrom({len(y)})") + suite.close("nystrom(n) LL == exact LL (same theta)", kn.logLikelihood(method="nystrom"), ke.logLikelihood(), + rtol=1e-3, severity="warn") + # the method must be explicit after an approximate fit + suite.raises("nystrom fit: predict(method='exact') raises", lambda: kn.predict(Xt, return_stdev=True, + method="exact")) + suite.true("method() reports the fit method", kv.method().startswith("vecchia") and + kn.method().startswith("nystrom"), detail=f"{kv.method()}, {kn.method()}") + +# %% [markdown] +# ## Accuracy and cost + +# %% +CASES = {"hartmann3": (1500, 5000), "otl_circuit": (1500, 5000)} +FLOORS = {"hartmann3": 0.98, "otl_circuit": 0.99} +for fname, (n_mid, n_big) in CASES.items(): + tf = TESTS[fname] + X, y, Xt, yt = design.dataset(tf, n_big, n_test=2000, seed=52) + Xm, ym = X[:n_mid], y[:n_mid] + ke, t_exact = timed(lambda: lk.Kriging(ym, Xm, "matern5_2")) + res_e = metrics.all_metrics(yt, *pred(ke, Xt)) + suite.metric(f"{fname}/n={n_mid}", "exact", fit_s=t_exact, **res_e) + suite.greater(f"{fname}/n={n_mid}: Q2 exact", res_e["q2"], FLOORS[fname]) + variants = {} + if HAS_METHOD: + variants.update({ + "vecchia(30)": lambda Xa, ya: (lk.Kriging(ya, Xa, "matern5_2", method="vecchia(30)"), "vecchia"), + "nystrom(100)": lambda Xa, ya: (lk.Kriging(ya, Xa, "matern5_2", method="nystrom(100)"), "nystrom"), + }) + if compat.has("NestedKriging"): + variants["nested NK (20 groups)"] = lambda Xa, ya: ( + lk.NestedKriging(ya, Xa, "matern5_2", nb_groups=20, aggregation="NK", seed=1), None) + variants["nested rBCM (20 groups)"] = lambda Xa, ya: ( + lk.NestedKriging(ya, Xa, "matern5_2", nb_groups=20, aggregation="rBCM", seed=1), None) + if compat.has("subsetOfData"): + def sod(Xa, ya): + idx = np.ravel(lk.Kriging.subsetOfData(Xa, n_max=500, method="kmeans", seed=1)).astype(int) + return lk.Kriging(ya[idx], Xa[idx], "matern5_2"), None + variants["subset of data (500)"] = sod + for label, make in variants.items(): + for n in (n_mid, n_big): + case = f"{fname}/n={n}" + with suite.runs(f"{case}/{label}: fit + predict"): + (k, meth), t = timed(lambda: make(X[:n], y[:n])) + m, s = pred(k, Xt, method=meth) if meth else ( + lambda o: (np.ravel(o[0]), np.ravel(o[1])))(k.predict(Xt, return_stdev=True)) + suite.finite(f"{case}/{label}: finite predictions", m) + res = metrics.all_metrics(yt, m, s) + suite.metric(case, label, fit_s=t, **res) + suite.greater(f"{case}/{label}: Q2", res["q2"], FLOORS[fname] - 0.05) + if n == n_mid: + suite.greater(f"{case}/{label}: Q2 - Q2(exact)", res["q2"] - res_e["q2"], -0.02, + severity="warn") + suite.less(f"{case}/{label}: fit time / exact fit time", t / t_exact, 1.0, severity="warn") +suite.table().round(4) + +# %% [markdown] +# ## `NestedKriging` aggregations: interpolation and variance + +# %% +if compat.has("NestedKriging"): + tf = TESTS["branin"] + X, y, Xt, yt = design.dataset(tf, 200, n_test=500, seed=53) + for agg in ["NK", "PoE", "gPoE", "BCM", "rBCM"]: + with suite.runs(f"nested {agg}: fit"): + k = lk.NestedKriging(y, X, "matern5_2", nb_groups=8, aggregation=agg, seed=2) + m, s = (np.ravel(a) for a in k.predict(Xt, return_stdev=True)[:2]) + suite.true(f"nested {agg}: stdev >= 0", np.all(s >= 0)) + suite.metric("branin/nested", agg, **metrics.all_metrics(yt, m, s)) + if agg == "NK": + mX = np.ravel(k.predict(X, return_stdev=True)[0]) + suite.less("nested NK: interpolates (max |m(X)-y| / sd(y))", np.max(np.abs(mX - y)) / np.std(y), + 1e-4) + display(suite.table().loc["branin/nested"].round(4)) + suite.raises("nested NK with a linear trend is rejected", + lambda: lk.NestedKriging(y, X, "matern5_2", nb_groups=4, aggregation="NK", regmodel="linear"), + severity="warn") + +# %% +suite.finalize() diff --git a/notebooks/python/src/08_api_persistence.py b/notebooks/python/src/08_api_persistence.py new file mode 100644 index 0000000..bc362a5 --- /dev/null +++ b/notebooks/python/src/08_api_persistence.py @@ -0,0 +1,178 @@ +# %% [markdown] +# # 08 — API: save/load, pickle, copy, accessors, scikit-learn estimators +# +# Persistence and introspection, for every model class: +# +# * `save` → `load` (class-specific and generic `pylibkriging.load`) gives a model +# with **identical** predictions and hyperparameters; +# * `pickle` / `copy` round trips, and a copy is independent of the original +# (updating one leaves the other untouched); +# * accessors mirror the constructor arguments (`kernel()`, `regmodel()`, +# `optim()`, `objective()`, `X()`, `y()`, ...); +# * scikit-learn estimators (`pylibkriging.sklearn`): `fit`/`predict`, +# `get_params`/`set_params`/`clone`, use inside `Pipeline` and `GridSearchCV`, +# same predictions as the native class. + +# %% +import copy +import pickle +import tempfile +from pathlib import Path + +import numpy as np +import pylibkriging as lk + +import lkbench +from lkbench import TESTS, design, compat + +suite = lkbench.Suite("py08_api_persistence") +print("pylibkriging", compat.version()) +tmp = Path(tempfile.mkdtemp()) + +tf = TESTS["hartmann3"] +X, y, Xt, yt = design.dataset(tf, 40, n_test=100, seed=61) + + +def mean_sd(k, Xn): + out = k.predict(Xn, return_stdev=True) + return np.ravel(out[0]), np.ravel(out[1]) + + +MODELS = { + "Kriging": lambda: lk.Kriging(y, X, "matern5_2", regmodel="linear"), + "Kriging nugget": lambda: lk.Kriging(y, X, "matern3_2", noise="nugget"), + "Kriging noise vector": lambda: lk.Kriging(y, X, "gauss", noise=np.full(len(y), 1e-4)), + "Kriging LOO": lambda: lk.Kriging(y, X, "matern5_2", objective="LOO"), + "Kriging LMP": lambda: lk.Kriging(y, X, "matern5_2", objective="LMP"), +} +if compat.has("method"): + MODELS["Kriging vecchia"] = lambda: lk.Kriging(y, X, "matern5_2", method="vecchia(10)") + MODELS["Kriging nystrom"] = lambda: lk.Kriging(y, X, "matern5_2", method="nystrom(20)") +if compat.has("WarpKriging"): + MODELS["WarpKriging"] = lambda: lk.WarpKriging(y, X, ["kumaraswamy", "none", "affine"], "matern5_2") +if compat.has("MLPKriging"): + MODELS["MLPKriging"] = lambda: lk.MLPKriging(y, X, hidden_dims=[8], d_out=2, activation="tanh", + kernel="matern5_2") +if compat.has("NestedKriging"): + MODELS["NestedKriging"] = lambda: lk.NestedKriging(y, X, "matern5_2", nb_groups=4, seed=1) + + +def predict_any(k, Xn): + meth = k.method() if hasattr(k, "method") and isinstance(k, lk.Kriging) else None + if meth and meth != "exact": + out = k.predict(Xn, return_stdev=True, method=meth.split("(")[0]) + return np.ravel(out[0]), np.ravel(out[1]) + return mean_sd(k, Xn) + +# %% [markdown] +# ## save / load, pickle, copy + +# %% +for name, make in MODELS.items(): + with suite.runs(f"{name}: fit"): + k = make() + m0, s0 = predict_any(k, Xt) + cls = type(k) + path = tmp / f"{name.replace(' ', '_')}.json" + if not hasattr(k, "save"): + suite.skip(f"{name}: save + load", "no save() in this pylibkriging") + continue + with suite.runs(f"{name}: save + load"): + k.save(str(path)) + k1 = cls.load(str(path)) + m1, s1 = predict_any(k1, Xt) + suite.close(f"{name}: load(save(k)) predicts the same mean", m1, m0, rtol=1e-12, atol=1e-12) + suite.close(f"{name}: load(save(k)) predicts the same stdev", s1, s0, rtol=1e-10, atol=1e-12) + if hasattr(k, "theta"): + suite.close(f"{name}: same theta after load", np.ravel(k1.theta()), np.ravel(k.theta()), + rtol=1e-14, atol=0) + if hasattr(lk, "load") and (compat.has("load(noise)") or "noise" not in name and "nugget" not in name): + with suite.runs(f"{name}: generic pylibkriging.load"): + k2 = lk.load(str(path)) + suite.true(f"{name}: generic load returns {cls.__name__}", type(k2).__name__ == cls.__name__, + detail=type(k2).__name__) + suite.close(f"{name}: generic load predicts the same", predict_any(k2, Xt)[0], m0, rtol=1e-12, + atol=1e-12) + if not compat.has("pickle"): + suite.skip(f"{name}: pickle / deepcopy", "models not picklable in this pylibkriging") + continue + with suite.runs(f"{name}: pickle round trip"): + k3 = pickle.loads(pickle.dumps(k)) + suite.close(f"{name}: pickle predicts the same", predict_any(k3, Xt)[0], m0, rtol=1e-12, atol=1e-12) + with suite.runs(f"{name}: deepcopy"): + k4 = copy.deepcopy(k) + suite.close(f"{name}: deepcopy predicts the same", predict_any(k4, Xt)[0], m0, rtol=1e-12, atol=1e-12) + +# %% [markdown] +# A copy is independent: updating it leaves the original model unchanged. + +# %% +k = lk.Kriging(y[:30], X[:30], "matern5_2") +m_before = mean_sd(k, Xt)[0] +kc = k.copy() if hasattr(k, "copy") else copy.deepcopy(k) +kc.update(y[30:], X[30:], refit=False) +suite.close("update on a copy leaves the original unchanged", mean_sd(k, Xt)[0], m_before, rtol=0, atol=0) +suite.true("the updated copy did change", not np.allclose(mean_sd(kc, Xt)[0], m_before)) + +# %% [markdown] +# ## Accessors mirror the constructor + +# %% +k = lk.Kriging(y, X, "matern3_2", regmodel="linear", normalize=False, optim="BFGS", objective="LMP") +for acc, expected in [("kernel", "matern3_2"), ("regmodel", "linear"), ("optim", "BFGS"), ("objective", "LMP"), + ("normalize", False)]: + suite.true(f"Kriging.{acc}() == {expected!r}", getattr(k, acc)() == expected, detail=str(getattr(k, acc)())) +suite.close("Kriging.X() is the design", np.asarray(k.X()), X, rtol=0, atol=0) +suite.close("Kriging.y() is the response", np.ravel(k.y()), y, rtol=0, atol=0) +# with normalize=True: do the accessors return the data as given, or as rescaled internally? +kn = lk.Kriging(y, X * 10, "matern3_2", normalize=True) +suite.true("Kriging.normalize() == True", kn.normalize() is True or kn.normalize() == 1) +suite.close("normalize=True: X() is the design as given", np.asarray(kn.X()), X * 10, rtol=1e-12, atol=0, + severity="warn") +suite.close("normalize=True: y() is the response as given", np.ravel(kn.y()), y, rtol=1e-12, atol=0, + severity="warn") +suite.true("Kriging.beta() has 1 + d coefficients (linear trend)", np.ravel(k.beta()).size == 1 + X.shape[1]) +suite.true("summary() is a non-empty string", isinstance(k.summary(), str) and len(k.summary()) > 20) +suite.true("is_theta_estim() after an optimized fit", bool(k.is_theta_estim())) + +# %% [markdown] +# ## scikit-learn estimators + +# %% +if compat.has("sklearn"): + from sklearn.base import clone + from sklearn.model_selection import GridSearchCV, cross_val_score + from sklearn.pipeline import make_pipeline + from sklearn.preprocessing import MinMaxScaler + from pylibkriging.sklearn import KrigingRegressor + + est = KrigingRegressor(kernel="matern5_2").fit(X, y) + m_sk, s_sk = est.predict(Xt, return_std=True) + suite.true("KrigingRegressor.predict returns 1-D arrays (scikit-learn convention)", + np.ndim(m_sk) == 1 and np.ndim(s_sk) == 1, detail=f"shapes {np.shape(m_sk)}, {np.shape(s_sk)}", + severity="warn") + m_sk, s_sk = np.ravel(m_sk), np.ravel(s_sk) + m_nat, s_nat = mean_sd(lk.Kriging(y, X, "matern5_2"), Xt) + suite.close("KrigingRegressor == Kriging (mean)", m_sk, m_nat, rtol=1e-8, atol=1e-10) + suite.close("KrigingRegressor == Kriging (stdev)", s_sk, s_nat, rtol=1e-6, atol=1e-10) + est2 = clone(est).set_params(kernel="gauss") + suite.true("clone + set_params", est2.get_params()["kernel"] == "gauss" and est.get_params()["kernel"] == + "matern5_2") + with suite.runs("Pipeline(MinMaxScaler, KrigingRegressor)"): + pipe = make_pipeline(MinMaxScaler(), KrigingRegressor(kernel="matern5_2")).fit(tf.scale(X), y) + suite.greater("pipeline Q2 on native inputs", pipe.score(tf.scale(Xt), yt), 0.8) + with suite.runs("GridSearchCV over kernels"): + gs = GridSearchCV(KrigingRegressor(), {"kernel": ["gauss", "matern3_2", "matern5_2"]}, cv=3).fit(X, y) + suite.true("GridSearchCV picked a kernel", gs.best_params_["kernel"] in ("gauss", "matern3_2", "matern5_2"), + detail=str(gs.best_params_)) + with suite.runs("cross_val_score"): + sc = cross_val_score(KrigingRegressor(kernel="matern5_2"), X, y, cv=4) + suite.finite("cross_val_score finite", sc) + with suite.runs("sample_y"): + S = est.sample_y(Xt[:10], n_samples=5, random_state=1) + suite.true("sample_y shape (q, n_samples)", np.shape(S) == (10, 5), detail=str(np.shape(S))) +else: + suite.skip("sklearn estimators", "pylibkriging.sklearn or scikit-learn not available") + +# %% +suite.finalize() diff --git a/notebooks/python/src/09_robustness.py b/notebooks/python/src/09_robustness.py new file mode 100644 index 0000000..71d7000 --- /dev/null +++ b/notebooks/python/src/09_robustness.py @@ -0,0 +1,164 @@ +# %% [markdown] +# # 09 — Robustness: degenerate and adversarial inputs +# +# What a user eventually feeds a surrogate model, and what must happen: +# +# | input | expected behaviour | +# |---|---| +# | NaN / Inf in X or y | rejected with an exception (never a silent NaN model) | +# | X and y with inconsistent sizes | rejected | +# | a wrong number of warping specs | rejected | +# | duplicated design points, noise-free | either rejected or a finite model (no NaN, no crash) | +# | duplicated design points with `noise="nugget"` | a finite model | +# | constant response y | a finite model predicting the constant | +# | very few points (n = 2, 3) | a finite model | +# | very clustered points (1e-8 apart) with the `gauss` kernel | finite predictions (ill-conditioning) | +# | inputs scaled by 1e6 or 1e-6 with `normalize=True` | the same predictions as on [0,1] | +# | y scaled by 1e8 / shifted by 1e6 | predictions scaled / shifted accordingly (equivariance) | +# | predicting far outside the design | the mean reverts to the trend, the stdev to (slightly above) √σ² | +# | many prediction points | cost linear in their number (mean and stdev only need O(n·q), not O(q²)) | + +# %% +import time + +import numpy as np +import pylibkriging as lk + +import lkbench +from lkbench import TESTS, design, compat + +suite = lkbench.Suite("py09_robustness") +print("pylibkriging", compat.version()) +tf = TESTS["branin"] +X, y, Xt, yt = design.dataset(tf, 20, n_test=200, seed=71) + + +def pred(k, Xn): + m, s, *_ = k.predict(Xn, return_stdev=True) + return np.ravel(m), np.ravel(s) + + +def finite_model(check, make, Xn=Xt, severity="error"): + """The model either raises (acceptable) or predicts finite values.""" + try: + k = make() + except Exception as exc: # noqa: BLE001 + suite.true(f"{check}: rejected or finite", True, detail=f"rejected: {type(exc).__name__}") + return None + m, s = pred(k, Xn) + suite.true(f"{check}: rejected or finite", np.all(np.isfinite(m)) and np.all(np.isfinite(s)), + detail=f"non-finite: {np.sum(~np.isfinite(m))} mean, {np.sum(~np.isfinite(s))} stdev", + severity=severity) + return k + +# %% [markdown] +# ## Invalid inputs are rejected + +# %% +Xn = X.copy(); Xn[3, 1] = np.nan +yn = y.copy(); yn[5] = np.nan +yi = y.copy(); yi[5] = np.inf +suite.raises("NaN in X is rejected", lambda: lk.Kriging(y, Xn, "matern5_2")) +suite.raises("NaN in y is rejected", lambda: lk.Kriging(yn, X, "matern5_2")) +suite.raises("Inf in y is rejected", lambda: lk.Kriging(yi, X, "matern5_2")) +suite.raises("len(y) != nrow(X) is rejected", lambda: lk.Kriging(y[:-1], X, "matern5_2")) +suite.raises("unknown kernel is rejected", lambda: lk.Kriging(y, X, "matern7_2")) +suite.raises("unknown trend is rejected", lambda: lk.Kriging(y, X, "matern5_2", regmodel="cubic")) +suite.raises("unknown objective is rejected", lambda: lk.Kriging(y, X, "matern5_2", objective="XYZ")) +suite.raises("noise vector of the wrong length is rejected", + lambda: lk.Kriging(y, X, "matern5_2", noise=np.ones(len(y) - 1))) +suite.raises("negative noise variance is rejected", + lambda: lk.Kriging(y, X, "matern5_2", noise=-np.ones(len(y))), severity="warn") +k = lk.Kriging(y, X, "matern5_2") +suite.raises("predict with the wrong number of columns is rejected", lambda: k.predict(np.ones((3, 3)), True)) +if compat.has("WarpKriging"): + suite.raises("WarpKriging with 1 spec for 2 columns is rejected", lambda: lk.WarpKriging(y, X, ["none"], "gauss")) + suite.raises("WarpKriging with an unknown spec is rejected", + lambda: lk.WarpKriging(y, X, ["none", "spline"], "gauss")) + +# %% [markdown] +# ## Degenerate designs and responses + +# %% +Xd = np.vstack([X, X[:3]]); yd = np.concatenate([y, y[:3]]) +finite_model("duplicated points, noise-free", lambda: lk.Kriging(yd, Xd, "matern5_2")) +if compat.has("noise"): + k = finite_model("duplicated points, nugget", lambda: lk.Kriging(yd, Xd, "matern5_2", noise="nugget")) + suite.true("duplicated points, nugget: a model is built", k is not None) +kc = finite_model("constant y", lambda: lk.Kriging(np.full(20, 3.7), X, "matern5_2")) +if kc is not None: + suite.close("constant y: predicts the constant", pred(kc, Xt)[0], np.full(len(Xt), 3.7), rtol=1e-6, atol=1e-6) +for n in (2, 3): + finite_model(f"n = {n}", lambda: lk.Kriging(y[:n], X[:n], "matern5_2")) +Xclose = np.vstack([X[:10], X[:10] + 1e-8]) +yclose = tf.unit(Xclose) +for kernel in ("gauss", "matern5_2"): + finite_model(f"points 1e-8 apart, {kernel}", lambda: lk.Kriging(yclose, Xclose, kernel)) +# a single input dimension constant over the design +Xconst = np.column_stack([X[:, 0], np.full(20, 0.5)]) +finite_model("one input constant over the design", lambda: lk.Kriging(tf.unit(Xconst), Xconst, "matern5_2")) + +# %% [markdown] +# ## Scale invariance and equivariance + +# %% +k0 = lk.Kriging(y, X, "matern5_2", normalize=True) +m0, s0 = pred(k0, Xt) +for scale in (1e-6, 1e6): + with suite.runs(f"X * {scale:g} with normalize=True: fit"): + ks = lk.Kriging(y, X * scale, "matern5_2", normalize=True) + suite.close(f"X * {scale:g}, normalize=True: same mean as on [0,1]", pred(ks, Xt * scale)[0], m0, + rtol=1e-3, atol=1e-3 * np.std(y)) +for a, b in [(1e8, 0.0), (1.0, 1e6), (1e-8, 0.0)]: + with suite.runs(f"y * {a:g} + {b:g}: fit"): + ka = lk.Kriging(a * y + b, X, "matern5_2", normalize=True) + ma, sa = pred(ka, Xt) + suite.close(f"y * {a:g} + {b:g}: mean equivariant", ma, a * m0 + b, rtol=1e-3, atol=1e-3 * a * np.std(y)) + suite.close(f"y * {a:g} + {b:g}: stdev scales by |a|", sa, abs(a) * s0, rtol=1e-2, + atol=1e-3 * a * np.std(y)) +# the same without normalize: the optimizer starts from a scale-dependent point, warn-level only +for a, b in [(1e6, 0.0), (1.0, 1e6)]: + ka = lk.Kriging(a * y + b, X, "matern5_2") + suite.close(f"y * {a:g} + {b:g} (normalize=False): mean equivariant", pred(ka, Xt)[0], + a * pred(lk.Kriging(y, X, "matern5_2"), Xt)[0] + b, rtol=1e-3, atol=1e-3 * a * np.std(y), + severity="warn") + +# %% [markdown] +# ## Far from the data + +# %% +k = lk.Kriging(y, X, "matern5_2") +far = np.array([[50.0, 50.0], [-30.0, 80.0]]) +m, s = pred(k, far) +suite.close("far away: mean -> beta (constant trend)", m, np.full(2, np.ravel(k.beta())[0]), rtol=1e-6, + atol=1e-8) +# universal kriging: sigma2 * (1 + 1 / (1' R^-1 1)) with the trend uncertainty, slightly above sigma2 +suite.between("far away: stdev / sqrt(sigma2)", np.max(s) / np.sqrt(k.sigma2()), 1.0 - 1e-6, 1.5) +suite.close("far away: same stdev at both points", s[0], s[1], rtol=1e-6) + +# %% [markdown] +# ## Many prediction points +# +# Mean and stdev at q points need O(n·q) operations and memory; only the full +# covariance (`return_cov=True`) is O(q²). The time for q = 1000, 2000, 4000, +# 8000 points must grow linearly (log2 of the ratio per doubling ~ 1). An +# exponent close to 2 means a q × q matrix is formed: a million points then needs +# terabytes. (Not run at 10⁶ points directly: it would exhaust the runner.) + +# %% +times = {} +for q in (1000, 2000, 4000, 8000): + Xq = design.uniform(q, 2, seed=72) + best = np.inf + for _ in range(3): + t0 = time.perf_counter() + pred(k, Xq) + best = min(best, time.perf_counter() - t0) + times[q] = best + suite.metric(f"predict_q={q}", "pylibkriging", seconds=best) +expo = np.log2(times[8000] / times[4000]) +suite.less("predict(stdev) time growth exponent in q (8000 vs 4000)", expo, 1.5, + detail=", ".join(f"q={q}: {t:.3f}s" for q, t in times.items())) + +# %% +suite.finalize() diff --git a/notebooks/python/src/10_compare_python_packages.py b/notebooks/python/src/10_compare_python_packages.py new file mode 100644 index 0000000..a291ec8 --- /dev/null +++ b/notebooks/python/src/10_compare_python_packages.py @@ -0,0 +1,215 @@ +# %% [markdown] +# # 10 — Comparison with Python GP packages +# +# Two kinds of comparisons, which should not be confused: +# +# 1. **Same model, same hyperparameters → same numbers.** With θ, σ², β fixed, +# libKriging's predictor and likelihood are textbook formulas: another +# implementation must give the same values to rounding error. Done against +# scikit-learn (`RBF` = libKriging's `gauss`, both product-form) and +# OpenTURNS (product of 1D Matérn 5/2 = libKriging's `matern5_2`). A gap +# here is a bug in one of the packages (error-level). +# 2. **Same problem, each package's default fit → accuracy and cost.** Each +# package estimates its hyperparameters its own way (optimizer, restarts, +# bounds, mean model), so differences are expected; the table is a +# benchmark, and the checks are warn-level: libKriging should not be +# markedly less accurate than the best package. +# +# Packages (each optional: skipped if not installed): scikit-learn, SMT, +# OpenTURNS, GPy. + +# %% +import time +import warnings + +import numpy as np +import pandas as pd +import pylibkriging as lk + +import lkbench +from lkbench import TESTS, design, metrics, compat + +warnings.filterwarnings("ignore") +suite = lkbench.Suite("py10_compare_python_packages") +sk = compat.optional("sklearn.gaussian_process") +smt = compat.optional("smt.surrogate_models") +ot = compat.optional("openturns") +GPy = compat.optional("GPy") +print("pylibkriging", compat.version(), "| sklearn", bool(sk), "| SMT", bool(smt), "| OpenTURNS", bool(ot), + "| GPy", bool(GPy)) + + +def lk_fixed(y, X, kernel, theta, sigma2, beta): + return lk.Kriging(y, X, kernel, optim="none", + parameters={"theta": np.reshape(theta, (1, -1)), "is_theta_estim": False, + "sigma2": float(sigma2), "is_sigma2_estim": False, + "beta": np.reshape(beta, (-1, 1)), "is_beta_estim": False}) + +# %% [markdown] +# ## 1. Same hyperparameters, same numbers + +# %% +tf = TESTS["hartmann3"] +X, y, Xt, yt = design.dataset(tf, 40, n_test=300, seed=81) +kg = lk.Kriging(y, X, "gauss") +theta, sigma2, beta = np.ravel(kg.theta()), float(kg.sigma2()), float(np.ravel(kg.beta())[0]) +k_simple = lk_fixed(y, X, "gauss", theta, sigma2, beta) # simple kriging, known beta +m_lk, s_lk, *_ = (np.ravel(a) for a in k_simple.predict(Xt, return_stdev=True)[:2]) + +if sk is not None: + from sklearn.gaussian_process import GaussianProcessRegressor + from sklearn.gaussian_process.kernels import RBF, ConstantKernel + + kern = ConstantKernel(sigma2, "fixed") * RBF(theta, "fixed") + gp = GaussianProcessRegressor(kern, alpha=1e-12, optimizer=None, normalize_y=False).fit(X, y - beta) + m_sk, s_sk = gp.predict(Xt, return_std=True) + suite.close("sklearn RBF == libKriging gauss: simple kriging mean", m_sk + beta, m_lk, rtol=1e-6, + atol=1e-7 * np.std(y)) + suite.close("sklearn RBF == libKriging gauss: simple kriging stdev (beta fixed)", s_sk, s_lk, rtol=1e-4, + atol=1e-6 * np.sqrt(sigma2)) + # with beta estimated, libKriging's stdev is the universal kriging one: the simple kriging variance + # plus the trend-estimation term (f - F'C^-1 c)^2 / (F'C^-1 F), computed here by hand + C = kern(X); c = kern(X, Xt) + w = np.linalg.solve(C, c) + one = np.ones((len(y), 1)) + v_uk = s_sk ** 2 + ((1 - one.T @ w) ** 2).ravel() / (one.T @ np.linalg.solve(C, one)).item() + s_uk = np.ravel(kg.predict(Xt, return_stdev=True)[1]) + suite.close("libKriging gauss (beta estimated) stdev == universal kriging formula", s_uk, np.sqrt(v_uk), + rtol=1e-4, atol=1e-6 * np.sqrt(sigma2)) + # the (concentrated) log-likelihood at the MLE equals the full Gaussian log-density at (theta, sigma2, beta) + lml = gp.log_marginal_likelihood_value_ + suite.close("sklearn log-marginal-likelihood == libKriging logLikelihood()", lml, kg.logLikelihood(), + rtol=1e-6) +else: + suite.skip("sklearn exact comparison", "scikit-learn not installed") + +# %% +if ot is not None: + km = lk.Kriging(y, X, "matern5_2") + th, s2 = np.ravel(km.theta()), float(km.sigma2()) + cov = ot.ProductCovarianceModel([ot.MaternModel([t], [1.0], 2.5) for t in th]) + cov.setAmplitude([np.sqrt(s2)]) + basis = ot.ConstantBasisFactory(3).build() + algo = ot.KrigingAlgorithm(ot.Sample(X), ot.Sample(y.reshape(-1, 1)), cov, basis) + algo.setOptimizeParameters(False) + ot.ResourceMap.SetAsScalar("GeneralLinearModelAlgorithm-DefaultOptimizationLowerBound", 1e-6) + algo.run() + res = algo.getResult() + m_ot = np.ravel(res.getConditionalMean(ot.Sample(Xt))) + v_ot = np.ravel(res.getConditionalMarginalVariance(ot.Sample(Xt))) + # OpenTURNS re-estimates beta by GLS: compare with libKriging's universal kriging at the same theta, sigma2 + ku = lk.Kriging(y, X, "matern5_2", optim="none", + parameters={"theta": th.reshape(1, -1), "is_theta_estim": False, + "sigma2": s2, "is_sigma2_estim": False}) + m_u, s_u = (np.ravel(a) for a in ku.predict(Xt, return_stdev=True)[:2]) + suite.close("OpenTURNS product Matern 5/2 == libKriging matern5_2: mean", m_ot, m_u, rtol=1e-6, + atol=1e-6 * np.std(y)) + suite.close("OpenTURNS product Matern 5/2 == libKriging matern5_2: variance (universal kriging)", v_ot, s_u ** 2, + rtol=1e-4, atol=1e-8 * s2) +else: + suite.skip("OpenTURNS exact comparison", "openturns not installed") + +# %% [markdown] +# ## 2. Default fits on a set of test functions +# +# Matérn 5/2 (or the closest the package offers), constant mean, maximum +# likelihood with each package's default optimizer, 3 designs per case. + +# %% +def fit_libkriging(X, y): + k = lk.Kriging(y, X, "matern5_2") + return lambda Z: tuple(np.ravel(a) for a in k.predict(Z, return_stdev=True)[:2]) + + +def fit_libkriging_ms(X, y): + k = lk.Kriging(y, X, "matern5_2", optim="BFGS10") + return lambda Z: tuple(np.ravel(a) for a in k.predict(Z, return_stdev=True)[:2]) + + +PACKAGES = {"pylibkriging": fit_libkriging, "pylibkriging BFGS10": fit_libkriging_ms} + +if sk is not None: + from sklearn.gaussian_process.kernels import Matern + + def fit_sklearn(X, y): + d = X.shape[1] + kern = ConstantKernel(1.0, (1e-3, 1e3)) * Matern(np.full(d, 0.5), (1e-3, 1e3), nu=2.5) + gp = GaussianProcessRegressor(kern, alpha=1e-10, normalize_y=True, n_restarts_optimizer=5, + random_state=0).fit(X, y) + return lambda Z: gp.predict(Z, return_std=True) + PACKAGES["scikit-learn"] = fit_sklearn + +if smt is not None: + def fit_smt(X, y): + m = smt.KRG(corr="matern52", poly="constant", print_global=False) + m.set_training_values(X, y) + m.train() + return lambda Z: (np.ravel(m.predict_values(Z)), np.sqrt(np.maximum(np.ravel(m.predict_variances(Z)), 0))) + PACKAGES["SMT"] = fit_smt + +if ot is not None: + def fit_openturns(X, y): + d = X.shape[1] + cov = ot.MaternModel([0.5] * d, [float(np.std(y))], 2.5) + algo = ot.KrigingAlgorithm(ot.Sample(X), ot.Sample(y.reshape(-1, 1)), cov, ot.ConstantBasisFactory(d).build()) + algo.run() + r = algo.getResult() + return lambda Z: (np.ravel(r.getConditionalMean(ot.Sample(Z))), + np.sqrt(np.maximum(np.ravel(r.getConditionalMarginalVariance(ot.Sample(Z))), 0))) + PACKAGES["OpenTURNS"] = fit_openturns + +if GPy is not None: + def fit_gpy(X, y): + kern = GPy.kern.Matern52(X.shape[1], ARD=True) + m = GPy.models.GPRegression(X, y.reshape(-1, 1), kern, mean_function=None) + m.Gaussian_noise.variance.fix(1e-8) + m.optimize_restarts(3, verbose=False) + def p(Z): + mu, var = m.predict(Z) + return np.ravel(mu), np.sqrt(np.maximum(np.ravel(var) - 1e-8, 0)) + return p + PACKAGES["GPy"] = fit_gpy + +CASES = {"branin": 30, "six_hump_camel": 40, "hartmann3": 50, "ishigami": 100, "friedman": 80, + "otl_circuit": 80, "piston": 100, "borehole": 80, "wing_weight": 100, "hartmann6": 120} +rows = [] +for fname, n in CASES.items(): + tf = TESTS[fname] + for rep in range(3): + X, y, Xt, yt = design.dataset(tf, n, n_test=1000, seed=100 + rep) + for pkg, fit in PACKAGES.items(): + try: + t0 = time.time() + predictor = fit(X, y) + t_fit = time.time() - t0 + t0 = time.time() + m, s = predictor(Xt) + t_pred = time.time() - t0 + rows.append(dict(function=fname, n=n, rep=rep, package=pkg, fit_s=t_fit, predict_s=t_pred, + **metrics.all_metrics(yt, m, s))) + except Exception as exc: # noqa: BLE001 + rows.append(dict(function=fname, n=n, rep=rep, package=pkg, error=str(exc)[:100])) + if pkg.startswith("pylibkriging"): + suite.true(f"{fname}/rep{rep}/{pkg}: fit", False, detail=str(exc)[:200]) +res = pd.DataFrame(rows) +summary = res.groupby(["function", "package"], sort=False)[["q2", "nlpd", "coverage95", "fit_s"]].median() +for (fname, pkg), r in summary.iterrows(): + suite.metric(fname, pkg, **r.to_dict()) +summary.unstack("package")["q2"].round(4) + +# %% +summary.unstack("package")[["fit_s"]].round(3) + +# %% +for fname in CASES: + s = summary.loc[fname]["q2"] + others = s.drop(index=[p for p in s.index if p.startswith("pylibkriging")]) + if len(others): + suite.greater(f"{fname}: Q2(pylibkriging) - best other package", s["pylibkriging"] - others.max(), -0.02, + severity="warn", detail=f"best: {others.idxmax()}") + suite.greater(f"{fname}: Q2(pylibkriging BFGS10) - best other package", s["pylibkriging BFGS10"] - others.max(), + -0.05, detail=f"best: {others.idxmax()}") if len(others) else None + suite.greater(f"{fname}: Q2(pylibkriging BFGS10) > 0.5", s["pylibkriging BFGS10"], 0.5) + +# %% +suite.finalize() diff --git a/notebooks/python/src/11_cross_binding.py b/notebooks/python/src/11_cross_binding.py new file mode 100644 index 0000000..098a3ae --- /dev/null +++ b/notebooks/python/src/11_cross_binding.py @@ -0,0 +1,80 @@ +# %% [markdown] +# # 11 — Cross-binding consistency: the Python side +# +# All bindings (Python, R, Octave, Julia) wrap the same C++ library: on the +# same data with the same options they must return the same numbers. Each +# language runs the protocol below on the shared datasets of `data/` and writes +# `results/xbind_.csv` (`lang,dataset,quantity,index,value`); +# `tools/report.py` then compares every language to Python. +# +# Protocol, for each dataset (Branin n=25, Hartmann 3 n=40, Borehole n=80, on +# the unit cube), kernel `matern5_2`, trend `constant`, θ₀ = 0.3 in every +# dimension: +# +# | quantity | how | tolerance in the report | +# |---|---|---| +# | `fixed_sigma2`, `fixed_beta`, `fixed_ll`, `fixed_loo` | `optim="none"`, θ = θ₀ fixed | rel. 1e-8 (error) | +# | `fixed_mean`, `fixed_stdev` | predict at the first 50 test points | rel. 1e-8 (error) | +# | `llfun`, `llfun_grad` | `logLikelihoodFun(θ₀)` with gradient | rel. 1e-8 (error) | +# | `fit_theta`, `fit_ll`, `fit_mean` | default fit (`BFGS`) | rel. 1e-4 (warn: optimizer paths may differ) | +# | `sim` | `simulate(nsim=3, seed=123)` at the first 5 test points (5 × 3, column-major) | rel. 1e-8 (warn: RNG may be binding-specific) | + +# %% +import csv +from pathlib import Path + +import numpy as np +import pylibkriging as lk + +import lkbench +from lkbench import compat +from lkbench.checks import RESULTS_DIR + +suite = lkbench.Suite("py11_cross_binding") +print("pylibkriging", compat.version()) +DATA = Path("../../data") +DATASETS = ["branin", "hartmann3", "borehole"] +rows = [] + + +def put(dataset, quantity, values): + for i, v in enumerate(np.ravel(values), start=1): + rows.append(("python", dataset, quantity, i, f"{float(v):.17g}")) + + +def load(name, stem): + return np.loadtxt(DATA / f"{name}_{stem}.csv", delimiter=",", ndmin=2) + +# %% +for name in DATASETS: + X, y, Xt = load(name, "X"), load(name, "y").ravel(), load(name, "Xt")[:50] + d = X.shape[1] + theta0 = np.full((1, d), 0.3) + kf = lk.Kriging(y, X, "matern5_2", regmodel="constant", optim="none", + parameters={"theta": theta0, "is_theta_estim": False}) + put(name, "fixed_sigma2", kf.sigma2()) + put(name, "fixed_beta", kf.beta()) + put(name, "fixed_ll", kf.logLikelihood()) + put(name, "fixed_loo", kf.leaveOneOut()) + m, s, *_ = kf.predict(Xt, return_stdev=True) + put(name, "fixed_mean", m) + put(name, "fixed_stdev", s) + ll, grad, _ = kf.logLikelihoodFun(theta0.ravel(), True) + put(name, "llfun", ll) + put(name, "llfun_grad", grad) + suite.close(f"{name}: logLikelihood() == logLikelihoodFun(theta0)", kf.logLikelihood(), ll, rtol=1e-10) + k = lk.Kriging(y, X, "matern5_2") + put(name, "fit_theta", k.theta()) + put(name, "fit_ll", k.logLikelihood()) + put(name, "fit_mean", k.predict(Xt, return_stdev=False)[0]) + put(name, "sim", np.ravel(np.asarray(kf.simulate(3, 123, Xt[:5])), order="F")) # column-major, as R/Octave/Julia + +RESULTS_DIR.mkdir(exist_ok=True) +with open(RESULTS_DIR / "xbind_python.csv", "w", newline="") as fh: + w = csv.writer(fh) + w.writerow(["lang", "dataset", "quantity", "index", "value"]) + w.writerows(rows) +print(len(rows), "values written") + +# %% +suite.finalize() diff --git a/notebooks/r/01_cross_binding.ipynb b/notebooks/r/01_cross_binding.ipynb new file mode 100644 index 0000000..d3c97ed --- /dev/null +++ b/notebooks/r/01_cross_binding.ipynb @@ -0,0 +1,92 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "c000", + "metadata": {}, + "source": [ + "# R 01 — Cross-binding consistency: the R side\n", + "\n", + "Same protocol as `notebooks/python/11_cross_binding.ipynb` (see there), on the\n", + "shared datasets of `data/`; writes `results/xbind_R.csv`, compared with the\n", + "Python values by `tools/report.py`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c001", + "metadata": {}, + "outputs": [], + "source": [ + "suppressPackageStartupMessages(library(rlibkriging))\n", + "source(\"checks.R\")\n", + "s <- new_suite(\"r01_cross_binding\")\n", + "cat(\"rlibkriging\", as.character(packageVersion(\"rlibkriging\")), \"\\n\")\n", + "rows <- list()\n", + "put <- function(dataset, quantity, values) {\n", + " v <- as.numeric(values)\n", + " rows[[length(rows) + 1]] <<- data.frame(lang = \"R\", dataset = dataset, quantity = quantity,\n", + " index = seq_along(v), value = sprintf(\"%.17g\", v))\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c002", + "metadata": {}, + "outputs": [], + "source": [ + "for (name in c(\"branin\", \"hartmann3\", \"borehole\")) {\n", + " X <- read_data(name, \"X\"); y <- as.numeric(read_data(name, \"y\")); Xt <- read_data(name, \"Xt\")[1:50, , drop = FALSE]\n", + " d <- ncol(X)\n", + " theta0 <- matrix(0.3, nrow = 1, ncol = d)\n", + " kf <- Kriging(y, X, \"matern5_2\", regmodel = \"constant\", optim = \"none\",\n", + " parameters = list(theta = theta0, is_theta_estim = FALSE))\n", + " put(name, \"fixed_sigma2\", kf$sigma2())\n", + " put(name, \"fixed_beta\", kf$beta())\n", + " put(name, \"fixed_ll\", logLikelihood(kf))\n", + " put(name, \"fixed_loo\", leaveOneOut(kf))\n", + " p <- predict(kf, Xt, return_stdev = TRUE)\n", + " put(name, \"fixed_mean\", p$mean)\n", + " put(name, \"fixed_stdev\", p$stdev)\n", + " llf <- logLikelihoodFun(kf, theta0, return_grad = TRUE)\n", + " put(name, \"llfun\", llf$logLikelihood)\n", + " put(name, \"llfun_grad\", llf$logLikelihoodGrad)\n", + " check_close(s, paste0(name, \": logLikelihood() == logLikelihoodFun(theta0)\"), logLikelihood(kf), llf$logLikelihood, rtol = 1e-10)\n", + " k <- Kriging(y, X, \"matern5_2\")\n", + " put(name, \"fit_theta\", k$theta())\n", + " put(name, \"fit_ll\", logLikelihood(k))\n", + " put(name, \"fit_mean\", predict(k, Xt, return_stdev = FALSE)$mean)\n", + " put(name, \"sim\", simulate(kf, nsim = 3, seed = 123, x = Xt[1:5, , drop = FALSE]))\n", + "}\n", + "out <- do.call(rbind, rows)\n", + "write.csv(out, file.path(results_dir, \"xbind_R.csv\"), row.names = FALSE, quote = FALSE)\n", + "cat(nrow(out), \"values written\\n\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c003", + "metadata": {}, + "outputs": [], + "source": [ + "finalize(s)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "R", + "language": "R", + "name": "ir" + }, + "language_info": { + "name": "R" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/r/02_vs_dicekriging_robustgasp.ipynb b/notebooks/r/02_vs_dicekriging_robustgasp.ipynb new file mode 100644 index 0000000..2bcfec0 --- /dev/null +++ b/notebooks/r/02_vs_dicekriging_robustgasp.ipynb @@ -0,0 +1,164 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "c000", + "metadata": {}, + "source": [ + "# R 02 — rlibkriging vs DiceKriging and RobustGaSP\n", + "\n", + "DiceKriging is the reference implementation libKriging's `Kriging` follows\n", + "(same product-form kernels and range parametrization, same concentrated\n", + "likelihood). Hence two levels of comparison:\n", + "\n", + "1. **Same hyperparameters → same numbers** (error-level): the concentrated\n", + " log-likelihood at a given θ (`DiceKriging::logLikFun` vs\n", + " `logLikelihoodFun`), and the universal kriging mean / stdev at fixed\n", + " (θ, σ², β) (`predict(type = \"UK\")`).\n", + "2. **Default fits** on the shared datasets of `data/` (Branin, Hartmann 3,\n", + " Borehole), with RobustGaSP as a third package: Q², fit time\n", + " (warn-level comparisons)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c001", + "metadata": {}, + "outputs": [], + "source": [ + "suppressPackageStartupMessages({\n", + " library(rlibkriging)\n", + " library(DiceKriging)\n", + "})\n", + "has_rgasp <- requireNamespace(\"RobustGaSP\", quietly = TRUE)\n", + "source(\"checks.R\")\n", + "s <- new_suite(\"r02_vs_dicekriging_robustgasp\")\n", + "cat(\"rlibkriging\", as.character(packageVersion(\"rlibkriging\")), \"| DiceKriging\", as.character(packageVersion(\"DiceKriging\")),\n", + " \"| RobustGaSP\", if (has_rgasp) as.character(packageVersion(\"RobustGaSP\")) else \"absent\", \"\\n\")\n", + "DATASETS <- c(\"branin\", \"hartmann3\", \"borehole\")" + ] + }, + { + "cell_type": "markdown", + "id": "c002", + "metadata": {}, + "source": [ + "## 1. Same hyperparameters, same numbers" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c003", + "metadata": {}, + "outputs": [], + "source": [ + "for (name in DATASETS) {\n", + " X <- read_data(name, \"X\"); y <- as.numeric(read_data(name, \"y\")); Xt <- read_data(name, \"Xt\")\n", + " d <- ncol(X)\n", + " colnames(X) <- colnames(Xt) <- paste0(\"x\", 1:d)\n", + " for (kernel in c(\"gauss\", \"matern5_2\", \"matern3_2\", \"exp\")) {\n", + " theta <- seq(0.2, 0.6, length.out = d)\n", + " k <- Kriging(y, X, kernel, optim = \"none\", parameters = list(theta = matrix(theta, nrow = 1), is_theta_estim = FALSE))\n", + " km0 <- km(~1, design = data.frame(X), response = y, covtype = kernel, coef.cov = theta,\n", + " coef.var = k$sigma2(), coef.trend = as.numeric(k$beta()), control = list(trace = FALSE))\n", + " # concentrated log-likelihood at theta (DiceKriging profiles sigma2 and beta the same way)\n", + " km_fit <- km(~1, design = data.frame(X), response = y, covtype = kernel, control = list(trace = FALSE))\n", + " ll_dk <- DiceKriging::logLikFun(theta, km_fit)\n", + " ll_lk <- logLikelihoodFun(k, matrix(theta, nrow = 1))$logLikelihood\n", + " check_close(s, sprintf(\"%s/%s: logLikelihood == DiceKriging logLikFun\", name, kernel), ll_lk, ll_dk, rtol = 1e-8)\n", + " p_dk <- predict(km0, newdata = data.frame(Xt), type = \"UK\", checkNames = FALSE)\n", + " p_lk <- predict(k, Xt, return_stdev = TRUE)\n", + " check_close(s, sprintf(\"%s/%s: UK mean == DiceKriging\", name, kernel), p_lk$mean, p_dk$mean, rtol = 1e-6, atol = 1e-8 * sd(y))\n", + " check_close(s, sprintf(\"%s/%s: UK stdev == DiceKriging\", name, kernel), p_lk$stdev, p_dk$sd, rtol = 1e-5, atol = 1e-8 * sd(y))\n", + " }\n", + "}" + ] + }, + { + "cell_type": "markdown", + "id": "c004", + "metadata": {}, + "source": [ + "## 2. Default fits" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c005", + "metadata": {}, + "outputs": [], + "source": [ + "results <- list()\n", + "for (name in DATASETS) {\n", + " X <- read_data(name, \"X\"); y <- as.numeric(read_data(name, \"y\")); Xt <- read_data(name, \"Xt\"); yt <- as.numeric(read_data(name, \"yt\"))\n", + " colnames(X) <- colnames(Xt) <- paste0(\"x\", 1:ncol(X))\n", + " fits <- list(\n", + " rlibkriging = function() { k <- Kriging(y, X, \"matern5_2\"); function(Z) predict(k, Z)$mean },\n", + " `rlibkriging BFGS10` = function() { k <- Kriging(y, X, \"matern5_2\", optim = \"BFGS10\"); function(Z) predict(k, Z)$mean },\n", + " DiceKriging = function() {\n", + " m <- km(~1, design = data.frame(X), response = y, covtype = \"matern5_2\", control = list(trace = FALSE))\n", + " function(Z) predict(m, newdata = data.frame(Z), type = \"UK\", checkNames = FALSE)$mean\n", + " })\n", + " if (has_rgasp) fits$RobustGaSP <- function() {\n", + " m <- RobustGaSP::rgasp(design = X, response = y, kernel_type = \"matern_5_2\")\n", + " function(Z) RobustGaSP::predict(m, Z)$mean\n", + " }\n", + " for (pkg in names(fits)) {\n", + " t0 <- Sys.time()\n", + " r <- tryCatch(fits[[pkg]](), error = function(e) e)\n", + " t_fit <- as.numeric(difftime(Sys.time(), t0, units = \"secs\"))\n", + " if (inherits(r, \"error\")) {\n", + " if (startsWith(pkg, \"rlibkriging\")) add_check(s, paste0(name, \"/\", pkg, \": fit\"), FALSE, detail = conditionMessage(r))\n", + " next\n", + " }\n", + " m <- as.numeric(r(Xt))\n", + " add_metric(s, name, pkg, q2 = q2(yt, m), rmse = rmse(yt, m), fit_s = t_fit)\n", + " results[[length(results) + 1]] <- data.frame(dataset = name, package = pkg, q2 = q2(yt, m), fit_s = t_fit)\n", + " }\n", + "}\n", + "res <- do.call(rbind, results)\n", + "print(reshape(res[, c(\"dataset\", \"package\", \"q2\")], idvar = \"dataset\", timevar = \"package\", direction = \"wide\"))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c006", + "metadata": {}, + "outputs": [], + "source": [ + "for (name in DATASETS) {\n", + " r <- res[res$dataset == name, ]\n", + " best_other <- max(r$q2[!startsWith(r$package, \"rlibkriging\")])\n", + " check_greater(s, paste0(name, \": Q2(rlibkriging) - best other package\"), r$q2[r$package == \"rlibkriging\"] - best_other, -0.02, severity = \"warn\")\n", + " check_greater(s, paste0(name, \": Q2(rlibkriging BFGS10) > 0.9\"), r$q2[r$package == \"rlibkriging BFGS10\"], 0.9)\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c007", + "metadata": {}, + "outputs": [], + "source": [ + "finalize(s)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "R", + "language": "R", + "name": "ir" + }, + "language_info": { + "name": "R" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/r/checks.R b/notebooks/r/checks.R new file mode 100644 index 0000000..272b1ea --- /dev/null +++ b/notebooks/r/checks.R @@ -0,0 +1,71 @@ +# Same CSV outputs as lkbench/checks.py (see there), for the R notebooks. +# Statuses: PASS, FAIL (error-level), WARN (warn-level), SKIP, KNOWN (listed in known_issues.csv). +results_dir <- Sys.getenv("LKBENCH_RESULTS", file.path("..", "..", "results")) +dir.create(results_dir, showWarnings = FALSE, recursive = TRUE) +known_file <- file.path("..", "..", "known_issues.csv") +.known <- if (file.exists(known_file)) read.csv(known_file, stringsAsFactors = FALSE) else + data.frame(suite = character(), check = character(), issue = character()) + +new_suite <- function(name) { + s <- new.env() + s$name <- name + s$checks <- list() + s$metrics <- list() + s$t0 <- Sys.time() + s +} + +.fmt <- function(v) if (is.null(v) || length(v) == 0) "" else if (is.numeric(v)) format(signif(v[1], 6)) else as.character(v[1]) + +add_check <- function(s, check, ok, value = NULL, expected = NULL, detail = "", severity = "error") { + ok <- isTRUE(ok) + status <- if (ok) "PASS" else if (severity == "error") "FAIL" else "WARN" + k <- .known[.known$suite == s$name & .known$check == check, ] + if (!ok && nrow(k) > 0) { status <- "KNOWN"; detail <- paste(k$issue[1], detail, sep = " | ") } + if (!ok) message(sprintf("[%s] %s: value=%s expected %s %s", status, check, .fmt(value), .fmt(expected), detail)) + s$checks[[length(s$checks) + 1]] <- data.frame(suite = s$name, lang = "R", check = check, severity = severity, + status = status, value = .fmt(value), expected = .fmt(expected), detail = detail, stringsAsFactors = FALSE) + invisible(ok) +} + +check_close <- function(s, check, a, b, rtol = 1e-6, atol = 0, detail = "", severity = "error") { + a <- as.numeric(a); b <- as.numeric(b) + if (length(a) != length(b)) return(add_check(s, check, FALSE, paste("length", length(a)), paste("length", length(b)), detail, severity)) + gap <- abs(a - b) + ok <- all(is.finite(a)) && all(gap <= atol + rtol * abs(b)) + rel <- max(gap / pmax(abs(b), 1e-300)) + add_check(s, check, ok, rel, sprintf("rtol %g, atol %g", rtol, atol), detail, severity) +} + +check_less <- function(s, check, value, bound, detail = "", severity = "error") + add_check(s, check, is.finite(value) && value < bound, value, paste("<", bound), detail, severity) + +check_greater <- function(s, check, value, bound, detail = "", severity = "error") + add_check(s, check, is.finite(value) && value > bound, value, paste(">", bound), detail, severity) + +check_runs <- function(s, check, expr, severity = "error") { + r <- tryCatch({ force(expr); NULL }, error = function(e) conditionMessage(e)) + add_check(s, check, is.null(r), if (is.null(r)) "ok" else "error", "no error", if (is.null(r)) "" else substr(r, 1, 200), severity) +} + +add_metric <- function(s, case, package, ...) { + v <- list(...) + for (n in names(v)) s$metrics[[length(s$metrics) + 1]] <- data.frame(suite = s$name, lang = "R", case = case, + package = package, metric = n, value = .fmt(v[[n]]), stringsAsFactors = FALSE) +} + +finalize <- function(s) { + add_metric(s, "_suite", "_", elapsed_s = as.numeric(difftime(Sys.time(), s$t0, units = "secs"))) + ch <- do.call(rbind, s$checks) + me <- do.call(rbind, s$metrics) + write.csv(ch, file.path(results_dir, paste0(s$name, ".checks.csv")), row.names = FALSE) + write.csv(me, file.path(results_dir, paste0(s$name, ".metrics.csv")), row.names = FALSE) + print(table(factor(ch$status, levels = c("PASS", "FAIL", "WARN", "KNOWN", "SKIP")))) + failed <- ch$check[ch$status == "FAIL"] + if (length(failed) > 0) stop(sprintf("%d check(s) failed in %s: %s", length(failed), s$name, paste(head(failed, 20), collapse = "; "))) + invisible(ch) +} + +rmse <- function(y, m) sqrt(mean((y - m)^2)) +q2 <- function(y, m) 1 - sum((y - m)^2) / sum((y - mean(y))^2) +read_data <- function(name, stem) as.matrix(read.csv(file.path("..", "..", "data", paste0(name, "_", stem, ".csv")), header = FALSE)) diff --git a/notebooks/r/src/01_cross_binding.R b/notebooks/r/src/01_cross_binding.R new file mode 100644 index 0000000..2950207 --- /dev/null +++ b/notebooks/r/src/01_cross_binding.R @@ -0,0 +1,49 @@ +# %% [markdown] +# # R 01 — Cross-binding consistency: the R side +# +# Same protocol as `notebooks/python/11_cross_binding.ipynb` (see there), on the +# shared datasets of `data/`; writes `results/xbind_R.csv`, compared with the +# Python values by `tools/report.py`. + +# %% +suppressPackageStartupMessages(library(rlibkriging)) +source("checks.R") +s <- new_suite("r01_cross_binding") +cat("rlibkriging", as.character(packageVersion("rlibkriging")), "\n") +rows <- list() +put <- function(dataset, quantity, values) { + v <- as.numeric(values) + rows[[length(rows) + 1]] <<- data.frame(lang = "R", dataset = dataset, quantity = quantity, + index = seq_along(v), value = sprintf("%.17g", v)) +} + +# %% +for (name in c("branin", "hartmann3", "borehole")) { + X <- read_data(name, "X"); y <- as.numeric(read_data(name, "y")); Xt <- read_data(name, "Xt")[1:50, , drop = FALSE] + d <- ncol(X) + theta0 <- matrix(0.3, nrow = 1, ncol = d) + kf <- Kriging(y, X, "matern5_2", regmodel = "constant", optim = "none", + parameters = list(theta = theta0, is_theta_estim = FALSE)) + put(name, "fixed_sigma2", kf$sigma2()) + put(name, "fixed_beta", kf$beta()) + put(name, "fixed_ll", logLikelihood(kf)) + put(name, "fixed_loo", leaveOneOut(kf)) + p <- predict(kf, Xt, return_stdev = TRUE) + put(name, "fixed_mean", p$mean) + put(name, "fixed_stdev", p$stdev) + llf <- logLikelihoodFun(kf, theta0, return_grad = TRUE) + put(name, "llfun", llf$logLikelihood) + put(name, "llfun_grad", llf$logLikelihoodGrad) + check_close(s, paste0(name, ": logLikelihood() == logLikelihoodFun(theta0)"), logLikelihood(kf), llf$logLikelihood, rtol = 1e-10) + k <- Kriging(y, X, "matern5_2") + put(name, "fit_theta", k$theta()) + put(name, "fit_ll", logLikelihood(k)) + put(name, "fit_mean", predict(k, Xt, return_stdev = FALSE)$mean) + put(name, "sim", simulate(kf, nsim = 3, seed = 123, x = Xt[1:5, , drop = FALSE])) +} +out <- do.call(rbind, rows) +write.csv(out, file.path(results_dir, "xbind_R.csv"), row.names = FALSE, quote = FALSE) +cat(nrow(out), "values written\n") + +# %% +finalize(s) diff --git a/notebooks/r/src/02_vs_dicekriging_robustgasp.R b/notebooks/r/src/02_vs_dicekriging_robustgasp.R new file mode 100644 index 0000000..bd20b33 --- /dev/null +++ b/notebooks/r/src/02_vs_dicekriging_robustgasp.R @@ -0,0 +1,97 @@ +# %% [markdown] +# # R 02 — rlibkriging vs DiceKriging and RobustGaSP +# +# DiceKriging is the reference implementation libKriging's `Kriging` follows +# (same product-form kernels and range parametrization, same concentrated +# likelihood). Hence two levels of comparison: +# +# 1. **Same hyperparameters → same numbers** (error-level): the concentrated +# log-likelihood at a given θ (`DiceKriging::logLikFun` vs +# `logLikelihoodFun`), and the universal kriging mean / stdev at fixed +# (θ, σ², β) (`predict(type = "UK")`). +# 2. **Default fits** on the shared datasets of `data/` (Branin, Hartmann 3, +# Borehole), with RobustGaSP as a third package: Q², fit time +# (warn-level comparisons). + +# %% +suppressPackageStartupMessages({ + library(rlibkriging) + library(DiceKriging) +}) +has_rgasp <- requireNamespace("RobustGaSP", quietly = TRUE) +source("checks.R") +s <- new_suite("r02_vs_dicekriging_robustgasp") +cat("rlibkriging", as.character(packageVersion("rlibkriging")), "| DiceKriging", as.character(packageVersion("DiceKriging")), + "| RobustGaSP", if (has_rgasp) as.character(packageVersion("RobustGaSP")) else "absent", "\n") +DATASETS <- c("branin", "hartmann3", "borehole") + +# %% [markdown] +# ## 1. Same hyperparameters, same numbers + +# %% +for (name in DATASETS) { + X <- read_data(name, "X"); y <- as.numeric(read_data(name, "y")); Xt <- read_data(name, "Xt") + d <- ncol(X) + colnames(X) <- colnames(Xt) <- paste0("x", 1:d) + for (kernel in c("gauss", "matern5_2", "matern3_2", "exp")) { + theta <- seq(0.2, 0.6, length.out = d) + k <- Kriging(y, X, kernel, optim = "none", parameters = list(theta = matrix(theta, nrow = 1), is_theta_estim = FALSE)) + km0 <- km(~1, design = data.frame(X), response = y, covtype = kernel, coef.cov = theta, + coef.var = k$sigma2(), coef.trend = as.numeric(k$beta()), control = list(trace = FALSE)) + # concentrated log-likelihood at theta (DiceKriging profiles sigma2 and beta the same way) + km_fit <- km(~1, design = data.frame(X), response = y, covtype = kernel, control = list(trace = FALSE)) + ll_dk <- DiceKriging::logLikFun(theta, km_fit) + ll_lk <- logLikelihoodFun(k, matrix(theta, nrow = 1))$logLikelihood + check_close(s, sprintf("%s/%s: logLikelihood == DiceKriging logLikFun", name, kernel), ll_lk, ll_dk, rtol = 1e-8) + p_dk <- predict(km0, newdata = data.frame(Xt), type = "UK", checkNames = FALSE) + p_lk <- predict(k, Xt, return_stdev = TRUE) + check_close(s, sprintf("%s/%s: UK mean == DiceKriging", name, kernel), p_lk$mean, p_dk$mean, rtol = 1e-6, atol = 1e-8 * sd(y)) + check_close(s, sprintf("%s/%s: UK stdev == DiceKriging", name, kernel), p_lk$stdev, p_dk$sd, rtol = 1e-5, atol = 1e-8 * sd(y)) + } +} + +# %% [markdown] +# ## 2. Default fits + +# %% +results <- list() +for (name in DATASETS) { + X <- read_data(name, "X"); y <- as.numeric(read_data(name, "y")); Xt <- read_data(name, "Xt"); yt <- as.numeric(read_data(name, "yt")) + colnames(X) <- colnames(Xt) <- paste0("x", 1:ncol(X)) + fits <- list( + rlibkriging = function() { k <- Kriging(y, X, "matern5_2"); function(Z) predict(k, Z)$mean }, + `rlibkriging BFGS10` = function() { k <- Kriging(y, X, "matern5_2", optim = "BFGS10"); function(Z) predict(k, Z)$mean }, + DiceKriging = function() { + m <- km(~1, design = data.frame(X), response = y, covtype = "matern5_2", control = list(trace = FALSE)) + function(Z) predict(m, newdata = data.frame(Z), type = "UK", checkNames = FALSE)$mean + }) + if (has_rgasp) fits$RobustGaSP <- function() { + m <- RobustGaSP::rgasp(design = X, response = y, kernel_type = "matern_5_2") + function(Z) RobustGaSP::predict(m, Z)$mean + } + for (pkg in names(fits)) { + t0 <- Sys.time() + r <- tryCatch(fits[[pkg]](), error = function(e) e) + t_fit <- as.numeric(difftime(Sys.time(), t0, units = "secs")) + if (inherits(r, "error")) { + if (startsWith(pkg, "rlibkriging")) add_check(s, paste0(name, "/", pkg, ": fit"), FALSE, detail = conditionMessage(r)) + next + } + m <- as.numeric(r(Xt)) + add_metric(s, name, pkg, q2 = q2(yt, m), rmse = rmse(yt, m), fit_s = t_fit) + results[[length(results) + 1]] <- data.frame(dataset = name, package = pkg, q2 = q2(yt, m), fit_s = t_fit) + } +} +res <- do.call(rbind, results) +print(reshape(res[, c("dataset", "package", "q2")], idvar = "dataset", timevar = "package", direction = "wide")) + +# %% +for (name in DATASETS) { + r <- res[res$dataset == name, ] + best_other <- max(r$q2[!startsWith(r$package, "rlibkriging")]) + check_greater(s, paste0(name, ": Q2(rlibkriging) - best other package"), r$q2[r$package == "rlibkriging"] - best_other, -0.02, severity = "warn") + check_greater(s, paste0(name, ": Q2(rlibkriging BFGS10) > 0.9"), r$q2[r$package == "rlibkriging BFGS10"], 0.9) +} + +# %% +finalize(s) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..4e92615 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,17 @@ +[build-system] +requires = ["setuptools>=64"] +build-backend = "setuptools.build_meta" + +[project] +name = "lkbench" +version = "0.1.0" +description = "Helpers (test functions, designs, metrics, checks) for the libKriging benchmark notebooks" +requires-python = ">=3.9" +dependencies = ["numpy", "scipy", "pandas", "matplotlib"] + +[project.optional-dependencies] +notebooks = ["nbclient", "nbformat", "ipykernel"] +compare = ["scikit-learn", "smt", "openturns"] + +[tool.setuptools] +packages = ["lkbench"] diff --git a/tools/make_data.py b/tools/make_data.py new file mode 100644 index 0000000..62294c5 --- /dev/null +++ b/tools/make_data.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python3 +"""Write the shared datasets of data/ (committed): the cross-binding notebooks of every +language read these exact files, so that all bindings fit the very same points. + +Plain CSV, no header, full precision; X is n x d (rows = observations), y is a column. +Regenerating them is only needed when a dataset is added: the files are the reference, +not the script (a new scipy could draw other LHS points).""" +from pathlib import Path + +import numpy as np + +from lkbench import TESTS, design + +DATA = Path(__file__).resolve().parents[1] / "data" +DATASETS = [("branin", 25), ("hartmann3", 40), ("borehole", 80)] + + +def main(): + DATA.mkdir(exist_ok=True) + for name, n in DATASETS: + X, y, Xt, yt = design.dataset(TESTS[name], n, n_test=200, seed=2024) + for stem, arr in [("X", X), ("y", y), ("Xt", Xt), ("yt", yt)]: + np.savetxt(DATA / f"{name}_{stem}.csv", arr.reshape(len(arr), -1), delimiter=",", fmt="%.17g") + print(f"data/{name}_*.csv: n={n}, d={X.shape[1]}, n_test={len(Xt)}") + + +if __name__ == "__main__": + main() diff --git a/tools/py2nb.py b/tools/py2nb.py new file mode 100644 index 0000000..06851b2 --- /dev/null +++ b/tools/py2nb.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +"""Convert a "percent" script (cells separated by ``# %%`` / ``# %% [markdown]``) into a +notebook without outputs. The notebooks of this repository are written this way +(sources in ``notebooks/**/src/``) so that they review as plain diffs; the +``.ipynb`` files are generated, and CI checks that they are in sync. + +Usage: python tools/py2nb.py SRC... [--check] + each ``notebooks//src/.`` gives ``notebooks//.ipynb``. +""" +import argparse +import re +import sys +from pathlib import Path + +import nbformat + +KERNELS = { + ".py": ({"name": "python3", "display_name": "Python 3", "language": "python"}, + {"name": "python"}, "#"), + ".R": ({"name": "ir", "display_name": "R", "language": "R"}, + {"name": "R"}, "#"), + ".jl": ({"name": "julia-1.11", "display_name": "Julia 1.11", "language": "julia"}, + {"name": "julia"}, "#"), + ".m": ({"name": "octave", "display_name": "Octave", "language": "octave"}, + {"name": "octave"}, "%"), +} + + +def convert(src): + kernelspec, lang_info, comment = KERNELS[src.suffix] + marker = re.compile(rf"^{re.escape(comment)} %%( \[markdown\])?\s*$") + cells, kind, buf = [], None, [] + + def flush(): + text = "\n".join(buf).strip("\n") + if kind == "markdown": + text = "\n".join(re.sub(rf"^{re.escape(comment)} ?", "", ln) for ln in text.splitlines()) + cells.append(nbformat.v4.new_markdown_cell(text)) + elif kind == "code" and text: + cells.append(nbformat.v4.new_code_cell(text)) + + for line in src.read_text().splitlines(): + m = marker.match(line) + if m: + flush() + kind, buf = ("markdown" if m.group(1) else "code"), [] + elif kind is not None: + buf.append(line) + flush() + nb = nbformat.v4.new_notebook(cells=cells) + nb.metadata["kernelspec"] = kernelspec + nb.metadata["language_info"] = lang_info + for i, c in enumerate(nb.cells): # stable ids: no spurious diffs on regeneration + c["id"] = f"c{i:03d}" + return nb + + +def main(argv=None): + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("sources", nargs="+", type=Path) + p.add_argument("--check", action="store_true", help="fail if a notebook differs from its source") + args = p.parse_args(argv) + stale = [] + for src in args.sources: + out = src.parent.parent / (src.stem + ".ipynb") + text = nbformat.writes(convert(src)) + "\n" + if args.check: + if not out.exists() or out.read_text() != text: + stale.append(out) + else: + out.write_text(text) + if stale: + print("out of date (run tools/py2nb.py):", *map(str, stale), sep="\n ") + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/report.py b/tools/report.py new file mode 100644 index 0000000..19c46c3 --- /dev/null +++ b/tools/report.py @@ -0,0 +1,204 @@ +#!/usr/bin/env python3 +"""Aggregate the outputs of every notebook (all languages) into one Markdown report. + +Reads, from a results directory (default: results/; in CI, the merged artifacts +of every job): + + *.checks.csv suite,lang,check,severity,status,value,expected,detail + *.metrics.csv suite,lang,case,package,metric,value + xbind_*.csv lang,dataset,quantity,index,value (cross-binding protocol) + +and reports: + + 1. a status table per notebook (PASS / FAIL / WARN / KNOWN / SKIP); + 2. every FAIL, WARN and KNOWN check, with its value and threshold; + 3. known issues whose check now passes ("FIXED?": remove them from known_issues.csv); + 4. cross-binding consistency: each language against Python, per quantity, with + the tolerances of notebooks/python/11_cross_binding.ipynb; + 5. drifts of the accuracy metrics (q2, rmse, nlpd) and of the fit times against a + baseline (baseline/metrics.csv), when one is given. + +Exit status 1 if any check FAILed, if a cross-binding quantity of error level +differs, or if --expect lists a suite that produced no output (crashed before +finalize). The Markdown goes to stdout and, with --summary, is appended to a file +(e.g. $GITHUB_STEP_SUMMARY). + +Usage: + python tools/report.py [--results results] [--baseline baseline/metrics.csv] + [--write-baseline baseline/metrics.csv] [--summary FILE] + [--expect py00_test_functions,...] +""" +import argparse +import csv +import sys +from pathlib import Path + +import numpy as np +import pandas as pd + +ROOT = Path(__file__).resolve().parents[1] +STATUSES = ["PASS", "FAIL", "WARN", "KNOWN", "SKIP"] +XBIND_TOL = { # quantity: (rtol, atol relative to the scale of the quantity, severity) + "fixed_sigma2": (1e-8, 0, "error"), "fixed_beta": (1e-8, 1e-10, "error"), "fixed_ll": (1e-8, 0, "error"), + "fixed_loo": (1e-8, 0, "error"), "fixed_mean": (1e-8, 1e-10, "error"), "fixed_stdev": (1e-6, 1e-8, "error"), + "llfun": (1e-8, 0, "error"), "llfun_grad": (1e-6, 1e-8, "error"), + "fit_theta": (1e-4, 1e-6, "warn"), "fit_ll": (1e-4, 0, "warn"), "fit_mean": (1e-4, 1e-6, "warn"), + "sim": (1e-8, 1e-10, "warn"), +} +TRACKED = {"q2": "higher", "rmse": "lower", "nlpd": "lower", "fit_s": "time"} + + +def read_all(results, pattern): + frames = [] + for p in sorted(Path(results).rglob(pattern)): + try: + frames.append(pd.read_csv(p, dtype=str, keep_default_na=False)) + except Exception as exc: # noqa: BLE001 + print(f"warning: cannot read {p}: {exc}", file=sys.stderr) + return pd.concat(frames, ignore_index=True) if frames else pd.DataFrame() + + +def md_table(df, max_rows=200): + if df.empty: + return "_none_\n" + df = df.head(max_rows) + cols = list(df.columns) + out = ["| " + " | ".join(cols) + " |", "|" + "---|" * len(cols)] + for _, r in df.iterrows(): + out.append("| " + " | ".join(str(r[c]).replace("|", "\\|").replace("\n", " ") for c in cols) + " |") + return "\n".join(out) + "\n" + + +def xbind_report(results): + df = read_all(results, "xbind_*.csv") + if df.empty: + return "_no cross-binding output_\n", 0 + df["value"] = pd.to_numeric(df["value"], errors="coerce") + ref = df[df["lang"] == "python"].set_index(["dataset", "quantity", "index"])["value"] + if ref.empty: + return "_no Python reference (xbind_python.csv) to compare with_\n", 0 + rows, n_err = [], 0 + for lang, sub in df[df["lang"] != "python"].groupby("lang"): + sub = sub.set_index(["dataset", "quantity", "index"])["value"] + for (dataset, quantity), grp in sub.groupby(level=[0, 1]): + rtol, atol, sev = XBIND_TOL.get(quantity, (1e-6, 0, "warn")) + try: + r = ref.loc[grp.index] + except KeyError: + rows.append((lang, dataset, quantity, "missing in Python", "", sev, "FAIL" if sev == "error" else "WARN")) + continue + a, b = grp.to_numpy(float), r.to_numpy(float) + scale = max(np.max(np.abs(b)), 1e-300) + gap = np.abs(a - b) + ok = bool(np.all(np.isfinite(a)) and np.all(gap <= rtol * np.abs(b) + atol * scale)) + rel = float(np.max(gap / np.maximum(np.abs(b), 1e-300))) + status = "PASS" if ok else ("FAIL" if sev == "error" else "WARN") + n_err += status == "FAIL" + rows.append((lang, dataset, quantity, f"{rel:.2e}", f"rtol {rtol:g}", sev, status)) + missing = set(ref.index.droplevel(2).unique()) - set(sub.index.droplevel(2).unique()) + for dataset, quantity in sorted(missing): + rows.append((lang, dataset, quantity, "missing", "", "warn", "WARN")) + tab = pd.DataFrame(rows, columns=["lang", "dataset", "quantity", "max rel. gap", "tolerance", "severity", "status"]) + summary = tab.groupby(["lang", "status"]).size().unstack(fill_value=0).reset_index() + text = md_table(summary) + "\n" + md_table(tab[tab["status"] != "PASS"]) + return text, n_err + + +def drift_report(metrics, baseline): + base = pd.read_csv(baseline, dtype=str, keep_default_na=False) + key = ["suite", "lang", "case", "package", "metric"] + cur = metrics[metrics["metric"].isin(TRACKED)].copy() + m = cur.merge(base, on=key, suffixes=("", "_base")) + m["value"] = pd.to_numeric(m["value"], errors="coerce") + m["value_base"] = pd.to_numeric(m["value_base"], errors="coerce") + rows = [] + for _, r in m.iterrows(): + kind, v, b = TRACKED[r["metric"]], r["value"], r["value_base"] + if not np.isfinite(b): + continue + if not np.isfinite(v): + bad = True + elif kind == "higher": + bad = v < b - 0.02 + elif kind == "lower": + bad = v > b + 0.1 * abs(b) + 1e-12 + else: # time: CI runners are noisy, only a 3x slowdown on a fit above 0.5 s is reported + bad = v > 3 * b and v > 0.5 + if bad: + rows.append(dict(suite=r["suite"], lang=r["lang"], case=r["case"], package=r["package"], + metric=r["metric"], baseline=f"{b:.4g}", now=f"{v:.4g}")) + return pd.DataFrame(rows) + + +def main(argv=None): + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("--results", default=str(ROOT / "results")) + p.add_argument("--baseline", help="metrics CSV of a reference run (drift report)") + p.add_argument("--write-baseline", help="write the current metrics as a new baseline") + p.add_argument("--summary", help="also append the report to this file") + p.add_argument("--expect", default="", help="comma-separated suites that must have produced output") + args = p.parse_args(argv) + + checks = read_all(args.results, "*.checks.csv") + metrics = read_all(args.results, "*.metrics.csv") + lines = ["# libKriging benchmark report\n"] + exit_code = 0 + + # 1. status per notebook + if checks.empty: + lines.append("_no check output found_\n") + exit_code = 1 + else: + tab = pd.crosstab([checks["suite"], checks["lang"]], checks["status"]) + tab = tab.reindex(columns=STATUSES, fill_value=0).reset_index() + tab.columns.name = None + tab.insert(2, "result", np.where(tab["FAIL"] > 0, "❌", np.where(tab["WARN"] + tab["KNOWN"] > 0, "⚠️", "✅"))) + lines += ["## Notebooks\n", md_table(tab)] + exit_code |= int((checks["status"] == "FAIL").any()) + + expected = [s for s in args.expect.split(",") if s] + missing = sorted(set(expected) - set(checks.get("suite", pd.Series(dtype=str)))) + if missing: + lines += ["## Notebooks without output (crashed before `finalize`)\n", md_table(pd.DataFrame({"suite": missing}))] + exit_code = 1 + + # 2. problems + if not checks.empty: + cols = ["suite", "lang", "check", "value", "expected", "detail"] + for status, title in [("FAIL", "Failures"), ("WARN", "Warnings"), ("KNOWN", "Known issues (still present)")]: + sub = checks[checks["status"] == status][cols] + lines += [f"## {title} ({len(sub)})\n", md_table(sub)] + + # 3. known issues that now pass + known_file = ROOT / "known_issues.csv" + if known_file.exists(): + known = pd.read_csv(known_file, dtype=str, keep_default_na=False) + passing = known.merge(checks[checks["status"] == "PASS"][["suite", "check"]], on=["suite", "check"]) + if not passing.empty: + lines += ["## Known issues that now pass — FIXED? (remove them from known_issues.csv)\n", + md_table(passing)] + + # 4. cross-binding + text, n_err = xbind_report(args.results) + lines += ["## Cross-binding consistency (vs Python)\n", text] + exit_code |= int(n_err > 0) + + # 5. drifts + if args.baseline and Path(args.baseline).exists() and not metrics.empty: + drift = drift_report(metrics, args.baseline) + lines += [f"## Metric drifts vs baseline ({len(drift)})\n", md_table(drift)] + if args.write_baseline and not metrics.empty: + keep = metrics[metrics["metric"].isin(TRACKED)] + Path(args.write_baseline).parent.mkdir(parents=True, exist_ok=True) + keep.to_csv(args.write_baseline, index=False, quoting=csv.QUOTE_MINIMAL) + + report = "\n".join(lines) + print(report) + if args.summary: + with open(args.summary, "a") as fh: + fh.write(report + "\n") + return exit_code + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/run_notebooks.py b/tools/run_notebooks.py new file mode 100644 index 0000000..bc1e8ac --- /dev/null +++ b/tools/run_notebooks.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python3 +"""Execute Jupyter notebooks and fail if any cell raises. + +Used by CI (.github/workflows/notebooks.yml) to keep the worked notebooks +runnable, and to refresh their stored outputs (`--inplace`). + +The binding notebooks start with an "Installation" section of `%%bash` cells that +create a virtual environment, install requirements and build libKriging. Those +cells provision an environment and are not what the notebooks demonstrate: with +`--skip-setup` no `%%bash` cell (nor the cell that puts a built mlibkriging on the +Octave path) is run: they keep their stored content, no local path ends up in the +outputs, and the notebook is executed against the libKriging already available to +the kernel (Python: installed; Julia: `--project`; Octave: `OCTAVE_PATH`). + +Usage: + python3 tools/notebooks/run_notebooks.py --skip-setup bindings/Python/*.ipynb + python3 tools/notebooks/run_notebooks.py --skip-setup --inplace --kernel python3 nb.ipynb + python3 tools/notebooks/run_notebooks.py --skip-cell '^repo_root = ' --prelude 'addpath(...)' nb.ipynb + +Requires nbclient, nbformat and ipykernel. +""" +import argparse +import copy +import re +import sys +import time +from pathlib import Path + +import nbformat +from nbclient import NotebookClient +from nbclient.exceptions import CellExecutionError + + +# Python notebooks: the installation cells are `%%bash` cells. Octave notebooks: one cell puts +# `build/installed` on the path, and errors out when libKriging is not there. +SETUP_PREFIXES = ("%%bash", "% Add mlibkriging to path") + + +def is_setup_cell(cell): + return cell.cell_type == "code" and cell.source.lstrip().startswith(SETUP_PREFIXES) + + +def output_errors(nb): + """Octave's kernel reports a failing statement as plain text ("error: ..."), not as an error output.""" + if nb.metadata.get("kernelspec", {}).get("language") != "octave": + return [] + found = [] + for cell in nb.cells: + if cell.cell_type != "code": + continue + for out in cell.get("outputs", []): + for line in out.get("text", "").splitlines(): + if line.startswith("error: "): + found.append(line.strip()) + return found + + +def run(path, kernel, timeout, skip_setup, inplace, skip_patterns=(), prelude=None): + nb = nbformat.read(path, as_version=4) + work = copy.deepcopy(nb) + skipped = {i for i, cell in enumerate(nb.cells) + if cell.cell_type == "code" and (skip_setup and is_setup_cell(cell) + or any(re.search(p, cell.source) for p in skip_patterns))} + kept = [i for i in range(len(nb.cells)) if i not in skipped] + work.cells = [work.cells[i] for i in kept] # the skipped cells are not sent to the kernel + if prelude: # runs first, is never stored + work.cells.insert(0, nbformat.v4.new_code_cell(prelude)) + + client = NotebookClient( + work, + kernel_name=kernel or nb.metadata.get("kernelspec", {}).get("name", "python3"), + timeout=timeout, + resources={"metadata": {"path": str(path.parent)}}, + record_timing=False, + ) + start = time.time() + try: + client.execute() + except CellExecutionError as exc: + return False, f"{exc}".strip()[-1500:], time.time() - start + + errors = output_errors(work) + if errors: + return False, "\n".join(list(dict.fromkeys(errors))[:10]), time.time() - start + + if inplace: + # Merge the outputs back, leaving the skipped setup cells as they were. + for i, done in zip(kept, work.cells[1 if prelude else 0:]): + if done.cell_type == "code": + nb.cells[i].outputs = done.outputs + nb.cells[i].execution_count = done.execution_count + nbformat.write(nb, path) + return True, f"{len(skipped)} setup cell(s) skipped" if skipped else "", time.time() - start + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("notebooks", nargs="+", type=Path) + parser.add_argument("--kernel", help="kernel name to use instead of the notebook's own kernelspec") + parser.add_argument("--timeout", type=int, default=1800, help="per-cell timeout in seconds (default 1800)") + parser.add_argument("--skip-setup", action="store_true", help="do not run the %%bash installation cells") + parser.add_argument("--inplace", action="store_true", help="write the outputs back into the notebooks") + parser.add_argument("--skip-cell", action="append", default=[], metavar="REGEX", + help="do not run the code cells whose source matches REGEX (repeatable)") + parser.add_argument("--prelude", help="code run in the kernel before the first cell, and not stored") + args = parser.parse_args(argv) + + failures = [] + for path in args.notebooks: + ok, info, seconds = run(path, args.kernel, args.timeout, args.skip_setup, args.inplace, + args.skip_cell, args.prelude) + print(f"{'ok ' if ok else 'FAIL'} {path} ({seconds:.0f}s){': ' + info if info else ''}", flush=True) + if not ok: + failures.append(path) + if failures: + print(f"\n{len(failures)} notebook(s) failed:", *map(str, failures), sep="\n ") + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From 2b38d1f5e4d49097a3308df176c8ec9c3c7c5450 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 14 Oct 2026 07:10:52 +0000 Subject: [PATCH 02/16] CI: archive the notebooks executed against libKriging master, link them from the README - report job: new `executed-notebooks-` artifact (executed notebooks only, 90 days), next to `report-`. - tools/run_notebooks.py: with --inplace, a notebook that fails is saved too, with the outputs of the cells run up to the failing one (timeouts and dead kernels are reported instead of aborting the loop); docstring updated for this repository. - README: "Executed notebooks (libKriging master)" section, with direct links to the latest artifacts of main (nightly.link) and to the runs on main. - workflow: read-only token (`permissions: contents: read`); the `channels` input reaches the shell through an environment variable, not by interpolation. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01QiFEE514dpwS57mcFWDeZG --- .github/workflows/notebooks.yml | 17 ++++++++++++- README.md | 28 +++++++++++++++++--- tools/run_notebooks.py | 45 ++++++++++++++++++--------------- 3 files changed, 65 insertions(+), 25 deletions(-) diff --git a/.github/workflows/notebooks.yml b/.github/workflows/notebooks.yml index dcbe029..7252f12 100644 --- a/.github/workflows/notebooks.yml +++ b/.github/workflows/notebooks.yml @@ -16,6 +16,11 @@ name: notebooks # A notebook fails (red job) when one of its error-level checks fails; the report # job fails when any notebook failed or crashed, or when the bindings disagree # (cross-binding protocol, notebooks/python/11_cross_binding.ipynb). +# +# The notebooks executed against libKriging master are saved with their outputs +# (failures included: a notebook whose checks fail is kept up to its failing cell) +# and archived as the `executed-notebooks-master` artifact of every run, linked +# from the README. on: push: @@ -39,6 +44,9 @@ on: # POST /repos//dispatches {"event_type": "libkriging-updated", "client_payload": {"ref": ""}} types: [libkriging-updated] +permissions: + contents: read + concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true @@ -73,11 +81,12 @@ jobs: id: refs env: GH_TOKEN: ${{ github.token }} + CHANNELS: ${{ inputs.channels || 'master,release' }} run: | sha=$(git ls-remote "https://github.com/${LIBKRIGING_REPO}" "${LIBKRIGING_REF}" | head -1 | cut -f1) [ -n "$sha" ] || sha="${LIBKRIGING_REF}" # already a commit tag=$(gh api "repos/${LIBKRIGING_REPO}/releases/latest" --jq .tag_name) - channels='${{ inputs.channels || 'master,release' }}' + channels="$CHANNELS" echo "sha=$sha" >> "$GITHUB_OUTPUT" echo "release_tag=$tag" >> "$GITHUB_OUTPUT" echo "channels=$(python -c "import json,sys; print(json.dumps(sys.argv[1].split(',')))" "$channels")" >> "$GITHUB_OUTPUT" @@ -377,3 +386,9 @@ jobs: results/ executed/ new-baseline/ + - if: always() && contains(needs.setup.outputs.channels, matrix.channel) + uses: actions/upload-artifact@v4 + with: + name: executed-notebooks-${{ matrix.channel }} + path: executed/ + retention-days: 90 diff --git a/README.md b/README.md index 7d2ee7f..39d4b2f 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,24 @@ its releases. Every notebook **checks** what it computes (error-level checks fai the run, warn-level ones are reported), and CI executes all of them and aggregates the findings of every language into one report. +## Executed notebooks (libKriging master) + +CI executes every notebook against the **development head of libKriging and of +its bindings** (master channel, see [CI](#ci)) and saves it with its outputs, +failing checks included (a notebook that fails is kept up to its failing cell): + +* **[latest executed notebooks (zip)](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-notebooks-master.zip)**: + the `executed-notebooks-master` artifact of the last run on `main` + (`python/`, `r/`, `octave/`, `julia/`); +* **[latest report, results and notebooks (zip)](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/report-master.zip)**: + the `report-master` artifact (aggregated `report.md`, all CSV results, executed + notebooks, refreshed baseline); +* **[all runs on `main`](https://github.com/libKriging/benchmarks/actions/workflows/notebooks.yml?query=branch%3Amain)**: + each run keeps these artifacts for 90 days, and its job summary shows the report. + +The same artifacts exist for the release channel (`executed-notebooks-release`, +`report-release`). + ## What is covered | notebook | problems (test functions) | libKriging features | compared with | @@ -66,8 +84,8 @@ mLibKriging on `OCTAVE_PATH`, STK on `STK_PATH` and `octave_kernel`; Julia needs Notebooks are generated from plain-text sources (`notebooks//src/`, cells separated by `# %%` / `# %% [markdown]`, or `% %%` in Octave), so that reviews are -plain diffs; outputs are not committed (CI publishes the executed notebooks as -artifacts). +plain diffs; outputs are not committed (CI archives the executed notebooks as +artifacts, see [Executed notebooks](#executed-notebooks-libkriging-master)). ```sh python tools/py2nb.py notebooks/*/src/* # regenerate @@ -94,7 +112,9 @@ pull request, weekly, on demand (`libkriging_ref`, `libkriging_repo`, another Octave version than Ubuntu's). One job per Python notebook, one per language for the others, then one report -per channel (job summary + artifact with the report, all CSV results, the -executed notebooks and a refreshed baseline). The initial `baseline/` files come +per channel (job summary + `report-` artifact with the report, all CSV +results, the executed notebooks and a refreshed baseline, + an +`executed-notebooks-` artifact with the executed notebooks only; see +[Executed notebooks](#executed-notebooks-libkriging-master)). The initial `baseline/` files come from a local run (Python notebooks only, 4-core container); to refresh them, commit the `new-baseline/` files of a trusted CI run. diff --git a/tools/run_notebooks.py b/tools/run_notebooks.py index bc1e8ac..fd2be35 100644 --- a/tools/run_notebooks.py +++ b/tools/run_notebooks.py @@ -1,21 +1,20 @@ #!/usr/bin/env python3 """Execute Jupyter notebooks and fail if any cell raises. -Used by CI (.github/workflows/notebooks.yml) to keep the worked notebooks -runnable, and to refresh their stored outputs (`--inplace`). +Used by CI (.github/workflows/notebooks.yml) to execute every notebook against the +installed libKriging and, with `--inplace`, to save the executed notebook with its +outputs. A notebook that fails is saved too, with the outputs of the cells run up +to the failing one (the error included): that is the run worth looking at. -The binding notebooks start with an "Installation" section of `%%bash` cells that -create a virtual environment, install requirements and build libKriging. Those -cells provision an environment and are not what the notebooks demonstrate: with -`--skip-setup` no `%%bash` cell (nor the cell that puts a built mlibkriging on the -Octave path) is run: they keep their stored content, no local path ends up in the -outputs, and the notebook is executed against the libKriging already available to -the kernel (Python: installed; Julia: `--project`; Octave: `OCTAVE_PATH`). +`--skip-setup` does not run the `%%bash` cells (nor a cell starting with +"% Add mlibkriging to path"), for notebooks that provision their own environment; +the notebooks of this repository have none and use the libKriging available to the +kernel (Python: installed; R: library; Octave: `OCTAVE_PATH`; Julia: environment). Usage: - python3 tools/notebooks/run_notebooks.py --skip-setup bindings/Python/*.ipynb - python3 tools/notebooks/run_notebooks.py --skip-setup --inplace --kernel python3 nb.ipynb - python3 tools/notebooks/run_notebooks.py --skip-cell '^repo_root = ' --prelude 'addpath(...)' nb.ipynb + python3 tools/run_notebooks.py notebooks/python/*.ipynb + python3 tools/run_notebooks.py --inplace --timeout 3600 --kernel julia-1.11 notebooks/julia/*.ipynb + python3 tools/run_notebooks.py --skip-cell '^repo_root = ' --prelude 'addpath(...)' nb.ipynb Requires nbclient, nbformat and ipykernel. """ @@ -74,23 +73,29 @@ def run(path, kernel, timeout, skip_setup, inplace, skip_patterns=(), prelude=No record_timing=False, ) start = time.time() + ok, info = True, "" try: client.execute() - except CellExecutionError as exc: - return False, f"{exc}".strip()[-1500:], time.time() - start - - errors = output_errors(work) - if errors: - return False, "\n".join(list(dict.fromkeys(errors))[:10]), time.time() - start + except Exception as exc: # noqa: BLE001 - CellExecutionError, timeout, dead kernel + msg = str(exc) if isinstance(exc, CellExecutionError) else f"{type(exc).__name__}: {exc}" + ok, info = False, msg.strip()[-1500:] + if ok: + errors = output_errors(work) + if errors: + ok, info = False, "\n".join(list(dict.fromkeys(errors))[:10]) + elapsed = time.time() - start if inplace: - # Merge the outputs back, leaving the skipped setup cells as they were. + # Merge the outputs back (also after a failure: the cells run so far keep their + # outputs, the error included), leaving the skipped setup cells as they were. for i, done in zip(kept, work.cells[1 if prelude else 0:]): if done.cell_type == "code": nb.cells[i].outputs = done.outputs nb.cells[i].execution_count = done.execution_count nbformat.write(nb, path) - return True, f"{len(skipped)} setup cell(s) skipped" if skipped else "", time.time() - start + if ok and skipped: + info = f"{len(skipped)} setup cell(s) skipped" + return ok, info, elapsed def main(argv=None): From 39918ccf5d4ea815a35733e96321e89c3a3d58e8 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 14 Oct 2026 12:46:01 +0000 Subject: [PATCH 03/16] Notebooks at the root, named by binding, kriging type and theme; tools/ for the rest; one CI job and artifact per notebook Layout - every notebook at the repository root, named __.ipynb, its suite (results/.*.csv, tools/known_issues.csv, tools/baseline/) named the same; - everything else under tools/: notebook sources (tools/src/), lkbench and its pyproject, R/Octave/Julia check helpers, binding installation (tools/install/), py2nb, run_notebooks, report, make_data, known issues, baselines; - the notebooks run from the root (paths relative to it); lkbench finds the repository from the working directory (or LKBENCH_ROOT) and raises when it cannot, instead of silently ignoring the known issues of a non-editable install. Notebooks - split by class: python_Kriging_high-dimension-normalize-screening / python_MLPKriging_deep-kernel-projected-branin, python_Kriging-vecchia-nystrom_large-n / python_NestedKriging_aggregations-large-n, julia_Kriging_cross-binding / julia_Kriging_vs-GaussianProcesses; - new R_, octave_ and julia_all-classes_api-tour: Kriging (noise-free, nugget, known noise, Vecchia, Nystrom), WarpKriging (continuous, categorical), MLPKriging, NestedKriging (NK, PoE), update, simulate, save/load, through each binding; - third-party packages installed by the notebooks that compare with them: scikit-learn/SMT/OpenTURNS/GPy (pip, numpy pinned), DiceKriging/RobustGaSP (CRAN), STK (git clone), GaussianProcesses.jl (Pkg.add). CI - setup: notebooks in sync with tools/src (no orphan) and all linked from the README; abbreviated commits resolved; release channel dropped with a warning when the repository has no release; one matrix entry per notebook and channel; - builds from the same libKriging commit: Python wheel, R package from bindings/R (no longer the rlibkriging repository), Octave mex, JLibKriging.jl (depot cached); release channel weekly or on demand; - one job per notebook (all languages), each uploading executed-- (executed notebook with outputs, failures included, and its results); - concurrency grouped by event as well. - tools/install/r.sh builds the bench-* executables that libKriging's install.lib target installs without building. README: one table per binding, each notebook linked to its source and to its last executed artifact on main. Run locally against libKriging master (ef052e1): all Python notebooks, the R and Octave tours and cross-binding notebooks, octave_Kriging_vs-STK pass (0 FAIL); R_Kriging_vs-DiceKriging-RobustGaSP not run (CRAN unreachable here); Julia not run. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01QiFEE514dpwS57mcFWDeZG --- .github/workflows/notebooks.yml | 457 +++++---- README.md | 181 ++-- ...ing.ipynb => R_Kriging_cross-binding.ipynb | 8 +- ... R_Kriging_vs-DiceKriging-RobustGaSP.ipynb | 50 +- R_all-classes_api-tour.ipynb | 320 ++++++ baseline/metrics-master.csv | 948 ----------------- baseline/metrics-release.csv | 916 ----------------- julia_Kriging_cross-binding.ipynb | 95 ++ ...> julia_Kriging_vs-GaussianProcesses.ipynb | 89 +- julia_all-classes_api-tour.ipynb | 313 ++++++ known_issues.csv | 11 - notebooks/julia/src/01_cross_binding.jl | 95 -- notebooks/octave/lkb_suite.m | 20 - notebooks/octave/read_data.m | 3 - ...pynb => octave_Kriging_cross-binding.ipynb | 9 +- ...s_stk.ipynb => octave_Kriging_vs-STK.ipynb | 52 +- octave_all-classes_api-tour.ipynb | 328 ++++++ ...thon_Kriging-nugget-noise_noisy-data.ipynb | 4 +- ...thon_Kriging-vecchia-nystrom_large-n.ipynb | 56 +- ...thon_Kriging_cross-binding-reference.ipynb | 8 +- ...g_high-dimension-normalize-screening.ipynb | 72 +- ...Kriging_kernels-trends-interpolation.ipynb | 8 +- ...ging_objectives-LL-LOO-LMP-gradients.ipynb | 4 +- ...on_Kriging_robustness-invalid-inputs.ipynb | 4 +- ...nb => python_Kriging_update-simulate.ipynb | 4 +- ...Kriging_vs-sklearn-SMT-OpenTURNS-GPy.ipynb | 59 +- ...Kriging_deep-kernel-projected-branin.ipynb | 113 +++ ...n_NestedKriging_aggregations-large-n.ipynb | 161 +++ ...Kriging_warpings-categorical-ordinal.ipynb | 4 +- ...all-classes_save-load-pickle-sklearn.ipynb | 42 +- ...odel_test-functions-reference-values.ipynb | 4 +- tools/baseline/metrics-master.csv | 956 ++++++++++++++++++ tools/baseline/metrics-release.csv | 924 +++++++++++++++++ tools/install/julia.sh | 20 + tools/install/octave.sh | 17 + tools/install/python.sh | 27 + tools/install/r.sh | 35 + {notebooks => tools}/julia/checks.jl | 17 +- tools/known_issues.csv | 11 + {lkbench => tools/lkbench}/__init__.py | 0 {lkbench => tools/lkbench}/checks.py | 32 +- {lkbench => tools/lkbench}/compat.py | 0 {lkbench => tools/lkbench}/design.py | 0 {lkbench => tools/lkbench}/functions.py | 2 +- {lkbench => tools/lkbench}/metrics.py | 0 {notebooks => tools}/octave/lkb_check.m | 0 {notebooks => tools}/octave/lkb_close.m | 0 {notebooks => tools}/octave/lkb_finalize.m | 2 +- {notebooks => tools}/octave/lkb_metric.m | 0 tools/octave/lkb_suite.m | 39 + tools/octave/read_data.m | 3 + tools/py2nb.py | 26 +- pyproject.toml => tools/pyproject.toml | 5 +- {notebooks => tools}/r/checks.R | 13 +- tools/report.py | 29 +- tools/run_notebooks.py | 11 +- .../src/R_Kriging_cross-binding.R | 8 +- .../src/R_Kriging_vs-DiceKriging-RobustGaSP.R | 24 +- tools/src/R_all-classes_api-tour.R | 205 ++++ tools/src/julia_Kriging_cross-binding.jl | 52 + .../src/julia_Kriging_vs-GaussianProcesses.jl | 71 ++ tools/src/julia_all-classes_api-tour.jl | 198 ++++ .../src/octave_Kriging_cross-binding.m | 9 +- .../src/octave_Kriging_vs-STK.m | 30 +- tools/src/octave_all-classes_api-tour.m | 213 ++++ .../python_Kriging-nugget-noise_noisy-data.py | 4 +- .../python_Kriging-vecchia-nystrom_large-n.py | 42 +- .../python_Kriging_cross-binding-reference.py | 8 +- ...ging_high-dimension-normalize-screening.py | 58 +- ...on_Kriging_kernels-trends-interpolation.py | 8 +- ...Kriging_objectives-LL-LOO-LMP-gradients.py | 4 +- ...ython_Kriging_robustness-invalid-inputs.py | 4 +- .../src/python_Kriging_update-simulate.py | 4 +- ...on_Kriging_vs-sklearn-SMT-OpenTURNS-GPy.py | 29 +- ...MLPKriging_deep-kernel-projected-branin.py | 70 ++ ...thon_NestedKriging_aggregations-large-n.py | 101 ++ ...arpKriging_warpings-categorical-ordinal.py | 4 +- ...on_all-classes_save-load-pickle-sklearn.py | 26 +- ...o-model_test-functions-reference-values.py | 4 +- 79 files changed, 5107 insertions(+), 2676 deletions(-) rename notebooks/r/01_cross_binding.ipynb => R_Kriging_cross-binding.ipynb (92%) rename notebooks/r/02_vs_dicekriging_robustgasp.ipynb => R_Kriging_vs-DiceKriging-RobustGaSP.ipynb (84%) create mode 100644 R_all-classes_api-tour.ipynb delete mode 100644 baseline/metrics-master.csv delete mode 100644 baseline/metrics-release.csv create mode 100644 julia_Kriging_cross-binding.ipynb rename notebooks/julia/01_cross_binding.ipynb => julia_Kriging_vs-GaussianProcesses.ipynb (54%) create mode 100644 julia_all-classes_api-tour.ipynb delete mode 100644 known_issues.csv delete mode 100644 notebooks/julia/src/01_cross_binding.jl delete mode 100644 notebooks/octave/lkb_suite.m delete mode 100644 notebooks/octave/read_data.m rename notebooks/octave/01_cross_binding.ipynb => octave_Kriging_cross-binding.ipynb (89%) rename notebooks/octave/02_vs_stk.ipynb => octave_Kriging_vs-STK.ipynb (76%) create mode 100644 octave_all-classes_api-tour.ipynb rename notebooks/python/03_noise.ipynb => python_Kriging-nugget-noise_noisy-data.ipynb (98%) rename notebooks/python/07_scalability.ipynb => python_Kriging-vecchia-nystrom_large-n.ipynb (74%) rename notebooks/python/11_cross_binding.ipynb => python_Kriging_cross-binding-reference.ipynb (94%) rename notebooks/python/06_high_dimension_mlp.ipynb => python_Kriging_high-dimension-normalize-screening.ipynb (61%) rename notebooks/python/01_kriging_kernels_trends.ipynb => python_Kriging_kernels-trends-interpolation.ipynb (97%) rename notebooks/python/02_objectives_gradients.ipynb => python_Kriging_objectives-LL-LOO-LMP-gradients.ipynb (98%) rename notebooks/python/09_robustness.ipynb => python_Kriging_robustness-invalid-inputs.ipynb (98%) rename notebooks/python/04_update_simulate.ipynb => python_Kriging_update-simulate.ipynb (98%) rename notebooks/python/10_compare_python_packages.ipynb => python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy.ipynb (90%) create mode 100644 python_MLPKriging_deep-kernel-projected-branin.ipynb create mode 100644 python_NestedKriging_aggregations-large-n.ipynb rename notebooks/python/05_warpkriging.ipynb => python_WarpKriging_warpings-categorical-ordinal.ipynb (98%) rename notebooks/python/08_api_persistence.ipynb => python_all-classes_save-load-pickle-sklearn.ipynb (89%) rename notebooks/python/00_test_functions.ipynb => python_no-model_test-functions-reference-values.ipynb (96%) create mode 100644 tools/baseline/metrics-master.csv create mode 100644 tools/baseline/metrics-release.csv create mode 100755 tools/install/julia.sh create mode 100755 tools/install/octave.sh create mode 100755 tools/install/python.sh create mode 100755 tools/install/r.sh rename {notebooks => tools}/julia/checks.jl (82%) create mode 100644 tools/known_issues.csv rename {lkbench => tools/lkbench}/__init__.py (100%) rename {lkbench => tools/lkbench}/checks.py (85%) rename {lkbench => tools/lkbench}/compat.py (100%) rename {lkbench => tools/lkbench}/design.py (100%) rename {lkbench => tools/lkbench}/functions.py (99%) rename {lkbench => tools/lkbench}/metrics.py (100%) rename {notebooks => tools}/octave/lkb_check.m (100%) rename {notebooks => tools}/octave/lkb_close.m (100%) rename {notebooks => tools}/octave/lkb_finalize.m (92%) rename {notebooks => tools}/octave/lkb_metric.m (100%) create mode 100644 tools/octave/lkb_suite.m create mode 100644 tools/octave/read_data.m rename pyproject.toml => tools/pyproject.toml (63%) rename {notebooks => tools}/r/checks.R (88%) rename notebooks/r/src/01_cross_binding.R => tools/src/R_Kriging_cross-binding.R (89%) rename notebooks/r/src/02_vs_dicekriging_robustgasp.R => tools/src/R_Kriging_vs-DiceKriging-RobustGaSP.R (84%) create mode 100644 tools/src/R_all-classes_api-tour.R create mode 100644 tools/src/julia_Kriging_cross-binding.jl create mode 100644 tools/src/julia_Kriging_vs-GaussianProcesses.jl create mode 100644 tools/src/julia_all-classes_api-tour.jl rename notebooks/octave/src/01_cross_binding.m => tools/src/octave_Kriging_cross-binding.m (86%) rename notebooks/octave/src/02_vs_stk.m => tools/src/octave_Kriging_vs-STK.m (75%) create mode 100644 tools/src/octave_all-classes_api-tour.m rename notebooks/python/src/03_noise.py => tools/src/python_Kriging-nugget-noise_noisy-data.py (97%) rename notebooks/python/src/07_scalability.py => tools/src/python_Kriging-vecchia-nystrom_large-n.py (72%) rename notebooks/python/src/11_cross_binding.py => tools/src/python_Kriging_cross-binding-reference.py (93%) rename notebooks/python/src/06_high_dimension_mlp.py => tools/src/python_Kriging_high-dimension-normalize-screening.py (57%) rename notebooks/python/src/01_kriging_kernels_trends.py => tools/src/python_Kriging_kernels-trends-interpolation.py (96%) rename notebooks/python/src/02_objectives_gradients.py => tools/src/python_Kriging_objectives-LL-LOO-LMP-gradients.py (98%) rename notebooks/python/src/09_robustness.py => tools/src/python_Kriging_robustness-invalid-inputs.py (98%) rename notebooks/python/src/04_update_simulate.py => tools/src/python_Kriging_update-simulate.py (98%) rename notebooks/python/src/10_compare_python_packages.py => tools/src/python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy.py (90%) create mode 100644 tools/src/python_MLPKriging_deep-kernel-projected-branin.py create mode 100644 tools/src/python_NestedKriging_aggregations-large-n.py rename notebooks/python/src/05_warpkriging.py => tools/src/python_WarpKriging_warpings-categorical-ordinal.py (97%) rename notebooks/python/src/08_api_persistence.py => tools/src/python_all-classes_save-load-pickle-sklearn.py (90%) rename notebooks/python/src/00_test_functions.py => tools/src/python_no-model_test-functions-reference-values.py (94%) diff --git a/.github/workflows/notebooks.yml b/.github/workflows/notebooks.yml index 7252f12..1d4b641 100644 --- a/.github/workflows/notebooks.yml +++ b/.github/workflows/notebooks.yml @@ -1,33 +1,32 @@ name: notebooks -# Executes every notebook of the benchmark against libKriging, then aggregates the -# checks of all languages into one report (tools/report.py), published as the job -# summary and as an artifact. +# Executes every notebook of the repository root against libKriging, one job per +# notebook, each job uploading its executed notebook (outputs saved, failures +# included) and its results as the artifact `executed--`; then +# aggregates the checks of all languages into one report per channel +# (tools/report.py), published as the job summary and as artifacts. # -# Two channels: -# master libKriging built from source at `libkriging_ref` (default: master), all -# languages: Python (wheel built here), R (rlibkriging repository head, -# which follows libKriging master), Octave (mLibKriging built here), -# Julia (JLibKriging.jl built against the same libKriging checkout). +# Channels: +# master libKriging and its bindings built from source at `libkriging_ref` +# (default: master) of `libkriging_repo`: Python wheel, R package +# (bindings/R), Octave mex, JLibKriging.jl, all from the same commit. +# The default for every event except the weekly schedule. # release the last published packages: pylibkriging from PyPI, rlibkriging from -# CRAN, mLibKriging built from the last release tag. A feature missing -# from the release is recorded as SKIP, not as a failure. +# CRAN, mLibKriging built from the last release tag (no Julia). A +# feature missing from the release is recorded as SKIP. Weekly, or on +# demand (`channels` input). # -# A notebook fails (red job) when one of its error-level checks fails; the report -# job fails when any notebook failed or crashed, or when the bindings disagree -# (cross-binding protocol, notebooks/python/11_cross_binding.ipynb). -# -# The notebooks executed against libKriging master are saved with their outputs -# (failures included: a notebook whose checks fail is kept up to its failing cell) -# and archived as the `executed-notebooks-master` artifact of every run, linked -# from the README. +# A notebook job fails when one of its error-level checks fails; the report job +# fails when any notebook failed or crashed, or when the bindings disagree +# (cross-binding protocol, python_Kriging_cross-binding-reference.ipynb). +# The binding installations are the scripts of tools/install/, usable locally too. on: push: branches: [main] pull_request: schedule: - - cron: '17 3 * * 1' # weekly, Monday 03:17 UTC + - cron: '17 3 * * 1' # weekly, Monday 03:17 UTC: master and release workflow_dispatch: inputs: libkriging_ref: @@ -38,7 +37,7 @@ on: default: 'libKriging/libKriging' channels: description: 'channels to run: "master", "release" or "master,release"' - default: 'master,release' + default: 'master' repository_dispatch: # libKriging can trigger a run after a merge: # POST /repos//dispatches {"event_type": "libkriging-updated", "client_payload": {"ref": ""}} @@ -48,7 +47,8 @@ permissions: contents: read concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + # per event too: a repository_dispatch on main does not cancel a push run on main + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true env: @@ -59,6 +59,7 @@ env: # NestedKriging's OpenMP loop can hang on 2-core runners (same workaround as libKriging's CI) OMP_NUM_THREADS: 1 PYTHON_VERSION: '3.12' + JULIA_VERSION: '1.11' jobs: # --------------------------------------------------------------------------- @@ -68,37 +69,64 @@ jobs: sha: ${{ steps.refs.outputs.sha }} release_tag: ${{ steps.refs.outputs.release_tag }} channels: ${{ steps.refs.outputs.channels }} + python: ${{ steps.matrix.outputs.python }} + r: ${{ steps.matrix.outputs.r }} + octave: ${{ steps.matrix.outputs.octave }} + julia: ${{ steps.matrix.outputs.julia }} steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: python-version: ${{ env.PYTHON_VERSION }} - - name: Notebooks in sync with their sources + - name: Notebooks in sync with their sources, and listed in the README run: | pip install nbformat - python tools/py2nb.py --check notebooks/*/src/* - - name: Resolve libKriging refs + python tools/py2nb.py --check + missing=$(for nb in *.ipynb; do grep -qF "($nb)" README.md || echo "$nb"; done) + if [ -n "$missing" ]; then echo "not linked from README.md: $missing"; exit 1; fi + - name: Resolve libKriging refs and channels id: refs env: GH_TOKEN: ${{ github.token }} - CHANNELS: ${{ inputs.channels || 'master,release' }} + CHANNELS: ${{ github.event_name == 'schedule' && 'master,release' || inputs.channels || 'master' }} run: | sha=$(git ls-remote "https://github.com/${LIBKRIGING_REPO}" "${LIBKRIGING_REF}" | head -1 | cut -f1) - [ -n "$sha" ] || sha="${LIBKRIGING_REF}" # already a commit - tag=$(gh api "repos/${LIBKRIGING_REPO}/releases/latest" --jq .tag_name) + if [ -z "$sha" ]; then # a commit, possibly abbreviated + sha=$(gh api "repos/${LIBKRIGING_REPO}/commits/${LIBKRIGING_REF}" --jq .sha) + fi channels="$CHANNELS" - echo "sha=$sha" >> "$GITHUB_OUTPUT" - echo "release_tag=$tag" >> "$GITHUB_OUTPUT" - echo "channels=$(python -c "import json,sys; print(json.dumps(sys.argv[1].split(',')))" "$channels")" >> "$GITHUB_OUTPUT" - echo "libKriging ${LIBKRIGING_REPO}@${LIBKRIGING_REF} = $sha; last release $tag; channels $channels" >> "$GITHUB_STEP_SUMMARY" + tag=$(gh api "repos/${LIBKRIGING_REPO}/releases/latest" --jq .tag_name 2>/dev/null || true) + if [ -z "$tag" ] && [[ "$channels" == *release* ]]; then + echo "::warning::${LIBKRIGING_REPO} has no release: release channel skipped" + channels=$(echo "$channels" | tr ',' '\n' | grep -vx release | paste -sd, -) + fi + { echo "sha=$sha"; echo "release_tag=$tag"; echo "channels=$channels"; } >> "$GITHUB_OUTPUT" + echo "libKriging ${LIBKRIGING_REPO}@${LIBKRIGING_REF} = $sha; last release ${tag:-none}; channels $channels" >> "$GITHUB_STEP_SUMMARY" + - name: One job per notebook and channel + id: matrix + env: + CHANNELS: ${{ steps.refs.outputs.channels }} + run: | + python - <<'EOF' >> "$GITHUB_OUTPUT" + import json, os + from pathlib import Path + channels = os.environ["CHANNELS"].split(",") + for lang, prefix in [("python", "python"), ("r", "R"), ("octave", "octave"), ("julia", "julia")]: + chans = [c for c in channels if c == "master" or lang != "julia"] # no Julia release + include = [{"channel": c, "notebook": nb.stem} for c in chans + for nb in sorted(Path(".").glob(f"{prefix}_*.ipynb"))] + print(f"{lang}={json.dumps({'include': include or [{'channel': 'none', 'notebook': 'none'}]})}") + EOF + cat "$GITHUB_OUTPUT" - # --------------------------------------------------------------------------- + # --------------------------------------------------------------------------- builds build-python: needs: setup if: contains(needs.setup.outputs.channels, 'master') runs-on: ubuntu-24.04 timeout-minutes: 120 steps: + - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: python-version: ${{ env.PYTHON_VERSION }} @@ -118,229 +146,270 @@ jobs: if: steps.cache.outputs.cache-hit != 'true' run: | sudo apt-get update && sudo apt-get install -y liblapack-dev libopenblas-dev - python -m pip install --upgrade pip - python -m pip install -r libKriging/bindings/Python/pylibkriging/requirements.txt \ - -r libKriging/bindings/Python/pylibkriging/dev-requirements.txt cmake - export CMAKE_BUILD_PARALLEL_LEVEL=4 - python -m pip wheel --no-build-isolation --no-deps -w dist ./libKriging + tools/install/python.sh wheel libKriging dist - uses: actions/upload-artifact@v4 with: - name: wheel-master + name: build-python-master path: dist/*.whl - # --------------------------------------------------------------------------- - python: - needs: [setup, build-python] - if: always() && needs.setup.result == 'success' + build-r: + needs: setup + if: contains(needs.setup.outputs.channels, 'master') runs-on: ubuntu-24.04 - timeout-minutes: 90 + timeout-minutes: 120 + steps: + - uses: actions/checkout@v4 + - uses: actions/checkout@v4 + with: + repository: ${{ env.LIBKRIGING_REPO }} + ref: ${{ needs.setup.outputs.sha }} + submodules: recursive + path: libKriging + - uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + - uses: r-lib/actions/setup-r@v2 + with: + use-public-rspm: true + - name: Build rlibkriging from libKriging's bindings/R + run: | + sudo apt-get update && sudo apt-get install -y liblapack-dev libopenblas-dev gfortran + tools/install/r.sh master "$RUNNER_TEMP/Rlib" libKriging + tar -C "$RUNNER_TEMP" -cf Rlib.tar Rlib + - uses: actions/upload-artifact@v4 + with: + name: build-r-master + path: Rlib.tar + + build-octave: + needs: setup + runs-on: ubuntu-24.04 + timeout-minutes: 120 strategy: - fail-fast: false matrix: channel: [master, release] - notebook: - - 00_test_functions - - 01_kriging_kernels_trends - - 02_objectives_gradients - - 03_noise - - 04_update_simulate - - 05_warpkriging - - 06_high_dimension_mlp - - 07_scalability - - 08_api_persistence - - 09_robustness - - 10_compare_python_packages - - 11_cross_binding - name: python ${{ matrix.channel }} ${{ matrix.notebook }} steps: - name: Channel enabled? id: on - run: | - echo 'enabled=${{ contains(needs.setup.outputs.channels, matrix.channel) && (matrix.channel != 'master' || needs.build-python.result == 'success') }}' >> "$GITHUB_OUTPUT" + run: echo 'enabled=${{ contains(needs.setup.outputs.channels, matrix.channel) }}' >> "$GITHUB_OUTPUT" - if: steps.on.outputs.enabled == 'true' uses: actions/checkout@v4 + - if: steps.on.outputs.enabled == 'true' + id: cache + uses: actions/cache@v4 + with: + path: mlibkriging + key: mlibkriging-${{ matrix.channel == 'master' && needs.setup.outputs.sha || needs.setup.outputs.release_tag }}-ubuntu-24.04-octave + - if: steps.on.outputs.enabled == 'true' && steps.cache.outputs.cache-hit != 'true' + uses: actions/checkout@v4 + with: + repository: ${{ env.LIBKRIGING_REPO }} + ref: ${{ matrix.channel == 'master' && needs.setup.outputs.sha || needs.setup.outputs.release_tag }} + submodules: recursive + path: libKriging - if: steps.on.outputs.enabled == 'true' uses: actions/setup-python@v5 with: python-version: ${{ env.PYTHON_VERSION }} - - if: steps.on.outputs.enabled == 'true' && matrix.channel == 'master' + - name: Build mLibKriging + if: steps.on.outputs.enabled == 'true' + run: | + sudo apt-get update && sudo apt-get install -y octave octave-dev liblapack-dev libopenblas-dev + tools/install/octave.sh "${{ steps.cache.outputs.cache-hit == 'true' && 'mlibkriging' || 'libKriging' }}" mlibkriging + - if: steps.on.outputs.enabled == 'true' + uses: actions/upload-artifact@v4 + with: + name: build-octave-${{ matrix.channel }} + path: mlibkriging/ + + build-julia: + needs: setup + if: contains(needs.setup.outputs.channels, 'master') + runs-on: ubuntu-24.04 + timeout-minutes: 120 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + - uses: julia-actions/setup-julia@v2 + with: + version: ${{ env.JULIA_VERSION }} + - id: cache + uses: actions/cache@v4 + with: + path: ~/.julia + key: julia-depot-${{ needs.setup.outputs.sha }}-julia${{ env.JULIA_VERSION }} + - if: steps.cache.outputs.cache-hit != 'true' + uses: actions/checkout@v4 + with: + repository: ${{ env.LIBKRIGING_REPO }} + ref: ${{ needs.setup.outputs.sha }} + submodules: recursive + path: libKriging + - name: Build JLibKriging.jl against the libKriging checkout + if: steps.cache.outputs.cache-hit != 'true' + run: | + sudo apt-get update && sudo apt-get install -y liblapack-dev libopenblas-dev + tools/install/julia.sh libKriging + + # --------------------------------------------------------------------------- notebooks + python: + needs: [setup, build-python] + if: always() && needs.setup.result == 'success' && !contains(needs.setup.outputs.python, '"none"') + runs-on: ubuntu-24.04 + timeout-minutes: 120 + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.setup.outputs.python) }} + name: ${{ matrix.notebook }} (${{ matrix.channel }}) + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + - if: matrix.channel == 'master' uses: actions/download-artifact@v4 with: - name: wheel-master + name: build-python-master path: dist - - name: Install - if: steps.on.outputs.enabled == 'true' - run: | - python -m pip install --upgrade pip - if [ "${{ matrix.channel }}" = master ]; then pip install dist/*.whl; else pip install pylibkriging; fi - pip install -e ".[notebooks,compare]" jinja2 - python -c "import pylibkriging as lk; print('pylibkriging', lk.__version__)" - - name: Execute the notebook - if: steps.on.outputs.enabled == 'true' - run: python tools/run_notebooks.py --inplace --timeout 3600 notebooks/python/${{ matrix.notebook }}.ipynb - - if: always() && steps.on.outputs.enabled == 'true' + - name: Install pylibkriging (${{ matrix.channel }}) + run: tools/install/python.sh ${{ matrix.channel }} dist + - name: Execute and save the notebook + run: python tools/run_notebooks.py --inplace --timeout 3600 ${{ matrix.notebook }}.ipynb + - if: always() uses: actions/upload-artifact@v4 with: - name: results-${{ matrix.channel }}-py-${{ matrix.notebook }} + name: executed-${{ matrix.channel }}-${{ matrix.notebook }} path: | + ${{ matrix.notebook }}.ipynb results/ - notebooks/python/${{ matrix.notebook }}.ipynb + retention-days: 90 - # --------------------------------------------------------------------------- r: - needs: setup + needs: [setup, build-r] + if: always() && needs.setup.result == 'success' && !contains(needs.setup.outputs.r, '"none"') runs-on: ubuntu-24.04 timeout-minutes: 90 strategy: fail-fast: false - matrix: - channel: [master, release] - name: R ${{ matrix.channel }} + matrix: ${{ fromJSON(needs.setup.outputs.r) }} + name: ${{ matrix.notebook }} (${{ matrix.channel }}) steps: - - name: Channel enabled? - id: on - run: echo 'enabled=${{ contains(needs.setup.outputs.channels, matrix.channel) }}' >> "$GITHUB_OUTPUT" - - if: steps.on.outputs.enabled == 'true' - uses: actions/checkout@v4 - - if: steps.on.outputs.enabled == 'true' - uses: actions/setup-python@v5 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: python-version: ${{ env.PYTHON_VERSION }} - - if: steps.on.outputs.enabled == 'true' - uses: r-lib/actions/setup-r@v2 + - uses: r-lib/actions/setup-r@v2 with: use-public-rspm: true - - name: Install - if: steps.on.outputs.enabled == 'true' - env: - GITHUB_PAT: ${{ github.token }} + - if: matrix.channel == 'master' + uses: actions/download-artifact@v4 + with: + name: build-r-master + - name: Install rlibkriging (${{ matrix.channel }}) run: | - sudo apt-get update && sudo apt-get install -y liblapack-dev gfortran cmake - pip install nbclient nbformat ipykernel jupyter_client - Rscript -e 'install.packages(c("IRkernel", "DiceKriging", "RobustGaSP", "remotes"))' + sudo apt-get update && sudo apt-get install -y liblapack-dev libopenblas-dev if [ "${{ matrix.channel }}" = master ]; then - Rscript -e 'remotes::install_github("libKriging/rlibkriging", upgrade = "never")' + tar -C "$RUNNER_TEMP" -xf Rlib.tar + echo "R_LIBS_USER=$RUNNER_TEMP/Rlib" >> "$GITHUB_ENV" + R_LIBS_USER="$RUNNER_TEMP/Rlib" Rscript -e 'library(rlibkriging)' + python -m pip install nbclient nbformat ipykernel jupyter_client + R_LIBS_USER="$RUNNER_TEMP/Rlib" Rscript -e 'if (!requireNamespace("IRkernel", quietly = TRUE)) install.packages("IRkernel", lib = Sys.getenv("R_LIBS_USER")); IRkernel::installspec(user = TRUE)' else - Rscript -e 'install.packages("rlibkriging")' + tools/install/r.sh release "$RUNNER_TEMP/Rlib" + echo "R_LIBS_USER=$RUNNER_TEMP/Rlib" >> "$GITHUB_ENV" fi - Rscript -e 'IRkernel::installspec(user = TRUE); cat("rlibkriging", as.character(packageVersion("rlibkriging")), "\n")' - - name: Execute the notebooks - if: steps.on.outputs.enabled == 'true' - run: python tools/run_notebooks.py --inplace --timeout 3600 notebooks/r/*.ipynb - - if: always() && steps.on.outputs.enabled == 'true' + - name: Execute and save the notebook + run: python tools/run_notebooks.py --inplace --timeout 3600 ${{ matrix.notebook }}.ipynb + - if: always() uses: actions/upload-artifact@v4 with: - name: results-${{ matrix.channel }}-r + name: executed-${{ matrix.channel }}-${{ matrix.notebook }} path: | + ${{ matrix.notebook }}.ipynb results/ - notebooks/r/*.ipynb + retention-days: 90 - # --------------------------------------------------------------------------- octave: - needs: setup + needs: [setup, build-octave] + if: always() && needs.setup.result == 'success' && !contains(needs.setup.outputs.octave, '"none"') runs-on: ubuntu-24.04 - timeout-minutes: 120 + timeout-minutes: 90 strategy: fail-fast: false - matrix: - channel: [master, release] - name: Octave ${{ matrix.channel }} - env: - # the release binaries target another Octave version than Ubuntu's: build from source - REF: ${{ matrix.channel == 'master' && needs.setup.outputs.sha || needs.setup.outputs.release_tag }} + matrix: ${{ fromJSON(needs.setup.outputs.octave) }} + name: ${{ matrix.notebook }} (${{ matrix.channel }}) steps: - - name: Channel enabled? - id: on - run: echo 'enabled=${{ contains(needs.setup.outputs.channels, matrix.channel) }}' >> "$GITHUB_OUTPUT" - - if: steps.on.outputs.enabled == 'true' - uses: actions/checkout@v4 - - if: steps.on.outputs.enabled == 'true' - uses: actions/setup-python@v5 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: python-version: ${{ env.PYTHON_VERSION }} - - name: Install Octave - if: steps.on.outputs.enabled == 'true' - run: | - sudo apt-get update && sudo apt-get install -y octave octave-dev liblapack-dev libopenblas-dev - pip install nbclient nbformat ipykernel octave_kernel - git clone --depth 1 https://github.com/stk-kriging/stk.git "$RUNNER_TEMP/stk" - - if: steps.on.outputs.enabled == 'true' - id: cache - uses: actions/cache@v4 + - uses: actions/download-artifact@v4 with: + name: build-octave-${{ matrix.channel }} path: mlibkriging - key: mlibkriging-${{ env.REF }}-${{ runner.os }}-octave - - if: steps.on.outputs.enabled == 'true' && steps.cache.outputs.cache-hit != 'true' - uses: actions/checkout@v4 - with: - repository: ${{ env.LIBKRIGING_REPO }} - ref: ${{ env.REF }} - submodules: recursive - path: libKriging - - name: Build mLibKriging - if: steps.on.outputs.enabled == 'true' && steps.cache.outputs.cache-hit != 'true' - run: | - cmake -S libKriging -B build -DCMAKE_BUILD_TYPE=Release -DENABLE_OCTAVE_BINDING=on \ - -DENABLE_PYTHON_BINDING=off -DENABLE_MATLAB_BINDING=off -DBUILD_SHARED_LIBS=off -DSTATIC_LIB=on - cmake --build build --target mLibKriging -j 4 - mkdir -p mlibkriging - cp build/bindings/Octave/mlibkriging/mLibKriging.mex libKriging/bindings/Octave/mlibkriging/*.m mlibkriging/ - - name: Execute the notebooks - if: steps.on.outputs.enabled == 'true' + - name: Install Octave and mLibKriging (${{ matrix.channel }}) run: | - export OCTAVE_PATH="$PWD/mlibkriging" STK_PATH="$RUNNER_TEMP/stk" - python tools/run_notebooks.py --inplace --timeout 3600 notebooks/octave/*.ipynb - - if: always() && steps.on.outputs.enabled == 'true' + sudo apt-get update && sudo apt-get install -y octave liblapack-dev libopenblas-dev + tools/install/octave.sh mlibkriging mlibkriging # already built: only the kernel and a smoke test + echo "OCTAVE_PATH=$PWD/mlibkriging" >> "$GITHUB_ENV" + - name: Execute and save the notebook + run: python tools/run_notebooks.py --inplace --timeout 3600 ${{ matrix.notebook }}.ipynb + - if: always() uses: actions/upload-artifact@v4 with: - name: results-${{ matrix.channel }}-octave + name: executed-${{ matrix.channel }}-${{ matrix.notebook }} path: | + ${{ matrix.notebook }}.ipynb results/ - notebooks/octave/*.ipynb + retention-days: 90 - # --------------------------------------------------------------------------- julia: - needs: setup - if: contains(needs.setup.outputs.channels, 'master') + needs: [setup, build-julia] + if: always() && needs.setup.result == 'success' && !contains(needs.setup.outputs.julia, '"none"') runs-on: ubuntu-24.04 timeout-minutes: 120 - name: Julia master + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.setup.outputs.julia) }} + name: ${{ matrix.notebook }} (${{ matrix.channel }}) steps: - uses: actions/checkout@v4 - - uses: actions/checkout@v4 - with: - repository: ${{ env.LIBKRIGING_REPO }} - ref: ${{ needs.setup.outputs.sha }} - submodules: recursive - path: libKriging - uses: actions/setup-python@v5 with: python-version: ${{ env.PYTHON_VERSION }} - uses: julia-actions/setup-julia@v2 with: - version: '1.11' - - name: Install + version: ${{ env.JULIA_VERSION }} + - id: cache + uses: actions/cache/restore@v4 + with: + path: ~/.julia + key: julia-depot-${{ needs.setup.outputs.sha }}-julia${{ env.JULIA_VERSION }} + - if: steps.cache.outputs.cache-hit != 'true' + uses: actions/checkout@v4 + with: + repository: ${{ env.LIBKRIGING_REPO }} + ref: ${{ needs.setup.outputs.sha }} + submodules: recursive + path: libKriging + - name: Install JLibKriging.jl (built by build-julia, or here on a cache miss) run: | sudo apt-get update && sudo apt-get install -y liblapack-dev libopenblas-dev - pip install nbclient nbformat ipykernel jupyter_client - # JLibKriging.jl compiles libKriging; LIBKRIGING_SRC_DIR makes it use this exact checkout - export LIBKRIGING_SRC_DIR="$PWD/libKriging" - julia -e 'using Pkg - Pkg.add(url="https://github.com/libKriging/JLibKriging.jl") - Pkg.add(["IJulia", "DelimitedFiles", "GaussianProcesses", "Optim"]) - using IJulia; installkernel("Julia"; specname="julia-1.11") - using JLibKriging' - - name: Execute the notebooks - run: python tools/run_notebooks.py --inplace --timeout 3600 --kernel julia-1.11 notebooks/julia/*.ipynb + tools/install/julia.sh libKriging + - name: Execute and save the notebook + run: python tools/run_notebooks.py --inplace --timeout 3600 --kernel julia-${{ env.JULIA_VERSION }} ${{ matrix.notebook }}.ipynb - if: always() uses: actions/upload-artifact@v4 with: - name: results-master-julia + name: executed-${{ matrix.channel }}-${{ matrix.notebook }} path: | + ${{ matrix.notebook }}.ipynb results/ - notebooks/julia/*.ipynb + retention-days: 90 - # --------------------------------------------------------------------------- + # --------------------------------------------------------------------------- report report: needs: [setup, python, r, octave, julia] if: always() && needs.setup.result == 'success' @@ -351,42 +420,44 @@ jobs: channel: [master, release] name: report ${{ matrix.channel }} steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - name: Channel enabled? + id: on + run: echo 'enabled=${{ contains(needs.setup.outputs.channels, matrix.channel) }}' >> "$GITHUB_OUTPUT" + - if: steps.on.outputs.enabled == 'true' + uses: actions/checkout@v4 + - if: steps.on.outputs.enabled == 'true' + uses: actions/setup-python@v5 with: python-version: ${{ env.PYTHON_VERSION }} - - if: contains(needs.setup.outputs.channels, matrix.channel) + - if: steps.on.outputs.enabled == 'true' uses: actions/download-artifact@v4 with: - pattern: results-${{ matrix.channel }}-* + pattern: executed-${{ matrix.channel }}-* path: artifacts - name: Report - if: contains(needs.setup.outputs.channels, matrix.channel) + if: steps.on.outputs.enabled == 'true' run: | pip install numpy pandas mkdir -p results executed find artifacts -path '*/results/*' -type f -exec cp {} results/ \; - for d in artifacts/*/; do if [ -d "$d/notebooks" ]; then cp -r "$d/notebooks/." executed/; fi; done - expect="py00_test_functions,py01_kernels_trends,py02_objectives_gradients,py03_noise,py04_update_simulate" - expect="$expect,py05_warpkriging,py06_high_dimension_mlp,py07_scalability,py08_api_persistence" - expect="$expect,py09_robustness,py10_compare_python_packages,py11_cross_binding" - expect="$expect,r01_cross_binding,r02_vs_dicekriging_robustgasp,oct01_cross_binding,oct02_vs_stk" - if [ "${{ matrix.channel }}" = master ]; then expect="$expect,jl01_cross_binding"; fi + find artifacts -maxdepth 2 -name '*.ipynb' -exec cp {} executed/ \; + expect=python,R,octave + if [ "${{ matrix.channel }}" = master ]; then expect="$expect,julia"; fi echo "## Channel: ${{ matrix.channel }}" >> "$GITHUB_STEP_SUMMARY" python tools/report.py --results results --expect "$expect" \ - --baseline baseline/metrics-${{ matrix.channel }}.csv \ + --baseline tools/baseline/metrics-${{ matrix.channel }}.csv \ --write-baseline new-baseline/metrics-${{ matrix.channel }}.csv \ --summary "$GITHUB_STEP_SUMMARY" > report.md - - if: always() && contains(needs.setup.outputs.channels, matrix.channel) + - if: always() && steps.on.outputs.enabled == 'true' uses: actions/upload-artifact@v4 with: name: report-${{ matrix.channel }} path: | report.md results/ - executed/ new-baseline/ - - if: always() && contains(needs.setup.outputs.channels, matrix.channel) + retention-days: 90 + - if: always() && steps.on.outputs.enabled == 'true' uses: actions/upload-artifact@v4 with: name: executed-notebooks-${{ matrix.channel }} diff --git a/README.md b/README.md index 39d4b2f..f578f01 100644 --- a/README.md +++ b/README.md @@ -3,95 +3,132 @@ Test and benchmark notebooks for [libKriging](https://github.com/libKriging/libKriging), kept outside the library so that they can follow both its development head and its releases. Every notebook **checks** what it computes (error-level checks fail -the run, warn-level ones are reported), and CI executes all of them and -aggregates the findings of every language into one report. - -## Executed notebooks (libKriging master) - -CI executes every notebook against the **development head of libKriging and of -its bindings** (master channel, see [CI](#ci)) and saves it with its outputs, -failing checks included (a notebook that fails is kept up to its failing cell): - -* **[latest executed notebooks (zip)](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-notebooks-master.zip)**: - the `executed-notebooks-master` artifact of the last run on `main` - (`python/`, `r/`, `octave/`, `julia/`); -* **[latest report, results and notebooks (zip)](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/report-master.zip)**: - the `report-master` artifact (aggregated `report.md`, all CSV results, executed - notebooks, refreshed baseline); -* **[all runs on `main`](https://github.com/libKriging/benchmarks/actions/workflows/notebooks.yml?query=branch%3Amain)**: - each run keeps these artifacts for 90 days, and its job summary shows the report. - -The same artifacts exist for the release channel (`executed-notebooks-release`, -`report-release`). - -## What is covered - -| notebook | problems (test functions) | libKriging features | compared with | -|---|---|---|---| -| [python/00](notebooks/python/00_test_functions.ipynb) | the 21 test functions themselves, against published optima / ranges | — | — | -| [python/01](notebooks/python/01_kriging_kernels_trends.ipynb) | Forrester, Gramacy & Lee 2012, Branin, six-hump camel, log-Goldstein-Price, Lim, Ishigami, Hartmann 3 | 4 kernels × 4 trends, interpolation, `predict` (cov, derivatives) | finite differences | -| [python/02](notebooks/python/02_objectives_gradients.ipynb) | Branin, Hartmann 3, Ishigami, Friedman | `LL` / `LOO` / `LMP` objectives and gradients, optimality of θ̂, `leaveOneOutVec`, multistart | finite differences, explicit refits | -| [python/03](notebooks/python/03_noise.ipynb) | Forrester, Friedman, Gramacy & Lee 2012, Branin (replicates) | `noise="nugget"`, heteroscedastic noise vector | true noise, replicate averaging | -| [python/04](notebooks/python/04_update_simulate.ipynb) | Branin, Hartmann 3, Hartmann 6 | `update`, `simulate`, `update_simulate` | fits from scratch, moments | -| [python/05](notebooks/python/05_warpkriging.ipynb) | Gramacy & Lee 2012 / 2008, mixed categorical Branin, ordinal Hartmann 3 | `WarpKriging`: affine, boxcox, kumaraswamy, knots, neural_mono, categorical, ordinal | plain `Kriging` | -| [python/06](notebooks/python/06_high_dimension_mlp.ipynb) | OTL circuit, piston, borehole, robot arm, Dette-Pepelyshev, wing weight, Welch (20D), projected Branin (10D) | `normalize`, screening, `MLPKriging` | plain `Kriging` | -| [python/07](notebooks/python/07_scalability.ipynb) | Hartmann 3, OTL circuit (n up to 5000), Branin | `method="vecchia(m)"`, `"nystrom(k)"`, `NestedKriging` (5 aggregations), `subsetOfData` | exact fit | -| [python/08](notebooks/python/08_api_persistence.ipynb) | Hartmann 3 | `save`/`load`, pickle, copy, accessors, scikit-learn estimators | — | -| [python/09](notebooks/python/09_robustness.ipynb) | Branin | invalid inputs, duplicates, constant y, ill-conditioning, scale equivariance, extrapolation, cost in q | — | -| [python/10](notebooks/python/10_compare_python_packages.ipynb) | 10 functions, 2D to 10D | default fits | scikit-learn, SMT, OpenTURNS, GPy (exact equality at fixed hyperparameters + accuracy/cost) | -| [python/11](notebooks/python/11_cross_binding.ipynb), [r/01](notebooks/r/01_cross_binding.ipynb), [octave/01](notebooks/octave/01_cross_binding.ipynb), [julia/01](notebooks/julia/01_cross_binding.ipynb) | shared datasets of `data/` | the same protocol in every binding | each other (report) | -| [r/02](notebooks/r/02_vs_dicekriging_robustgasp.ipynb) | Branin, Hartmann 3, Borehole | `rlibkriging` | DiceKriging (exact equality), RobustGaSP | -| [octave/02](notebooks/octave/02_vs_stk.ipynb) | Branin, Hartmann 3, Borehole | `mLibKriging` | STK (exact equality, Gaussian kernel) | -| [julia/01](notebooks/julia/01_cross_binding.ipynb) | Branin, Hartmann 3, Borehole | `JLibKriging` | GaussianProcesses.jl | - -Test functions: [`lkbench/functions.py`](lkbench/functions.py), after the +the run, warn-level ones are reported), and CI executes each of them against the +**development head of libKriging and of its bindings**, saves it with its outputs, +and aggregates the findings of every language into one report. + +The notebooks are at the root of the repository, named +`__.ipynb`; everything else (notebook sources, +helpers, binding installation, CI scripts, known issues, baselines) is under +[`tools/`](tools), and the shared datasets under [`data/`](data). + +## Notebooks + +The **executed** column links to the notebook as executed by the last CI run on +`main` against libKriging master (outputs saved, failing checks included: a +notebook that fails is kept up to its failing cell), a zip of the +`executed-master-` artifact (through [nightly.link](https://nightly.link), +which serves the last artifact of a branch at a stable URL). Every run keeps +these artifacts for 90 days: [all runs on `main`](https://github.com/libKriging/benchmarks/actions/workflows/notebooks.yml?query=branch%3Amain). +All the executed notebooks of a run, in one zip: +[executed-notebooks-master](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-notebooks-master.zip); +the aggregated report with all CSV results: +[report-master](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/report-master.zip). + +### Python (`pylibkriging`) + +| notebook | kriging type | theme: problems, features | compared with | executed | +|---|---|---|---|---| +| [python_no-model_test-functions-reference-values](python_no-model_test-functions-reference-values.ipynb) | — | the 21 test functions of [`tools/lkbench/functions.py`](tools/lkbench/functions.py) against published optima / ranges | — | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-python_no-model_test-functions-reference-values.zip) | +| [python_Kriging_kernels-trends-interpolation](python_Kriging_kernels-trends-interpolation.ipynb) | `Kriging` | 4 kernels × 4 trends on Forrester, Gramacy & Lee, Branin, six-hump camel, log-Goldstein-Price, Lim, Ishigami, Hartmann 3: interpolation, `predict` (covariance, derivatives) | finite differences | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-python_Kriging_kernels-trends-interpolation.zip) | +| [python_Kriging_objectives-LL-LOO-LMP-gradients](python_Kriging_objectives-LL-LOO-LMP-gradients.ipynb) | `Kriging` | `LL` / `LOO` / `LMP` objectives and gradients, optimality of θ̂, `leaveOneOutVec`, multistart (Branin, Hartmann 3, Ishigami, Friedman) | finite differences, explicit refits | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-python_Kriging_objectives-LL-LOO-LMP-gradients.zip) | +| [python_Kriging-nugget-noise_noisy-data](python_Kriging-nugget-noise_noisy-data.ipynb) | `Kriging` with `noise="nugget"` or variances | estimated nugget, heteroscedastic noise, calibration (Forrester, Friedman, Gramacy & Lee, Branin with replicates) | true noise, replicate averaging | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-python_Kriging-nugget-noise_noisy-data.zip) | +| [python_Kriging_update-simulate](python_Kriging_update-simulate.ipynb) | `Kriging` | `update`, `simulate`, `update_simulate` (Branin, Hartmann 3, Hartmann 6) | fits from scratch, moments | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-python_Kriging_update-simulate.zip) | +| [python_WarpKriging_warpings-categorical-ordinal](python_WarpKriging_warpings-categorical-ordinal.ipynb) | `WarpKriging` | affine, boxcox, kumaraswamy, knots, neural_mono, categorical, ordinal warpings (Gramacy & Lee 2012 / 2008, mixed categorical Branin, ordinal Hartmann 3) | plain `Kriging` | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-python_WarpKriging_warpings-categorical-ordinal.zip) | +| [python_Kriging_high-dimension-normalize-screening](python_Kriging_high-dimension-normalize-screening.ipynb) | `Kriging` | `normalize`, screening, 6D to 20D engineering functions (OTL circuit, piston, borehole, robot arm, Dette-Pepelyshev, wing weight, Welch) | unit-cube fit | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-python_Kriging_high-dimension-normalize-screening.zip) | +| [python_MLPKriging_deep-kernel-projected-branin](python_MLPKriging_deep-kernel-projected-branin.ipynb) | `MLPKriging` | deep kernel on a 10D function with a 2D structure, activations, multistart | plain `Kriging` | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-python_MLPKriging_deep-kernel-projected-branin.zip) | +| [python_Kriging-vecchia-nystrom_large-n](python_Kriging-vecchia-nystrom_large-n.ipynb) | `Kriging` with `method="vecchia(m)"` / `"nystrom(k)"` | large n (Hartmann 3, OTL circuit, n up to 5000), limit cases, `subsetOfData` | exact fit | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-python_Kriging-vecchia-nystrom_large-n.zip) | +| [python_NestedKriging_aggregations-large-n](python_NestedKriging_aggregations-large-n.ipynb) | `NestedKriging` | `NK`, `PoE`, `gPoE`, `BCM`, `rBCM` aggregations, large n | exact fit | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-python_NestedKriging_aggregations-large-n.zip) | +| [python_all-classes_save-load-pickle-sklearn](python_all-classes_save-load-pickle-sklearn.ipynb) | all classes | `save`/`load`, pickle, copy, accessors, scikit-learn estimators (Hartmann 3) | — | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-python_all-classes_save-load-pickle-sklearn.zip) | +| [python_Kriging_robustness-invalid-inputs](python_Kriging_robustness-invalid-inputs.ipynb) | `Kriging` | invalid inputs, duplicates, constant y, ill-conditioning, scale equivariance, extrapolation, cost in q (Branin) | — | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-python_Kriging_robustness-invalid-inputs.zip) | +| [python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy](python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy.ipynb) | `Kriging` | default fits on 10 functions, 2D to 10D | scikit-learn, SMT, OpenTURNS, GPy (exact equality at fixed hyperparameters + accuracy/cost) | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy.zip) | +| [python_Kriging_cross-binding-reference](python_Kriging_cross-binding-reference.ipynb) | `Kriging` | the cross-binding protocol on the datasets of `data/`: the Python reference | the other bindings (report) | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-python_Kriging_cross-binding-reference.zip) | + +### R (`rlibkriging`) + +| notebook | kriging type | theme: problems, features | compared with | executed | +|---|---|---|---|---| +| [R_all-classes_api-tour](R_all-classes_api-tour.ipynb) | all classes | `Kriging` (noise-free, nugget, known noise, Vecchia, Nyström), `WarpKriging` (continuous, categorical), `MLPKriging`, `NestedKriging`, `update`, `simulate`, `save`/`load` (Branin) | — | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-R_all-classes_api-tour.zip) | +| [R_Kriging_cross-binding](R_Kriging_cross-binding.ipynb) | `Kriging` | the cross-binding protocol, R side | Python (report) | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-R_Kriging_cross-binding.zip) | +| [R_Kriging_vs-DiceKriging-RobustGaSP](R_Kriging_vs-DiceKriging-RobustGaSP.ipynb) | `Kriging` | default fits (Branin, Hartmann 3, Borehole) | DiceKriging (exact equality), RobustGaSP | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-R_Kriging_vs-DiceKriging-RobustGaSP.zip) | + +### Octave (`mLibKriging`) + +| notebook | kriging type | theme: problems, features | compared with | executed | +|---|---|---|---|---| +| [octave_all-classes_api-tour](octave_all-classes_api-tour.ipynb) | all classes | the R tour, with the positional arguments of the Octave binding | — | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-octave_all-classes_api-tour.zip) | +| [octave_Kriging_cross-binding](octave_Kriging_cross-binding.ipynb) | `Kriging` | the cross-binding protocol, Octave side | Python (report) | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-octave_Kriging_cross-binding.zip) | +| [octave_Kriging_vs-STK](octave_Kriging_vs-STK.ipynb) | `Kriging` | default fits (Branin, Hartmann 3, Borehole) | STK (exact equality, Gaussian kernel) | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-octave_Kriging_vs-STK.zip) | + +### Julia (`JLibKriging`) + +| notebook | kriging type | theme: problems, features | compared with | executed | +|---|---|---|---|---| +| [julia_all-classes_api-tour](julia_all-classes_api-tour.ipynb) | all classes | the R tour, with the Julia binding | — | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-julia_all-classes_api-tour.zip) | +| [julia_Kriging_cross-binding](julia_Kriging_cross-binding.ipynb) | `Kriging` | the cross-binding protocol, Julia side | Python (report) | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-julia_Kriging_cross-binding.zip) | +| [julia_Kriging_vs-GaussianProcesses](julia_Kriging_vs-GaussianProcesses.ipynb) | `Kriging` | default fits (Branin, Hartmann 3, Borehole) | GaussianProcesses.jl | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-julia_Kriging_vs-GaussianProcesses.zip) | + +A notebook that compares libKriging with another package installs that package +itself, in its first cells (pip, CRAN, `Pkg.add`, or a clone of STK); a package +that cannot be installed is reported (warn-level) and its comparison skipped. + +Test functions: [`tools/lkbench/functions.py`](tools/lkbench/functions.py), after the [Virtual Library of Simulation Experiments](https://www.sfu.ca/~ssurjano/) (Surjanovic & Bingham). ## Checks and statuses Each notebook records its checks with `lkbench.Suite` (Python) or the small -equivalents in `notebooks/{r,octave,julia}/` and writes -`results/.checks.csv` and `results/.metrics.csv`. +equivalents of `tools/r`, `tools/octave` and `tools/julia`, under a suite named +after the notebook, and writes `results/.checks.csv` and +`results/.metrics.csv`. | status | meaning | fails CI | |---|---|---| | PASS | | | | FAIL | an error-level check failed: a correctness property (gradient vs finite differences, `update` vs fit from scratch, equality with another package at fixed hyperparameters, ...) or an accuracy floor | yes | | WARN | a warn-level check failed: performance comparison, optimizer behaviour, judgement thresholds | no | -| KNOWN | an error-level failure listed in [`known_issues.csv`](known_issues.csv), with the issue | no | +| KNOWN | a failing check (of either level) listed in [`tools/known_issues.csv`](tools/known_issues.csv), with the issue | no | | SKIP | feature absent from the installed version (release channel) | no | `tools/report.py` aggregates all results: status per notebook, failures, warnings, known issues still present, known issues that now pass (to be removed from the list), cross-binding consistency against Python, and metric drifts -against [`baseline/`](baseline). +against [`tools/baseline/`](tools/baseline). ## Running locally +From the repository root (the notebooks' paths are relative to it): + ```sh -pip install -e ".[notebooks,compare]" jinja2 pylibkriging # or a pylibkriging built from source -python tools/run_notebooks.py notebooks/python/*.ipynb -python tools/report.py +git clone --recurse-submodules https://github.com/libKriging/libKriging ../libKriging # for a master build +tools/install/python.sh master ../libKriging # or: tools/install/python.sh release +tools/install/r.sh master ~/Rlib ../libKriging # then R_LIBS_USER=~/Rlib +tools/install/octave.sh ../libKriging ~/mlib # then OCTAVE_PATH=~/mlib +tools/install/julia.sh ../libKriging +python tools/run_notebooks.py --inplace python_Kriging_update-simulate.ipynb +python tools/report.py --expect python ``` -R needs `rlibkriging`, `DiceKriging`, `RobustGaSP` and `IRkernel`; Octave needs -mLibKriging on `OCTAVE_PATH`, STK on `STK_PATH` and `octave_kernel`; Julia needs -`JLibKriging`, `IJulia`, `DelimitedFiles` (and optionally `GaussianProcesses`, -`Optim`). See the CI workflow for exact commands. +The scripts of [`tools/install/`](tools/install) are the ones CI runs; each says +what it needs (BLAS/LAPACK headers, Octave, R, Julia). ## Editing notebooks -Notebooks are generated from plain-text sources (`notebooks//src/`, cells +Notebooks are generated from plain-text sources ([`tools/src/`](tools/src), cells separated by `# %%` / `# %% [markdown]`, or `% %%` in Octave), so that reviews are plain diffs; outputs are not committed (CI archives the executed notebooks as -artifacts, see [Executed notebooks](#executed-notebooks-libkriging-master)). +artifacts, see [Notebooks](#notebooks)). ```sh -python tools/py2nb.py notebooks/*/src/* # regenerate -python tools/py2nb.py --check notebooks/*/src/* # CI: sources and notebooks in sync +python tools/py2nb.py # regenerate every notebook from tools/src/ +python tools/py2nb.py --check # CI: sources and notebooks in sync, no notebook without source ``` +A new notebook is a new source in `tools/src/` named `__`, +with its suite named the same, and a line in the tables above (CI checks that +every notebook is linked from this README); CI picks it up automatically. + Shared datasets (`data/`, used by the cross-binding notebooks of every language) are written once by `tools/make_data.py` and committed: the files are the reference. @@ -103,18 +140,18 @@ pull request, weekly, on demand (`libkriging_ref`, `libkriging_repo`, `channels` inputs) and on a `repository_dispatch` of type `libkriging-updated` (sent by libKriging after a merge, with the commit in `client_payload.ref`): -* **master channel**: libKriging built from source at the requested ref, in - every language (Python wheel, Octave mex, JLibKriging.jl against the same - checkout; R from the `libKriging/rlibkriging` repository, which follows - libKriging master); -* **release channel**: pylibkriging from PyPI, rlibkriging from CRAN, - mLibKriging built from the last release tag (the published binaries target - another Octave version than Ubuntu's). - -One job per Python notebook, one per language for the others, then one report -per channel (job summary + `report-` artifact with the report, all CSV -results, the executed notebooks and a refreshed baseline, + an -`executed-notebooks-` artifact with the executed notebooks only; see -[Executed notebooks](#executed-notebooks-libkriging-master)). The initial `baseline/` files come -from a local run (Python notebooks only, 4-core container); to refresh them, -commit the `new-baseline/` files of a trusted CI run. +* **master channel** (every event): libKriging built from source at the + requested ref, with every binding from the same checkout: Python wheel, R + package (`bindings/R`), Octave mex, JLibKriging.jl; +* **release channel** (weekly, or on demand): pylibkriging from PyPI, + rlibkriging from CRAN, mLibKriging built from the last release tag (the + published binaries target another Octave version than Ubuntu's); no Julia. + +One build job per binding and channel, then **one job per notebook**, each +uploading its executed notebook and its results as the artifact +`executed--`; then one report per channel (job summary, +`report-` artifact with the report, all CSV results and a refreshed +baseline, and `executed-notebooks-` with all the executed notebooks). +The initial `tools/baseline/` files come from a local run (Python notebooks only, +4-core container); to refresh them, commit the `new-baseline/` files of a +trusted CI run. diff --git a/notebooks/r/01_cross_binding.ipynb b/R_Kriging_cross-binding.ipynb similarity index 92% rename from notebooks/r/01_cross_binding.ipynb rename to R_Kriging_cross-binding.ipynb index d3c97ed..93c1073 100644 --- a/notebooks/r/01_cross_binding.ipynb +++ b/R_Kriging_cross-binding.ipynb @@ -5,9 +5,9 @@ "id": "c000", "metadata": {}, "source": [ - "# R 01 — Cross-binding consistency: the R side\n", + "# R · Kriging — cross-binding consistency: the R side\n", "\n", - "Same protocol as `notebooks/python/11_cross_binding.ipynb` (see there), on the\n", + "Same protocol as `python_Kriging_cross-binding-reference.ipynb` (see there), on the\n", "shared datasets of `data/`; writes `results/xbind_R.csv`, compared with the\n", "Python values by `tools/report.py`." ] @@ -20,8 +20,8 @@ "outputs": [], "source": [ "suppressPackageStartupMessages(library(rlibkriging))\n", - "source(\"checks.R\")\n", - "s <- new_suite(\"r01_cross_binding\")\n", + "source(\"tools/r/checks.R\")\n", + "s <- new_suite(\"R_Kriging_cross-binding\")\n", "cat(\"rlibkriging\", as.character(packageVersion(\"rlibkriging\")), \"\\n\")\n", "rows <- list()\n", "put <- function(dataset, quantity, values) {\n", diff --git a/notebooks/r/02_vs_dicekriging_robustgasp.ipynb b/R_Kriging_vs-DiceKriging-RobustGaSP.ipynb similarity index 84% rename from notebooks/r/02_vs_dicekriging_robustgasp.ipynb rename to R_Kriging_vs-DiceKriging-RobustGaSP.ipynb index 2bcfec0..e48cbac 100644 --- a/notebooks/r/02_vs_dicekriging_robustgasp.ipynb +++ b/R_Kriging_vs-DiceKriging-RobustGaSP.ipynb @@ -5,7 +5,7 @@ "id": "c000", "metadata": {}, "source": [ - "# R 02 — rlibkriging vs DiceKriging and RobustGaSP\n", + "# R · Kriging vs DiceKriging and RobustGaSP\n", "\n", "DiceKriging is the reference implementation libKriging's `Kriging` follows\n", "(same product-form kernels and range parametrization, same concentrated\n", @@ -20,10 +20,40 @@ " (warn-level comparisons)." ] }, + { + "cell_type": "markdown", + "id": "c001", + "metadata": {}, + "source": [ + "## Installing DiceKriging and RobustGaSP\n", + "\n", + "From CRAN (the configured repository, else cloud.r-project.org), into the\n", + "first library of `.libPaths()`, unless already installed. DiceKriging is\n", + "required; without RobustGaSP its comparison is skipped (warn-level)." + ] + }, { "cell_type": "code", "execution_count": null, - "id": "c001", + "id": "c002", + "metadata": {}, + "outputs": [], + "source": [ + "repos <- getOption(\"repos\")\n", + "if (is.null(repos) || any(repos == \"@CRAN@\")) repos <- c(CRAN = \"https://cloud.r-project.org\")\n", + "for (p in c(\"DiceKriging\", \"RobustGaSP\")) {\n", + " if (!requireNamespace(p, quietly = TRUE)) {\n", + " message(\"installing \", p)\n", + " tryCatch(install.packages(p, repos = repos, lib = .libPaths()[1]),\n", + " error = function(e) message(p, \" could not be installed: \", conditionMessage(e)))\n", + " }\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c003", "metadata": {}, "outputs": [], "source": [ @@ -32,8 +62,8 @@ " library(DiceKriging)\n", "})\n", "has_rgasp <- requireNamespace(\"RobustGaSP\", quietly = TRUE)\n", - "source(\"checks.R\")\n", - "s <- new_suite(\"r02_vs_dicekriging_robustgasp\")\n", + "source(\"tools/r/checks.R\")\n", + "s <- new_suite(\"R_Kriging_vs-DiceKriging-RobustGaSP\")\n", "cat(\"rlibkriging\", as.character(packageVersion(\"rlibkriging\")), \"| DiceKriging\", as.character(packageVersion(\"DiceKriging\")),\n", " \"| RobustGaSP\", if (has_rgasp) as.character(packageVersion(\"RobustGaSP\")) else \"absent\", \"\\n\")\n", "DATASETS <- c(\"branin\", \"hartmann3\", \"borehole\")" @@ -41,7 +71,7 @@ }, { "cell_type": "markdown", - "id": "c002", + "id": "c004", "metadata": {}, "source": [ "## 1. Same hyperparameters, same numbers" @@ -50,7 +80,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c003", + "id": "c005", "metadata": {}, "outputs": [], "source": [ @@ -78,7 +108,7 @@ }, { "cell_type": "markdown", - "id": "c004", + "id": "c006", "metadata": {}, "source": [ "## 2. Default fits" @@ -87,7 +117,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c005", + "id": "c007", "metadata": {}, "outputs": [], "source": [ @@ -126,7 +156,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c006", + "id": "c008", "metadata": {}, "outputs": [], "source": [ @@ -141,7 +171,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c007", + "id": "c009", "metadata": {}, "outputs": [], "source": [ diff --git a/R_all-classes_api-tour.ipynb b/R_all-classes_api-tour.ipynb new file mode 100644 index 0000000..507404f --- /dev/null +++ b/R_all-classes_api-tour.ipynb @@ -0,0 +1,320 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "c000", + "metadata": {}, + "source": [ + "# R · all classes — every libKriging model class and fit method through rlibkriging\n", + "\n", + "One problem (Branin on [0,1]², Q² on 500 test points), every class and every fit\n", + "method of the R binding, each with the checks that hold for it whatever the\n", + "data:\n", + "\n", + "| model | checks (error-level unless stated) |\n", + "|---|---|\n", + "| `Kriging` (noise-free) | interpolation, Q² floor, conditional simulations through the data, `update` = fit on all points (same θ), `save`/`load` |\n", + "| `Kriging(noise = \"nugget\")` | nugget > 0, smoothing next to the design points, single-start optimum as good as the multistart one (warn), `save`/`load` |\n", + "| `Kriging(noise = )` | the stored variances, finite predictions |\n", + "| `Kriging(method = \"vecchia(m)\")`, `\"nystrom(k)\"` | `k$method()`, `predict(method = ...)` finite, Q² floor (warn) |\n", + "| `WarpKriging` (continuous and categorical warpings) | finite predictions, Q² floor (warn), `save`/`load` |\n", + "| `MLPKriging` | finite predictions, Q² floor (warn), default optimizer as good as a multistart BFGS (warn), `save`/`load` |\n", + "| `NestedKriging` (`NK`, `PoE`) | `NK` interpolates, stdev ≥ 0, `save`/`load` |\n", + "\n", + "A class missing from the installed rlibkriging (release channel) is recorded as\n", + "SKIP." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c001", + "metadata": {}, + "outputs": [], + "source": [ + "suppressPackageStartupMessages(library(rlibkriging))\n", + "source(\"tools/r/checks.R\")\n", + "s <- new_suite(\"R_all-classes_api-tour\")\n", + "cat(\"rlibkriging\", as.character(packageVersion(\"rlibkriging\")), \"\\n\")\n", + "\n", + "branin <- function(U) { # on [0,1]^2\n", + " x1 <- 15 * U[, 1] - 5; x2 <- 15 * U[, 2]\n", + " (x2 - 5.1 / (4 * pi^2) * x1^2 + 5 / pi * x1 - 6)^2 + 10 * (1 - 1 / (8 * pi)) * cos(x1) + 10\n", + "}\n", + "set.seed(1)\n", + "lhs <- function(n, d) sapply(1:d, function(j) (sample(n) - runif(n)) / n)\n", + "X <- lhs(30, 2); y <- branin(X)\n", + "Xt <- matrix(runif(1000), ncol = 2); yt <- branin(Xt)\n", + "tmp <- tempdir()\n", + "has_class <- function(name) exists(name, mode = \"function\")\n", + "skip <- function(s, check, reason) {\n", + " s$checks[[length(s$checks) + 1]] <- data.frame(suite = s$name, lang = \"R\", check = check, severity = \"info\",\n", + " status = \"SKIP\", value = \"\", expected = \"\", detail = reason, stringsAsFactors = FALSE)\n", + " message(\"[SKIP] \", check, \": \", reason)\n", + "}\n", + "roundtrip_same_mean <- function(s, label, k, Xn) {\n", + " f <- file.path(tmp, paste0(gsub(\"[^A-Za-z0-9]\", \"_\", label), \".json\"))\n", + " check_runs(s, paste0(label, \": save + load\"), {\n", + " save(k, f)\n", + " k2 <- rlibkriging::load(f)\n", + " check_close(s, paste0(label, \": load(save(k)) predicts the same mean\"),\n", + " predict(k2, Xn, return_stdev = FALSE)$mean, predict(k, Xn, return_stdev = FALSE)$mean, rtol = 1e-10)\n", + " })\n", + "}" + ] + }, + { + "cell_type": "markdown", + "id": "c002", + "metadata": {}, + "source": [ + "## `Kriging`, noise-free" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c003", + "metadata": {}, + "outputs": [], + "source": [ + "k <- Kriging(y, X, \"matern5_2\")\n", + "print(k)\n", + "p <- predict(k, Xt, return_stdev = TRUE)\n", + "add_metric(s, \"branin\", \"Kriging\", q2 = q2(yt, p$mean), rmse = rmse(yt, p$mean))\n", + "check_greater(s, \"Kriging: Q2\", q2(yt, p$mean), 0.95)\n", + "check_less(s, \"Kriging: interpolates (max |m(X) - y| / sd(y))\",\n", + " max(abs(predict(k, X, return_stdev = FALSE)$mean - y)) / sd(y), 1e-6)\n", + "sims <- simulate(k, nsim = 5, seed = 2, x = X)\n", + "check_less(s, \"Kriging: conditional simulations go through the data (max |sim(X) - y| / sigma)\",\n", + " max(abs(sims - y)) / sqrt(k$sigma2()), 1e-4)\n", + "check_greater(s, \"Kriging: stdev > 0 off the design\", min(p$stdev), 0)\n", + "# update with 5 new points (no refit) = a fit on the 35 points at the same theta\n", + "Xu <- lhs(5, 2); yu <- branin(Xu)\n", + "ku <- copy(k)\n", + "update(ku, yu, Xu, refit = FALSE)\n", + "k35 <- Kriging(c(y, yu), rbind(X, Xu), \"matern5_2\", optim = \"none\",\n", + " parameters = list(theta = matrix(k$theta(), nrow = 1), is_theta_estim = FALSE,\n", + " sigma2 = k$sigma2(), is_sigma2_estim = FALSE))\n", + "check_close(s, \"Kriging: update(refit = FALSE) == fit on all points (same theta, sigma2)\",\n", + " predict(ku, Xt, return_stdev = FALSE)$mean, predict(k35, Xt, return_stdev = FALSE)$mean,\n", + " rtol = 1e-6, atol = 1e-8 * sd(y))\n", + "roundtrip_same_mean(s, \"Kriging\", k, Xt)" + ] + }, + { + "cell_type": "markdown", + "id": "c004", + "metadata": {}, + "source": [ + "## `Kriging` with noise: estimated nugget, known variances\n", + "\n", + "Noise of standard deviation 5% of sd(y). The nugget model is fitted with 10\n", + "restarts (`optim = \"BFGS10\"`); the default single start is compared with it\n", + "(warn-level: a lower likelihood means a local optimum). As DiceKriging's\n", + "nugget, the predictor returns y exactly *at* the design points (the nugget is\n", + "a micro-scale variation): smoothing is checked next to them." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c005", + "metadata": {}, + "outputs": [], + "source": [ + "sd_noise <- 0.05 * sd(y)\n", + "yn <- y + rnorm(length(y), sd = sd_noise)\n", + "kn <- Kriging(yn, X, \"matern5_2\", noise = \"nugget\", optim = \"BFGS10\")\n", + "pn <- predict(kn, Xt, return_stdev = TRUE)\n", + "add_metric(s, \"branin noisy\", \"Kriging nugget BFGS10\", q2 = q2(yt, pn$mean), nugget = kn$nugget(),\n", + " ll = logLikelihood(kn))\n", + "check_greater(s, \"Kriging(nugget): estimated nugget > 0\", kn$nugget(), 0)\n", + "check_greater(s, \"Kriging(nugget): Q2\", q2(yt, pn$mean), 0.95)\n", + "near <- pmin(X + 1e-7, 1)\n", + "check_greater(s, \"Kriging(nugget): smooths the noisy data next to the design points (rms residual / noise sd)\",\n", + " rmse(yn, predict(kn, near, return_stdev = FALSE)$mean) / sd_noise, 0.3)\n", + "kn1 <- Kriging(yn, X, \"matern5_2\", noise = \"nugget\")\n", + "add_metric(s, \"branin noisy\", \"Kriging nugget BFGS\", q2 = q2(yt, predict(kn1, Xt, return_stdev = FALSE)$mean),\n", + " nugget = kn1$nugget(), ll = logLikelihood(kn1))\n", + "check_greater(s, \"Kriging(nugget): LL(single start) - LL(10 restarts)\", logLikelihood(kn1) - logLikelihood(kn), -1,\n", + " severity = \"warn\")\n", + "roundtrip_same_mean(s, \"Kriging(nugget)\", kn, Xt)\n", + "kh <- Kriging(yn, X, \"matern5_2\", noise = rep(sd_noise^2, length(yn)))\n", + "check_close(s, \"Kriging(noise = variances): noise() returns them\", kh$noise(), rep(sd_noise^2, length(yn)),\n", + " rtol = 1e-12)\n", + "ph <- predict(kh, Xt, return_stdev = TRUE)\n", + "check_less(s, \"Kriging(noise = variances): non-finite predictions\", sum(!is.finite(c(ph$mean, ph$stdev))), 1)\n", + "add_metric(s, \"branin noisy\", \"Kriging known noise\", q2 = q2(yt, ph$mean))" + ] + }, + { + "cell_type": "markdown", + "id": "c006", + "metadata": {}, + "source": [ + "## `Kriging` with an approximate fit: Vecchia, Nyström (n = 400)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c007", + "metadata": {}, + "outputs": [], + "source": [ + "Xl <- lhs(400, 2); yl <- branin(Xl)\n", + "for (m in c(\"vecchia(20)\", \"nystrom(60)\")) {\n", + " short <- sub(\"\\\\(.*\", \"\", m)\n", + " check_runs(s, paste0(\"Kriging(method = '\", m, \"'): fit + predict\"), {\n", + " ka <- Kriging(yl, Xl, \"matern5_2\", method = m)\n", + " add_check(s, paste0(\"Kriging(method = '\", m, \"'): method()\"), identical(ka$method(), m), ka$method(), m)\n", + " pa <- predict(ka, Xt, return_stdev = TRUE, method = short)\n", + " check_less(s, paste0(\"Kriging(method = '\", m, \"'): non-finite predictions\"),\n", + " sum(!is.finite(c(pa$mean, pa$stdev))), 1)\n", + " add_metric(s, \"branin n=400\", paste(\"Kriging\", m), q2 = q2(yt, pa$mean))\n", + " check_greater(s, paste0(\"Kriging(method = '\", m, \"'): Q2\"), q2(yt, pa$mean), 0.95, severity = \"warn\")\n", + " })\n", + "}" + ] + }, + { + "cell_type": "markdown", + "id": "c008", + "metadata": {}, + "source": [ + "## `WarpKriging`: continuous warpings, then a categorical input\n", + "\n", + "The categorical case: x₂ restricted to 5 levels (coded 0, …, 4), embedded in ℝ²." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c009", + "metadata": {}, + "outputs": [], + "source": [ + "if (has_class(\"WarpKriging\")) {\n", + " check_runs(s, \"WarpKriging(kumaraswamy, kumaraswamy): fit + predict\", {\n", + " kw <- WarpKriging(y, X, warping = c(\"kumaraswamy\", \"kumaraswamy\"), kernel = \"matern5_2\")\n", + " pw <- predict(kw, Xt, return_stdev = TRUE)\n", + " check_less(s, \"WarpKriging(kumaraswamy): non-finite predictions\", sum(!is.finite(c(pw$mean, pw$stdev))), 1)\n", + " add_metric(s, \"branin\", \"WarpKriging kumaraswamy\", q2 = q2(yt, pw$mean))\n", + " check_greater(s, \"WarpKriging(kumaraswamy): Q2\", q2(yt, pw$mean), 0.9, severity = \"warn\")\n", + " roundtrip_same_mean(s, \"WarpKriging(kumaraswamy)\", kw, Xt)\n", + " })\n", + " lev <- function(U) cbind(U[, 1], round(U[, 2] * 4)) # level codes 0..4\n", + " unlev <- function(L) cbind(L[, 1], L[, 2] / 4)\n", + " Xc <- lev(lhs(40, 2)); yc <- branin(unlev(Xc))\n", + " Xtc <- lev(Xt); ytc <- branin(unlev(Xtc))\n", + " check_runs(s, \"WarpKriging(none, categorical(5,2)): fit + predict\", {\n", + " kc <- WarpKriging(yc, Xc, warping = c(\"none\", \"categorical(5,2)\"), kernel = \"matern5_2\")\n", + " pc <- predict(kc, Xtc, return_stdev = TRUE)\n", + " check_less(s, \"WarpKriging(categorical): non-finite predictions\", sum(!is.finite(c(pc$mean, pc$stdev))), 1)\n", + " add_metric(s, \"branin, x2 categorical\", \"WarpKriging categorical\", q2 = q2(ytc, pc$mean))\n", + " check_greater(s, \"WarpKriging(categorical): Q2\", q2(ytc, pc$mean), 0.8, severity = \"warn\")\n", + " roundtrip_same_mean(s, \"WarpKriging(categorical)\", kc, Xtc)\n", + " })\n", + "} else skip(s, \"WarpKriging\", \"not in this rlibkriging\")" + ] + }, + { + "cell_type": "markdown", + "id": "c010", + "metadata": {}, + "source": [ + "## `MLPKriging`: a joint feature map ℝ² → ℝ² before the kernel" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c011", + "metadata": {}, + "outputs": [], + "source": [ + "if (has_class(\"MLPKriging\")) {\n", + " check_runs(s, \"MLPKriging: fit + predict\", {\n", + " km <- MLPKriging(y, X, hidden_dims = c(8), d_out = 2, activation = \"tanh\", kernel = \"matern5_2\", optim = \"BFGS10\")\n", + " pm <- predict(km, Xt, return_stdev = TRUE)\n", + " check_less(s, \"MLPKriging: non-finite predictions\", sum(!is.finite(c(pm$mean, pm$stdev))), 1)\n", + " add_metric(s, \"branin\", \"MLPKriging tanh [8] -> 2, BFGS10\", q2 = q2(yt, pm$mean))\n", + " check_greater(s, \"MLPKriging: Q2\", q2(yt, pm$mean), 0.9, severity = \"warn\")\n", + " roundtrip_same_mean(s, \"MLPKriging\", km, Xt)\n", + " # the default optimizer (\"BFGS+Adam\")\n", + " kd <- MLPKriging(y, X, hidden_dims = c(8), d_out = 2, activation = \"tanh\", kernel = \"matern5_2\")\n", + " qd <- q2(yt, predict(kd, Xt, return_stdev = FALSE)$mean)\n", + " add_metric(s, \"branin\", \"MLPKriging tanh [8] -> 2, default optim\", q2 = qd)\n", + " check_greater(s, \"MLPKriging: Q2(default optim) - Q2(BFGS10)\", qd - q2(yt, pm$mean), -0.05, severity = \"warn\")\n", + " })\n", + "} else skip(s, \"MLPKriging\", \"not in this rlibkriging\")" + ] + }, + { + "cell_type": "markdown", + "id": "c012", + "metadata": {}, + "source": [ + "## `NestedKriging`: 4 groups of 100 points, `NK` and `PoE` aggregations" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c013", + "metadata": {}, + "outputs": [], + "source": [ + "if (has_class(\"NestedKriging\")) {\n", + " for (agg in c(\"NK\", \"PoE\")) {\n", + " check_runs(s, paste0(\"NestedKriging(\", agg, \"): fit + predict\"), {\n", + " kk <- NestedKriging(yl, Xl, kernel = \"matern5_2\", nb_groups = 4, aggregation = agg, seed = 3)\n", + " pk <- predict(kk, Xt, return_stdev = TRUE)\n", + " check_less(s, paste0(\"NestedKriging(\", agg, \"): stdev < 0\"), sum(pk$stdev < 0), 1)\n", + " add_metric(s, \"branin n=400\", paste(\"NestedKriging\", agg), q2 = q2(yt, pk$mean))\n", + " check_greater(s, paste0(\"NestedKriging(\", agg, \"): Q2\"), q2(yt, pk$mean), 0.95, severity = \"warn\")\n", + " if (agg == \"NK\")\n", + " check_less(s, \"NestedKriging(NK): interpolates (max |m(X) - y| / sd(y))\",\n", + " max(abs(predict(kk, Xl, return_stdev = FALSE)$mean - yl)) / sd(yl), 1e-3)\n", + " roundtrip_same_mean(s, paste0(\"NestedKriging(\", agg, \")\"), kk, Xt)\n", + " })\n", + " }\n", + "} else skip(s, \"NestedKriging\", \"not in this rlibkriging\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c014", + "metadata": {}, + "outputs": [], + "source": [ + "me <- do.call(rbind, s$metrics)\n", + "print(me[me$metric == \"q2\", c(\"case\", \"package\", \"value\")], row.names = FALSE)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c015", + "metadata": {}, + "outputs": [], + "source": [ + "finalize(s)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "R", + "language": "R", + "name": "ir" + }, + "language_info": { + "name": "R" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/baseline/metrics-master.csv b/baseline/metrics-master.csv deleted file mode 100644 index 98a29f4..0000000 --- a/baseline/metrics-master.csv +++ /dev/null @@ -1,948 +0,0 @@ -suite,lang,case,package,metric,value -py01_kernels_trends,python,forrester/gauss/constant,pylibkriging,fit_s,0.00161386 -py01_kernels_trends,python,forrester/gauss/constant,pylibkriging,rmse,0.0848271 -py01_kernels_trends,python,forrester/gauss/constant,pylibkriging,q2,0.999634 -py01_kernels_trends,python,forrester/gauss/constant,pylibkriging,nlpd,-3.06486 -py01_kernels_trends,python,forrester/gauss/linear,pylibkriging,fit_s,0.00126195 -py01_kernels_trends,python,forrester/gauss/linear,pylibkriging,rmse,0.0855468 -py01_kernels_trends,python,forrester/gauss/linear,pylibkriging,q2,0.999628 -py01_kernels_trends,python,forrester/gauss/linear,pylibkriging,nlpd,-3.11029 -py01_kernels_trends,python,forrester/gauss/interactive,pylibkriging,fit_s,0.00143647 -py01_kernels_trends,python,forrester/gauss/interactive,pylibkriging,rmse,0.0855468 -py01_kernels_trends,python,forrester/gauss/interactive,pylibkriging,q2,0.999628 -py01_kernels_trends,python,forrester/gauss/interactive,pylibkriging,nlpd,-3.11029 -py01_kernels_trends,python,forrester/gauss/quadratic,pylibkriging,fit_s,0.000566483 -py01_kernels_trends,python,forrester/gauss/quadratic,pylibkriging,rmse,1.57249 -py01_kernels_trends,python,forrester/gauss/quadratic,pylibkriging,q2,0.874394 -py01_kernels_trends,python,forrester/gauss/quadratic,pylibkriging,nlpd,5.33879 -py01_kernels_trends,python,forrester/exp/constant,pylibkriging,fit_s,0.00041604 -py01_kernels_trends,python,forrester/exp/constant,pylibkriging,rmse,0.721582 -py01_kernels_trends,python,forrester/exp/constant,pylibkriging,q2,0.973551 -py01_kernels_trends,python,forrester/exp/constant,pylibkriging,nlpd,1.54551 -py01_kernels_trends,python,forrester/exp/linear,pylibkriging,fit_s,0.000438213 -py01_kernels_trends,python,forrester/exp/linear,pylibkriging,rmse,0.793413 -py01_kernels_trends,python,forrester/exp/linear,pylibkriging,q2,0.968024 -py01_kernels_trends,python,forrester/exp/linear,pylibkriging,nlpd,1.50639 -py01_kernels_trends,python,forrester/exp/interactive,pylibkriging,fit_s,0.000480413 -py01_kernels_trends,python,forrester/exp/interactive,pylibkriging,rmse,0.793413 -py01_kernels_trends,python,forrester/exp/interactive,pylibkriging,q2,0.968024 -py01_kernels_trends,python,forrester/exp/interactive,pylibkriging,nlpd,1.50639 -py01_kernels_trends,python,forrester/exp/quadratic,pylibkriging,fit_s,0.000730038 -py01_kernels_trends,python,forrester/exp/quadratic,pylibkriging,rmse,0.723267 -py01_kernels_trends,python,forrester/exp/quadratic,pylibkriging,q2,0.973428 -py01_kernels_trends,python,forrester/exp/quadratic,pylibkriging,nlpd,1.28782 -py01_kernels_trends,python,forrester/matern3_2/constant,pylibkriging,fit_s,0.00060606 -py01_kernels_trends,python,forrester/matern3_2/constant,pylibkriging,rmse,0.334007 -py01_kernels_trends,python,forrester/matern3_2/constant,pylibkriging,q2,0.994333 -py01_kernels_trends,python,forrester/matern3_2/constant,pylibkriging,nlpd,-0.227715 -py01_kernels_trends,python,forrester/matern3_2/linear,pylibkriging,fit_s,0.000588894 -py01_kernels_trends,python,forrester/matern3_2/linear,pylibkriging,rmse,0.364477 -py01_kernels_trends,python,forrester/matern3_2/linear,pylibkriging,q2,0.993252 -py01_kernels_trends,python,forrester/matern3_2/linear,pylibkriging,nlpd,-0.187262 -py01_kernels_trends,python,forrester/matern3_2/interactive,pylibkriging,fit_s,0.000905514 -py01_kernels_trends,python,forrester/matern3_2/interactive,pylibkriging,rmse,0.364477 -py01_kernels_trends,python,forrester/matern3_2/interactive,pylibkriging,q2,0.993252 -py01_kernels_trends,python,forrester/matern3_2/interactive,pylibkriging,nlpd,-0.187262 -py01_kernels_trends,python,forrester/matern3_2/quadratic,pylibkriging,fit_s,0.000975609 -py01_kernels_trends,python,forrester/matern3_2/quadratic,pylibkriging,rmse,0.435609 -py01_kernels_trends,python,forrester/matern3_2/quadratic,pylibkriging,q2,0.990361 -py01_kernels_trends,python,forrester/matern3_2/quadratic,pylibkriging,nlpd,-0.183076 -py01_kernels_trends,python,forrester/matern5_2/constant,pylibkriging,fit_s,0.000726223 -py01_kernels_trends,python,forrester/matern5_2/constant,pylibkriging,rmse,0.281945 -py01_kernels_trends,python,forrester/matern5_2/constant,pylibkriging,q2,0.995962 -py01_kernels_trends,python,forrester/matern5_2/constant,pylibkriging,nlpd,-1.01945 -py01_kernels_trends,python,forrester/matern5_2/constant,pylibkriging BFGS10,rmse,0.281945 -py01_kernels_trends,python,forrester/matern5_2/constant,pylibkriging BFGS10,q2,0.995962 -py01_kernels_trends,python,forrester/matern5_2/constant,pylibkriging BFGS10,nlpd,-1.01945 -py01_kernels_trends,python,forrester/matern5_2/linear,pylibkriging,fit_s,0.000932932 -py01_kernels_trends,python,forrester/matern5_2/linear,pylibkriging,rmse,0.298959 -py01_kernels_trends,python,forrester/matern5_2/linear,pylibkriging,q2,0.99546 -py01_kernels_trends,python,forrester/matern5_2/linear,pylibkriging,nlpd,-0.95551 -py01_kernels_trends,python,forrester/matern5_2/interactive,pylibkriging,fit_s,0.000725269 -py01_kernels_trends,python,forrester/matern5_2/interactive,pylibkriging,rmse,0.298959 -py01_kernels_trends,python,forrester/matern5_2/interactive,pylibkriging,q2,0.99546 -py01_kernels_trends,python,forrester/matern5_2/interactive,pylibkriging,nlpd,-0.95551 -py01_kernels_trends,python,forrester/matern5_2/quadratic,pylibkriging,fit_s,0.00131941 -py01_kernels_trends,python,forrester/matern5_2/quadratic,pylibkriging,rmse,0.365039 -py01_kernels_trends,python,forrester/matern5_2/quadratic,pylibkriging,q2,0.993231 -py01_kernels_trends,python,forrester/matern5_2/quadratic,pylibkriging,nlpd,-0.790032 -py01_kernels_trends,python,gramacy_lee_2012/gauss/constant,pylibkriging,fit_s,0.0016458 -py01_kernels_trends,python,gramacy_lee_2012/gauss/constant,pylibkriging,rmse,0.191274 -py01_kernels_trends,python,gramacy_lee_2012/gauss/constant,pylibkriging,q2,0.978742 -py01_kernels_trends,python,gramacy_lee_2012/gauss/constant,pylibkriging,nlpd,-0.704948 -py01_kernels_trends,python,gramacy_lee_2012/gauss/linear,pylibkriging,fit_s,0.00319505 -py01_kernels_trends,python,gramacy_lee_2012/gauss/linear,pylibkriging,rmse,0.236028 -py01_kernels_trends,python,gramacy_lee_2012/gauss/linear,pylibkriging,q2,0.96763 -py01_kernels_trends,python,gramacy_lee_2012/gauss/linear,pylibkriging,nlpd,-0.580993 -py01_kernels_trends,python,gramacy_lee_2012/gauss/interactive,pylibkriging,fit_s,0.00250673 -py01_kernels_trends,python,gramacy_lee_2012/gauss/interactive,pylibkriging,rmse,0.236028 -py01_kernels_trends,python,gramacy_lee_2012/gauss/interactive,pylibkriging,q2,0.96763 -py01_kernels_trends,python,gramacy_lee_2012/gauss/interactive,pylibkriging,nlpd,-0.580993 -py01_kernels_trends,python,gramacy_lee_2012/gauss/quadratic,pylibkriging,fit_s,0.00243235 -py01_kernels_trends,python,gramacy_lee_2012/gauss/quadratic,pylibkriging,rmse,0.138869 -py01_kernels_trends,python,gramacy_lee_2012/gauss/quadratic,pylibkriging,q2,0.988795 -py01_kernels_trends,python,gramacy_lee_2012/gauss/quadratic,pylibkriging,nlpd,-0.530437 -py01_kernels_trends,python,gramacy_lee_2012/exp/constant,pylibkriging,fit_s,0.000752449 -py01_kernels_trends,python,gramacy_lee_2012/exp/constant,pylibkriging,rmse,0.196586 -py01_kernels_trends,python,gramacy_lee_2012/exp/constant,pylibkriging,q2,0.977545 -py01_kernels_trends,python,gramacy_lee_2012/exp/constant,pylibkriging,nlpd,-0.0621192 -py01_kernels_trends,python,gramacy_lee_2012/exp/linear,pylibkriging,fit_s,0.0010047 -py01_kernels_trends,python,gramacy_lee_2012/exp/linear,pylibkriging,rmse,0.20979 -py01_kernels_trends,python,gramacy_lee_2012/exp/linear,pylibkriging,q2,0.974427 -py01_kernels_trends,python,gramacy_lee_2012/exp/linear,pylibkriging,nlpd,-0.0466568 -py01_kernels_trends,python,gramacy_lee_2012/exp/interactive,pylibkriging,fit_s,0.000931501 -py01_kernels_trends,python,gramacy_lee_2012/exp/interactive,pylibkriging,rmse,0.20979 -py01_kernels_trends,python,gramacy_lee_2012/exp/interactive,pylibkriging,q2,0.974427 -py01_kernels_trends,python,gramacy_lee_2012/exp/interactive,pylibkriging,nlpd,-0.0466568 -py01_kernels_trends,python,gramacy_lee_2012/exp/quadratic,pylibkriging,fit_s,0.0025785 -py01_kernels_trends,python,gramacy_lee_2012/exp/quadratic,pylibkriging,rmse,0.220579 -py01_kernels_trends,python,gramacy_lee_2012/exp/quadratic,pylibkriging,q2,0.971729 -py01_kernels_trends,python,gramacy_lee_2012/exp/quadratic,pylibkriging,nlpd,0.136521 -py01_kernels_trends,python,gramacy_lee_2012/matern3_2/constant,pylibkriging,fit_s,0.000761509 -py01_kernels_trends,python,gramacy_lee_2012/matern3_2/constant,pylibkriging,rmse,0.177253 -py01_kernels_trends,python,gramacy_lee_2012/matern3_2/constant,pylibkriging,q2,0.981744 -py01_kernels_trends,python,gramacy_lee_2012/matern3_2/constant,pylibkriging,nlpd,-0.482212 -py01_kernels_trends,python,gramacy_lee_2012/matern3_2/linear,pylibkriging,fit_s,0.00132823 -py01_kernels_trends,python,gramacy_lee_2012/matern3_2/linear,pylibkriging,rmse,0.211499 -py01_kernels_trends,python,gramacy_lee_2012/matern3_2/linear,pylibkriging,q2,0.974009 -py01_kernels_trends,python,gramacy_lee_2012/matern3_2/linear,pylibkriging,nlpd,-0.375844 -py01_kernels_trends,python,gramacy_lee_2012/matern3_2/interactive,pylibkriging,fit_s,0.000978708 -py01_kernels_trends,python,gramacy_lee_2012/matern3_2/interactive,pylibkriging,rmse,0.211499 -py01_kernels_trends,python,gramacy_lee_2012/matern3_2/interactive,pylibkriging,q2,0.974009 -py01_kernels_trends,python,gramacy_lee_2012/matern3_2/interactive,pylibkriging,nlpd,-0.375844 -py01_kernels_trends,python,gramacy_lee_2012/matern3_2/quadratic,pylibkriging,fit_s,0.00228167 -py01_kernels_trends,python,gramacy_lee_2012/matern3_2/quadratic,pylibkriging,rmse,0.173012 -py01_kernels_trends,python,gramacy_lee_2012/matern3_2/quadratic,pylibkriging,q2,0.982607 -py01_kernels_trends,python,gramacy_lee_2012/matern3_2/quadratic,pylibkriging,nlpd,-0.223308 -py01_kernels_trends,python,gramacy_lee_2012/matern5_2/constant,pylibkriging,fit_s,0.00131416 -py01_kernels_trends,python,gramacy_lee_2012/matern5_2/constant,pylibkriging,rmse,0.177638 -py01_kernels_trends,python,gramacy_lee_2012/matern5_2/constant,pylibkriging,q2,0.981665 -py01_kernels_trends,python,gramacy_lee_2012/matern5_2/constant,pylibkriging,nlpd,-0.575421 -py01_kernels_trends,python,gramacy_lee_2012/matern5_2/constant,pylibkriging BFGS10,rmse,0.177638 -py01_kernels_trends,python,gramacy_lee_2012/matern5_2/constant,pylibkriging BFGS10,q2,0.981665 -py01_kernels_trends,python,gramacy_lee_2012/matern5_2/constant,pylibkriging BFGS10,nlpd,-0.575422 -py01_kernels_trends,python,gramacy_lee_2012/matern5_2/linear,pylibkriging,fit_s,0.00130415 -py01_kernels_trends,python,gramacy_lee_2012/matern5_2/linear,pylibkriging,rmse,0.219759 -py01_kernels_trends,python,gramacy_lee_2012/matern5_2/linear,pylibkriging,q2,0.971939 -py01_kernels_trends,python,gramacy_lee_2012/matern5_2/linear,pylibkriging,nlpd,-0.458933 -py01_kernels_trends,python,gramacy_lee_2012/matern5_2/interactive,pylibkriging,fit_s,0.00119758 -py01_kernels_trends,python,gramacy_lee_2012/matern5_2/interactive,pylibkriging,rmse,0.219759 -py01_kernels_trends,python,gramacy_lee_2012/matern5_2/interactive,pylibkriging,q2,0.971939 -py01_kernels_trends,python,gramacy_lee_2012/matern5_2/interactive,pylibkriging,nlpd,-0.458933 -py01_kernels_trends,python,gramacy_lee_2012/matern5_2/quadratic,pylibkriging,fit_s,0.00169826 -py01_kernels_trends,python,gramacy_lee_2012/matern5_2/quadratic,pylibkriging,rmse,0.160506 -py01_kernels_trends,python,gramacy_lee_2012/matern5_2/quadratic,pylibkriging,q2,0.985031 -py01_kernels_trends,python,gramacy_lee_2012/matern5_2/quadratic,pylibkriging,nlpd,-0.337276 -py01_kernels_trends,python,branin/gauss/constant,pylibkriging,fit_s,0.00104284 -py01_kernels_trends,python,branin/gauss/constant,pylibkriging,rmse,7.2629 -py01_kernels_trends,python,branin/gauss/constant,pylibkriging,q2,0.980655 -py01_kernels_trends,python,branin/gauss/constant,pylibkriging,nlpd,4.3152 -py01_kernels_trends,python,branin/gauss/linear,pylibkriging,fit_s,0.00129008 -py01_kernels_trends,python,branin/gauss/linear,pylibkriging,rmse,7.25733 -py01_kernels_trends,python,branin/gauss/linear,pylibkriging,q2,0.980685 -py01_kernels_trends,python,branin/gauss/linear,pylibkriging,nlpd,4.30813 -py01_kernels_trends,python,branin/gauss/interactive,pylibkriging,fit_s,0.00280094 -py01_kernels_trends,python,branin/gauss/interactive,pylibkriging,rmse,7.2576 -py01_kernels_trends,python,branin/gauss/interactive,pylibkriging,q2,0.980683 -py01_kernels_trends,python,branin/gauss/interactive,pylibkriging,nlpd,4.30176 -py01_kernels_trends,python,branin/gauss/quadratic,pylibkriging,fit_s,0.00112319 -py01_kernels_trends,python,branin/gauss/quadratic,pylibkriging,rmse,7.27808 -py01_kernels_trends,python,branin/gauss/quadratic,pylibkriging,q2,0.980574 -py01_kernels_trends,python,branin/gauss/quadratic,pylibkriging,nlpd,4.29399 -py01_kernels_trends,python,branin/exp/constant,pylibkriging,fit_s,0.00222707 -py01_kernels_trends,python,branin/exp/constant,pylibkriging,rmse,14.5887 -py01_kernels_trends,python,branin/exp/constant,pylibkriging,q2,0.921947 -py01_kernels_trends,python,branin/exp/constant,pylibkriging,nlpd,4.09457 -py01_kernels_trends,python,branin/exp/linear,pylibkriging,fit_s,0.00213099 -py01_kernels_trends,python,branin/exp/linear,pylibkriging,rmse,14.9622 -py01_kernels_trends,python,branin/exp/linear,pylibkriging,q2,0.9179 -py01_kernels_trends,python,branin/exp/linear,pylibkriging,nlpd,4.0909 -py01_kernels_trends,python,branin/exp/interactive,pylibkriging,fit_s,0.00196409 -py01_kernels_trends,python,branin/exp/interactive,pylibkriging,rmse,11.6963 -py01_kernels_trends,python,branin/exp/interactive,pylibkriging,q2,0.94983 -py01_kernels_trends,python,branin/exp/interactive,pylibkriging,nlpd,3.79671 -py01_kernels_trends,python,branin/exp/quadratic,pylibkriging,fit_s,0.00227737 -py01_kernels_trends,python,branin/exp/quadratic,pylibkriging,rmse,9.33909 -py01_kernels_trends,python,branin/exp/quadratic,pylibkriging,q2,0.968014 -py01_kernels_trends,python,branin/exp/quadratic,pylibkriging,nlpd,3.62362 -py01_kernels_trends,python,branin/matern3_2/constant,pylibkriging,fit_s,0.00244212 -py01_kernels_trends,python,branin/matern3_2/constant,pylibkriging,rmse,3.24796 -py01_kernels_trends,python,branin/matern3_2/constant,pylibkriging,q2,0.996131 -py01_kernels_trends,python,branin/matern3_2/constant,pylibkriging,nlpd,2.30444 -py01_kernels_trends,python,branin/matern3_2/linear,pylibkriging,fit_s,0.00244641 -py01_kernels_trends,python,branin/matern3_2/linear,pylibkriging,rmse,3.14402 -py01_kernels_trends,python,branin/matern3_2/linear,pylibkriging,q2,0.996375 -py01_kernels_trends,python,branin/matern3_2/linear,pylibkriging,nlpd,2.28067 -py01_kernels_trends,python,branin/matern3_2/interactive,pylibkriging,fit_s,0.00703263 -py01_kernels_trends,python,branin/matern3_2/interactive,pylibkriging,rmse,3.19226 -py01_kernels_trends,python,branin/matern3_2/interactive,pylibkriging,q2,0.996263 -py01_kernels_trends,python,branin/matern3_2/interactive,pylibkriging,nlpd,2.20359 -py01_kernels_trends,python,branin/matern3_2/quadratic,pylibkriging,fit_s,0.00404716 -py01_kernels_trends,python,branin/matern3_2/quadratic,pylibkriging,rmse,3.01232 -py01_kernels_trends,python,branin/matern3_2/quadratic,pylibkriging,q2,0.996672 -py01_kernels_trends,python,branin/matern3_2/quadratic,pylibkriging,nlpd,2.11627 -py01_kernels_trends,python,branin/matern5_2/constant,pylibkriging,fit_s,0.0029068 -py01_kernels_trends,python,branin/matern5_2/constant,pylibkriging,rmse,0.52012 -py01_kernels_trends,python,branin/matern5_2/constant,pylibkriging,q2,0.999901 -py01_kernels_trends,python,branin/matern5_2/constant,pylibkriging,nlpd,0.283399 -py01_kernels_trends,python,branin/matern5_2/constant,pylibkriging BFGS10,rmse,0.52012 -py01_kernels_trends,python,branin/matern5_2/constant,pylibkriging BFGS10,q2,0.999901 -py01_kernels_trends,python,branin/matern5_2/constant,pylibkriging BFGS10,nlpd,0.283395 -py01_kernels_trends,python,branin/matern5_2/linear,pylibkriging,fit_s,0.00212407 -py01_kernels_trends,python,branin/matern5_2/linear,pylibkriging,rmse,0.460868 -py01_kernels_trends,python,branin/matern5_2/linear,pylibkriging,q2,0.999922 -py01_kernels_trends,python,branin/matern5_2/linear,pylibkriging,nlpd,0.209741 -py01_kernels_trends,python,branin/matern5_2/interactive,pylibkriging,fit_s,0.0018971 -py01_kernels_trends,python,branin/matern5_2/interactive,pylibkriging,rmse,0.498484 -py01_kernels_trends,python,branin/matern5_2/interactive,pylibkriging,q2,0.999909 -py01_kernels_trends,python,branin/matern5_2/interactive,pylibkriging,nlpd,0.239079 -py01_kernels_trends,python,branin/matern5_2/quadratic,pylibkriging,fit_s,0.00570393 -py01_kernels_trends,python,branin/matern5_2/quadratic,pylibkriging,rmse,0.67472 -py01_kernels_trends,python,branin/matern5_2/quadratic,pylibkriging,q2,0.999833 -py01_kernels_trends,python,branin/matern5_2/quadratic,pylibkriging,nlpd,0.419266 -py01_kernels_trends,python,six_hump_camel/gauss/constant,pylibkriging,fit_s,0.00197244 -py01_kernels_trends,python,six_hump_camel/gauss/constant,pylibkriging,rmse,18.9501 -py01_kernels_trends,python,six_hump_camel/gauss/constant,pylibkriging,q2,0.536354 -py01_kernels_trends,python,six_hump_camel/gauss/constant,pylibkriging,nlpd,84.151 -py01_kernels_trends,python,six_hump_camel/gauss/linear,pylibkriging,fit_s,0.00170588 -py01_kernels_trends,python,six_hump_camel/gauss/linear,pylibkriging,rmse,18.9579 -py01_kernels_trends,python,six_hump_camel/gauss/linear,pylibkriging,q2,0.535976 -py01_kernels_trends,python,six_hump_camel/gauss/linear,pylibkriging,nlpd,44.8696 -py01_kernels_trends,python,six_hump_camel/gauss/interactive,pylibkriging,fit_s,0.0020082 -py01_kernels_trends,python,six_hump_camel/gauss/interactive,pylibkriging,rmse,18.9819 -py01_kernels_trends,python,six_hump_camel/gauss/interactive,pylibkriging,q2,0.5348 -py01_kernels_trends,python,six_hump_camel/gauss/interactive,pylibkriging,nlpd,35.2765 -py01_kernels_trends,python,six_hump_camel/gauss/quadratic,pylibkriging,fit_s,0.00248289 -py01_kernels_trends,python,six_hump_camel/gauss/quadratic,pylibkriging,rmse,10.9058 -py01_kernels_trends,python,six_hump_camel/gauss/quadratic,pylibkriging,q2,0.846441 -py01_kernels_trends,python,six_hump_camel/gauss/quadratic,pylibkriging,nlpd,4.92017 -py01_kernels_trends,python,six_hump_camel/exp/constant,pylibkriging,fit_s,0.00100708 -py01_kernels_trends,python,six_hump_camel/exp/constant,pylibkriging,rmse,16.9653 -py01_kernels_trends,python,six_hump_camel/exp/constant,pylibkriging,q2,0.628393 -py01_kernels_trends,python,six_hump_camel/exp/constant,pylibkriging,nlpd,4.48595 -py01_kernels_trends,python,six_hump_camel/exp/linear,pylibkriging,fit_s,0.00118208 -py01_kernels_trends,python,six_hump_camel/exp/linear,pylibkriging,rmse,17.1781 -py01_kernels_trends,python,six_hump_camel/exp/linear,pylibkriging,q2,0.619012 -py01_kernels_trends,python,six_hump_camel/exp/linear,pylibkriging,nlpd,4.47061 -py01_kernels_trends,python,six_hump_camel/exp/interactive,pylibkriging,fit_s,0.00123763 -py01_kernels_trends,python,six_hump_camel/exp/interactive,pylibkriging,rmse,17.1856 -py01_kernels_trends,python,six_hump_camel/exp/interactive,pylibkriging,q2,0.618679 -py01_kernels_trends,python,six_hump_camel/exp/interactive,pylibkriging,nlpd,4.43133 -py01_kernels_trends,python,six_hump_camel/exp/quadratic,pylibkriging,fit_s,0.0013876 -py01_kernels_trends,python,six_hump_camel/exp/quadratic,pylibkriging,rmse,7.99484 -py01_kernels_trends,python,six_hump_camel/exp/quadratic,pylibkriging,q2,0.917476 -py01_kernels_trends,python,six_hump_camel/exp/quadratic,pylibkriging,nlpd,3.35612 -py01_kernels_trends,python,six_hump_camel/matern3_2/constant,pylibkriging,fit_s,0.00175476 -py01_kernels_trends,python,six_hump_camel/matern3_2/constant,pylibkriging,rmse,18.4958 -py01_kernels_trends,python,six_hump_camel/matern3_2/constant,pylibkriging,q2,0.558321 -py01_kernels_trends,python,six_hump_camel/matern3_2/constant,pylibkriging,nlpd,18.8349 -py01_kernels_trends,python,six_hump_camel/matern3_2/linear,pylibkriging,fit_s,0.00206542 -py01_kernels_trends,python,six_hump_camel/matern3_2/linear,pylibkriging,rmse,18.5562 -py01_kernels_trends,python,six_hump_camel/matern3_2/linear,pylibkriging,q2,0.555433 -py01_kernels_trends,python,six_hump_camel/matern3_2/linear,pylibkriging,nlpd,14.9494 -py01_kernels_trends,python,six_hump_camel/matern3_2/interactive,pylibkriging,fit_s,0.00178266 -py01_kernels_trends,python,six_hump_camel/matern3_2/interactive,pylibkriging,rmse,18.561 -py01_kernels_trends,python,six_hump_camel/matern3_2/interactive,pylibkriging,q2,0.555202 -py01_kernels_trends,python,six_hump_camel/matern3_2/interactive,pylibkriging,nlpd,12.9923 -py01_kernels_trends,python,six_hump_camel/matern3_2/quadratic,pylibkriging,fit_s,0.00338221 -py01_kernels_trends,python,six_hump_camel/matern3_2/quadratic,pylibkriging,rmse,8.91645 -py01_kernels_trends,python,six_hump_camel/matern3_2/quadratic,pylibkriging,q2,0.897353 -py01_kernels_trends,python,six_hump_camel/matern3_2/quadratic,pylibkriging,nlpd,3.94939 -py01_kernels_trends,python,six_hump_camel/matern5_2/constant,pylibkriging,fit_s,0.00228882 -py01_kernels_trends,python,six_hump_camel/matern5_2/constant,pylibkriging,rmse,18.3236 -py01_kernels_trends,python,six_hump_camel/matern5_2/constant,pylibkriging,q2,0.566507 -py01_kernels_trends,python,six_hump_camel/matern5_2/constant,pylibkriging,nlpd,18.4322 -py01_kernels_trends,python,six_hump_camel/matern5_2/constant,pylibkriging BFGS10,rmse,1.36208 -py01_kernels_trends,python,six_hump_camel/matern5_2/constant,pylibkriging BFGS10,q2,0.997605 -py01_kernels_trends,python,six_hump_camel/matern5_2/constant,pylibkriging BFGS10,nlpd,1.03215 -py01_kernels_trends,python,six_hump_camel/matern5_2/linear,pylibkriging,fit_s,0.00242186 -py01_kernels_trends,python,six_hump_camel/matern5_2/linear,pylibkriging,rmse,18.7127 -py01_kernels_trends,python,six_hump_camel/matern5_2/linear,pylibkriging,q2,0.547901 -py01_kernels_trends,python,six_hump_camel/matern5_2/linear,pylibkriging,nlpd,24.5771 -py01_kernels_trends,python,six_hump_camel/matern5_2/interactive,pylibkriging,fit_s,0.00234485 -py01_kernels_trends,python,six_hump_camel/matern5_2/interactive,pylibkriging,rmse,18.7194 -py01_kernels_trends,python,six_hump_camel/matern5_2/interactive,pylibkriging,q2,0.547577 -py01_kernels_trends,python,six_hump_camel/matern5_2/interactive,pylibkriging,nlpd,20.4255 -py01_kernels_trends,python,six_hump_camel/matern5_2/quadratic,pylibkriging,fit_s,0.00251341 -py01_kernels_trends,python,six_hump_camel/matern5_2/quadratic,pylibkriging,rmse,10.2878 -py01_kernels_trends,python,six_hump_camel/matern5_2/quadratic,pylibkriging,q2,0.863351 -py01_kernels_trends,python,six_hump_camel/matern5_2/quadratic,pylibkriging,nlpd,4.46344 -py01_kernels_trends,python,log_goldstein_price/gauss/constant,pylibkriging,fit_s,0.00588727 -py01_kernels_trends,python,log_goldstein_price/gauss/constant,pylibkriging,rmse,0.295934 -py01_kernels_trends,python,log_goldstein_price/gauss/constant,pylibkriging,q2,0.920537 -py01_kernels_trends,python,log_goldstein_price/gauss/constant,pylibkriging,nlpd,1.58666 -py01_kernels_trends,python,log_goldstein_price/gauss/linear,pylibkriging,fit_s,0.00581551 -py01_kernels_trends,python,log_goldstein_price/gauss/linear,pylibkriging,rmse,0.296355 -py01_kernels_trends,python,log_goldstein_price/gauss/linear,pylibkriging,q2,0.92031 -py01_kernels_trends,python,log_goldstein_price/gauss/linear,pylibkriging,nlpd,1.2962 -py01_kernels_trends,python,log_goldstein_price/gauss/interactive,pylibkriging,fit_s,0.00644922 -py01_kernels_trends,python,log_goldstein_price/gauss/interactive,pylibkriging,rmse,0.285136 -py01_kernels_trends,python,log_goldstein_price/gauss/interactive,pylibkriging,q2,0.926229 -py01_kernels_trends,python,log_goldstein_price/gauss/interactive,pylibkriging,nlpd,1.04122 -py01_kernels_trends,python,log_goldstein_price/gauss/quadratic,pylibkriging,fit_s,0.00578475 -py01_kernels_trends,python,log_goldstein_price/gauss/quadratic,pylibkriging,rmse,0.237308 -py01_kernels_trends,python,log_goldstein_price/gauss/quadratic,pylibkriging,q2,0.948902 -py01_kernels_trends,python,log_goldstein_price/gauss/quadratic,pylibkriging,nlpd,0.146484 -py01_kernels_trends,python,log_goldstein_price/exp/constant,pylibkriging,fit_s,0.00258231 -py01_kernels_trends,python,log_goldstein_price/exp/constant,pylibkriging,rmse,0.334876 -py01_kernels_trends,python,log_goldstein_price/exp/constant,pylibkriging,q2,0.898247 -py01_kernels_trends,python,log_goldstein_price/exp/constant,pylibkriging,nlpd,0.339235 -py01_kernels_trends,python,log_goldstein_price/exp/linear,pylibkriging,fit_s,0.00211 -py01_kernels_trends,python,log_goldstein_price/exp/linear,pylibkriging,rmse,0.333447 -py01_kernels_trends,python,log_goldstein_price/exp/linear,pylibkriging,q2,0.899114 -py01_kernels_trends,python,log_goldstein_price/exp/linear,pylibkriging,nlpd,0.336515 -py01_kernels_trends,python,log_goldstein_price/exp/interactive,pylibkriging,fit_s,0.0025239 -py01_kernels_trends,python,log_goldstein_price/exp/interactive,pylibkriging,rmse,0.336819 -py01_kernels_trends,python,log_goldstein_price/exp/interactive,pylibkriging,q2,0.897063 -py01_kernels_trends,python,log_goldstein_price/exp/interactive,pylibkriging,nlpd,0.329566 -py01_kernels_trends,python,log_goldstein_price/exp/quadratic,pylibkriging,fit_s,0.00309515 -py01_kernels_trends,python,log_goldstein_price/exp/quadratic,pylibkriging,rmse,0.327816 -py01_kernels_trends,python,log_goldstein_price/exp/quadratic,pylibkriging,q2,0.902492 -py01_kernels_trends,python,log_goldstein_price/exp/quadratic,pylibkriging,nlpd,0.322056 -py01_kernels_trends,python,log_goldstein_price/matern3_2/constant,pylibkriging,fit_s,0.00425816 -py01_kernels_trends,python,log_goldstein_price/matern3_2/constant,pylibkriging,rmse,0.684362 -py01_kernels_trends,python,log_goldstein_price/matern3_2/constant,pylibkriging,q2,0.575038 -py01_kernels_trends,python,log_goldstein_price/matern3_2/constant,pylibkriging,nlpd,0.972803 -py01_kernels_trends,python,log_goldstein_price/matern3_2/linear,pylibkriging,fit_s,0.00413442 -py01_kernels_trends,python,log_goldstein_price/matern3_2/linear,pylibkriging,rmse,0.650201 -py01_kernels_trends,python,log_goldstein_price/matern3_2/linear,pylibkriging,q2,0.616404 -py01_kernels_trends,python,log_goldstein_price/matern3_2/linear,pylibkriging,nlpd,0.921455 -py01_kernels_trends,python,log_goldstein_price/matern3_2/interactive,pylibkriging,fit_s,0.00704145 -py01_kernels_trends,python,log_goldstein_price/matern3_2/interactive,pylibkriging,rmse,0.274326 -py01_kernels_trends,python,log_goldstein_price/matern3_2/interactive,pylibkriging,q2,0.931717 -py01_kernels_trends,python,log_goldstein_price/matern3_2/interactive,pylibkriging,nlpd,0.0549679 -py01_kernels_trends,python,log_goldstein_price/matern3_2/quadratic,pylibkriging,fit_s,0.00996184 -py01_kernels_trends,python,log_goldstein_price/matern3_2/quadratic,pylibkriging,rmse,0.263745 -py01_kernels_trends,python,log_goldstein_price/matern3_2/quadratic,pylibkriging,q2,0.936883 -py01_kernels_trends,python,log_goldstein_price/matern3_2/quadratic,pylibkriging,nlpd,0.0446267 -py01_kernels_trends,python,log_goldstein_price/matern5_2/constant,pylibkriging,fit_s,0.00410438 -py01_kernels_trends,python,log_goldstein_price/matern5_2/constant,pylibkriging,rmse,0.674391 -py01_kernels_trends,python,log_goldstein_price/matern5_2/constant,pylibkriging,q2,0.58733 -py01_kernels_trends,python,log_goldstein_price/matern5_2/constant,pylibkriging,nlpd,0.944765 -py01_kernels_trends,python,log_goldstein_price/matern5_2/constant,pylibkriging BFGS10,rmse,0.268216 -py01_kernels_trends,python,log_goldstein_price/matern5_2/constant,pylibkriging BFGS10,q2,0.934725 -py01_kernels_trends,python,log_goldstein_price/matern5_2/constant,pylibkriging BFGS10,nlpd,0.169585 -py01_kernels_trends,python,log_goldstein_price/matern5_2/linear,pylibkriging,fit_s,0.00728607 -py01_kernels_trends,python,log_goldstein_price/matern5_2/linear,pylibkriging,rmse,0.268808 -py01_kernels_trends,python,log_goldstein_price/matern5_2/linear,pylibkriging,q2,0.934436 -py01_kernels_trends,python,log_goldstein_price/matern5_2/linear,pylibkriging,nlpd,0.142382 -py01_kernels_trends,python,log_goldstein_price/matern5_2/interactive,pylibkriging,fit_s,0.00524259 -py01_kernels_trends,python,log_goldstein_price/matern5_2/interactive,pylibkriging,rmse,0.271088 -py01_kernels_trends,python,log_goldstein_price/matern5_2/interactive,pylibkriging,q2,0.933319 -py01_kernels_trends,python,log_goldstein_price/matern5_2/interactive,pylibkriging,nlpd,0.128788 -py01_kernels_trends,python,log_goldstein_price/matern5_2/quadratic,pylibkriging,fit_s,0.011641 -py01_kernels_trends,python,log_goldstein_price/matern5_2/quadratic,pylibkriging,rmse,0.253093 -py01_kernels_trends,python,log_goldstein_price/matern5_2/quadratic,pylibkriging,q2,0.941878 -py01_kernels_trends,python,log_goldstein_price/matern5_2/quadratic,pylibkriging,nlpd,0.025469 -py01_kernels_trends,python,lim_polynomial/gauss/constant,pylibkriging,fit_s,0.00113583 -py01_kernels_trends,python,lim_polynomial/gauss/constant,pylibkriging,rmse,0.026127 -py01_kernels_trends,python,lim_polynomial/gauss/constant,pylibkriging,q2,0.999737 -py01_kernels_trends,python,lim_polynomial/gauss/constant,pylibkriging,nlpd,-2.69817 -py01_kernels_trends,python,lim_polynomial/gauss/linear,pylibkriging,fit_s,0.00152946 -py01_kernels_trends,python,lim_polynomial/gauss/linear,pylibkriging,rmse,1.06243 -py01_kernels_trends,python,lim_polynomial/gauss/linear,pylibkriging,q2,0.564691 -py01_kernels_trends,python,lim_polynomial/gauss/linear,pylibkriging,nlpd,2.7394 -py01_kernels_trends,python,lim_polynomial/gauss/interactive,pylibkriging,fit_s,0.00153446 -py01_kernels_trends,python,lim_polynomial/gauss/interactive,pylibkriging,rmse,1.06504 -py01_kernels_trends,python,lim_polynomial/gauss/interactive,pylibkriging,q2,0.562551 -py01_kernels_trends,python,lim_polynomial/gauss/interactive,pylibkriging,nlpd,2.33575 -py01_kernels_trends,python,lim_polynomial/gauss/quadratic,pylibkriging,fit_s,0.00196862 -py01_kernels_trends,python,lim_polynomial/gauss/quadratic,pylibkriging,rmse,0.281816 -py01_kernels_trends,python,lim_polynomial/gauss/quadratic,pylibkriging,q2,0.969371 -py01_kernels_trends,python,lim_polynomial/gauss/quadratic,pylibkriging,nlpd,0.71682 -py01_kernels_trends,python,lim_polynomial/exp/constant,pylibkriging,fit_s,0.000828028 -py01_kernels_trends,python,lim_polynomial/exp/constant,pylibkriging,rmse,0.249023 -py01_kernels_trends,python,lim_polynomial/exp/constant,pylibkriging,q2,0.976085 -py01_kernels_trends,python,lim_polynomial/exp/constant,pylibkriging,nlpd,0.394159 -py01_kernels_trends,python,lim_polynomial/exp/linear,pylibkriging,fit_s,0.000962973 -py01_kernels_trends,python,lim_polynomial/exp/linear,pylibkriging,rmse,0.245558 -py01_kernels_trends,python,lim_polynomial/exp/linear,pylibkriging,q2,0.976746 -py01_kernels_trends,python,lim_polynomial/exp/linear,pylibkriging,nlpd,0.224593 -py01_kernels_trends,python,lim_polynomial/exp/interactive,pylibkriging,fit_s,0.00143671 -py01_kernels_trends,python,lim_polynomial/exp/interactive,pylibkriging,rmse,0.239032 -py01_kernels_trends,python,lim_polynomial/exp/interactive,pylibkriging,q2,0.977965 -py01_kernels_trends,python,lim_polynomial/exp/interactive,pylibkriging,nlpd,0.226133 -py01_kernels_trends,python,lim_polynomial/exp/quadratic,pylibkriging,fit_s,0.00274634 -py01_kernels_trends,python,lim_polynomial/exp/quadratic,pylibkriging,rmse,0.237713 -py01_kernels_trends,python,lim_polynomial/exp/quadratic,pylibkriging,q2,0.978208 -py01_kernels_trends,python,lim_polynomial/exp/quadratic,pylibkriging,nlpd,-0.139582 -py01_kernels_trends,python,lim_polynomial/matern3_2/constant,pylibkriging,fit_s,0.0011251 -py01_kernels_trends,python,lim_polynomial/matern3_2/constant,pylibkriging,rmse,0.0600911 -py01_kernels_trends,python,lim_polynomial/matern3_2/constant,pylibkriging,q2,0.998607 -py01_kernels_trends,python,lim_polynomial/matern3_2/constant,pylibkriging,nlpd,-1.34658 -py01_kernels_trends,python,lim_polynomial/matern3_2/linear,pylibkriging,fit_s,0.000877857 -py01_kernels_trends,python,lim_polynomial/matern3_2/linear,pylibkriging,rmse,0.0584376 -py01_kernels_trends,python,lim_polynomial/matern3_2/linear,pylibkriging,q2,0.998683 -py01_kernels_trends,python,lim_polynomial/matern3_2/linear,pylibkriging,nlpd,-1.34912 -py01_kernels_trends,python,lim_polynomial/matern3_2/interactive,pylibkriging,fit_s,0.00140738 -py01_kernels_trends,python,lim_polynomial/matern3_2/interactive,pylibkriging,rmse,0.0576096 -py01_kernels_trends,python,lim_polynomial/matern3_2/interactive,pylibkriging,q2,0.99872 -py01_kernels_trends,python,lim_polynomial/matern3_2/interactive,pylibkriging,nlpd,-1.35376 -py01_kernels_trends,python,lim_polynomial/matern3_2/quadratic,pylibkriging,fit_s,0.00383472 -py01_kernels_trends,python,lim_polynomial/matern3_2/quadratic,pylibkriging,rmse,0.215336 -py01_kernels_trends,python,lim_polynomial/matern3_2/quadratic,pylibkriging,q2,0.982117 -py01_kernels_trends,python,lim_polynomial/matern3_2/quadratic,pylibkriging,nlpd,-0.350837 -py01_kernels_trends,python,lim_polynomial/matern5_2/constant,pylibkriging,fit_s,0.000718832 -py01_kernels_trends,python,lim_polynomial/matern5_2/constant,pylibkriging,rmse,0.0246348 -py01_kernels_trends,python,lim_polynomial/matern5_2/constant,pylibkriging,q2,0.999766 -py01_kernels_trends,python,lim_polynomial/matern5_2/constant,pylibkriging,nlpd,-2.59598 -py01_kernels_trends,python,lim_polynomial/matern5_2/constant,pylibkriging BFGS10,rmse,0.0246338 -py01_kernels_trends,python,lim_polynomial/matern5_2/constant,pylibkriging BFGS10,q2,0.999766 -py01_kernels_trends,python,lim_polynomial/matern5_2/constant,pylibkriging BFGS10,nlpd,-2.59598 -py01_kernels_trends,python,lim_polynomial/matern5_2/linear,pylibkriging,fit_s,0.0013113 -py01_kernels_trends,python,lim_polynomial/matern5_2/linear,pylibkriging,rmse,0.0232483 -py01_kernels_trends,python,lim_polynomial/matern5_2/linear,pylibkriging,q2,0.999792 -py01_kernels_trends,python,lim_polynomial/matern5_2/linear,pylibkriging,nlpd,-2.58843 -py01_kernels_trends,python,lim_polynomial/matern5_2/interactive,pylibkriging,fit_s,0.00150442 -py01_kernels_trends,python,lim_polynomial/matern5_2/interactive,pylibkriging,rmse,0.0230248 -py01_kernels_trends,python,lim_polynomial/matern5_2/interactive,pylibkriging,q2,0.999796 -py01_kernels_trends,python,lim_polynomial/matern5_2/interactive,pylibkriging,nlpd,-2.58839 -py01_kernels_trends,python,lim_polynomial/matern5_2/quadratic,pylibkriging,fit_s,0.00300908 -py01_kernels_trends,python,lim_polynomial/matern5_2/quadratic,pylibkriging,rmse,0.209802 -py01_kernels_trends,python,lim_polynomial/matern5_2/quadratic,pylibkriging,q2,0.983025 -py01_kernels_trends,python,lim_polynomial/matern5_2/quadratic,pylibkriging,nlpd,-0.403998 -py01_kernels_trends,python,ishigami/gauss/constant,pylibkriging,fit_s,0.0101857 -py01_kernels_trends,python,ishigami/gauss/constant,pylibkriging,rmse,2.07486 -py01_kernels_trends,python,ishigami/gauss/constant,pylibkriging,q2,0.668191 -py01_kernels_trends,python,ishigami/gauss/constant,pylibkriging,nlpd,2.80326 -py01_kernels_trends,python,ishigami/gauss/linear,pylibkriging,fit_s,0.0193923 -py01_kernels_trends,python,ishigami/gauss/linear,pylibkriging,rmse,2.8459 -py01_kernels_trends,python,ishigami/gauss/linear,pylibkriging,q2,0.375762 -py01_kernels_trends,python,ishigami/gauss/linear,pylibkriging,nlpd,3.41919 -py01_kernels_trends,python,ishigami/gauss/interactive,pylibkriging,fit_s,0.0174267 -py01_kernels_trends,python,ishigami/gauss/interactive,pylibkriging,rmse,2.85366 -py01_kernels_trends,python,ishigami/gauss/interactive,pylibkriging,q2,0.372353 -py01_kernels_trends,python,ishigami/gauss/interactive,pylibkriging,nlpd,3.28643 -py01_kernels_trends,python,ishigami/gauss/quadratic,pylibkriging,fit_s,0.00865173 -py01_kernels_trends,python,ishigami/gauss/quadratic,pylibkriging,rmse,2.08981 -py01_kernels_trends,python,ishigami/gauss/quadratic,pylibkriging,q2,0.663392 -py01_kernels_trends,python,ishigami/gauss/quadratic,pylibkriging,nlpd,2.80097 -py01_kernels_trends,python,ishigami/exp/constant,pylibkriging,fit_s,0.00899458 -py01_kernels_trends,python,ishigami/exp/constant,pylibkriging,rmse,1.41632 -py01_kernels_trends,python,ishigami/exp/constant,pylibkriging,q2,0.845391 -py01_kernels_trends,python,ishigami/exp/constant,pylibkriging,nlpd,1.75239 -py01_kernels_trends,python,ishigami/exp/linear,pylibkriging,fit_s,0.0115979 -py01_kernels_trends,python,ishigami/exp/linear,pylibkriging,rmse,1.4095 -py01_kernels_trends,python,ishigami/exp/linear,pylibkriging,q2,0.846877 -py01_kernels_trends,python,ishigami/exp/linear,pylibkriging,nlpd,1.74779 -py01_kernels_trends,python,ishigami/exp/interactive,pylibkriging,fit_s,0.00745058 -py01_kernels_trends,python,ishigami/exp/interactive,pylibkriging,rmse,1.41511 -py01_kernels_trends,python,ishigami/exp/interactive,pylibkriging,q2,0.845657 -py01_kernels_trends,python,ishigami/exp/interactive,pylibkriging,nlpd,1.75024 -py01_kernels_trends,python,ishigami/exp/quadratic,pylibkriging,fit_s,0.00843048 -py01_kernels_trends,python,ishigami/exp/quadratic,pylibkriging,rmse,1.40329 -py01_kernels_trends,python,ishigami/exp/quadratic,pylibkriging,q2,0.848223 -py01_kernels_trends,python,ishigami/exp/quadratic,pylibkriging,nlpd,1.73254 -py01_kernels_trends,python,ishigami/matern3_2/constant,pylibkriging,fit_s,0.0480943 -py01_kernels_trends,python,ishigami/matern3_2/constant,pylibkriging,rmse,1.32971 -py01_kernels_trends,python,ishigami/matern3_2/constant,pylibkriging,q2,0.863723 -py01_kernels_trends,python,ishigami/matern3_2/constant,pylibkriging,nlpd,1.50265 -py01_kernels_trends,python,ishigami/matern3_2/linear,pylibkriging,fit_s,0.047653 -py01_kernels_trends,python,ishigami/matern3_2/linear,pylibkriging,rmse,1.30244 -py01_kernels_trends,python,ishigami/matern3_2/linear,pylibkriging,q2,0.869255 -py01_kernels_trends,python,ishigami/matern3_2/linear,pylibkriging,nlpd,1.47906 -py01_kernels_trends,python,ishigami/matern3_2/interactive,pylibkriging,fit_s,0.060349 -py01_kernels_trends,python,ishigami/matern3_2/interactive,pylibkriging,rmse,1.30591 -py01_kernels_trends,python,ishigami/matern3_2/interactive,pylibkriging,q2,0.868558 -py01_kernels_trends,python,ishigami/matern3_2/interactive,pylibkriging,nlpd,1.47384 -py01_kernels_trends,python,ishigami/matern3_2/quadratic,pylibkriging,fit_s,0.0471144 -py01_kernels_trends,python,ishigami/matern3_2/quadratic,pylibkriging,rmse,1.32197 -py01_kernels_trends,python,ishigami/matern3_2/quadratic,pylibkriging,q2,0.865305 -py01_kernels_trends,python,ishigami/matern3_2/quadratic,pylibkriging,nlpd,1.46827 -py01_kernels_trends,python,ishigami/matern5_2/constant,pylibkriging,fit_s,0.0145204 -py01_kernels_trends,python,ishigami/matern5_2/constant,pylibkriging,rmse,3.00566 -py01_kernels_trends,python,ishigami/matern5_2/constant,pylibkriging,q2,0.303709 -py01_kernels_trends,python,ishigami/matern5_2/constant,pylibkriging,nlpd,3.42085 -py01_kernels_trends,python,ishigami/matern5_2/constant,pylibkriging BFGS10,rmse,1.33903 -py01_kernels_trends,python,ishigami/matern5_2/constant,pylibkriging BFGS10,q2,0.861806 -py01_kernels_trends,python,ishigami/matern5_2/constant,pylibkriging BFGS10,nlpd,1.66132 -py01_kernels_trends,python,ishigami/matern5_2/linear,pylibkriging,fit_s,0.0518982 -py01_kernels_trends,python,ishigami/matern5_2/linear,pylibkriging,rmse,1.3434 -py01_kernels_trends,python,ishigami/matern5_2/linear,pylibkriging,q2,0.860901 -py01_kernels_trends,python,ishigami/matern5_2/linear,pylibkriging,nlpd,1.65103 -py01_kernels_trends,python,ishigami/matern5_2/interactive,pylibkriging,fit_s,0.0539804 -py01_kernels_trends,python,ishigami/matern5_2/interactive,pylibkriging,rmse,1.34479 -py01_kernels_trends,python,ishigami/matern5_2/interactive,pylibkriging,q2,0.860615 -py01_kernels_trends,python,ishigami/matern5_2/interactive,pylibkriging,nlpd,1.64153 -py01_kernels_trends,python,ishigami/matern5_2/quadratic,pylibkriging,fit_s,0.052948 -py01_kernels_trends,python,ishigami/matern5_2/quadratic,pylibkriging,rmse,1.37082 -py01_kernels_trends,python,ishigami/matern5_2/quadratic,pylibkriging,q2,0.855166 -py01_kernels_trends,python,ishigami/matern5_2/quadratic,pylibkriging,nlpd,1.62228 -py01_kernels_trends,python,hartmann3/gauss/constant,pylibkriging,fit_s,0.00805306 -py01_kernels_trends,python,hartmann3/gauss/constant,pylibkriging,rmse,0.156226 -py01_kernels_trends,python,hartmann3/gauss/constant,pylibkriging,q2,0.971116 -py01_kernels_trends,python,hartmann3/gauss/constant,pylibkriging,nlpd,-0.555623 -py01_kernels_trends,python,hartmann3/gauss/linear,pylibkriging,fit_s,0.0129044 -py01_kernels_trends,python,hartmann3/gauss/linear,pylibkriging,rmse,0.153297 -py01_kernels_trends,python,hartmann3/gauss/linear,pylibkriging,q2,0.972189 -py01_kernels_trends,python,hartmann3/gauss/linear,pylibkriging,nlpd,-0.568404 -py01_kernels_trends,python,hartmann3/gauss/interactive,pylibkriging,fit_s,0.00773382 -py01_kernels_trends,python,hartmann3/gauss/interactive,pylibkriging,rmse,0.712948 -py01_kernels_trends,python,hartmann3/gauss/interactive,pylibkriging,q2,0.398455 -py01_kernels_trends,python,hartmann3/gauss/interactive,pylibkriging,nlpd,1.74554 -py01_kernels_trends,python,hartmann3/gauss/quadratic,pylibkriging,fit_s,0.00593877 -py01_kernels_trends,python,hartmann3/gauss/quadratic,pylibkriging,rmse,0.73238 -py01_kernels_trends,python,hartmann3/gauss/quadratic,pylibkriging,q2,0.365217 -py01_kernels_trends,python,hartmann3/gauss/quadratic,pylibkriging,nlpd,2.14587 -py01_kernels_trends,python,hartmann3/exp/constant,pylibkriging,fit_s,0.00374722 -py01_kernels_trends,python,hartmann3/exp/constant,pylibkriging,rmse,0.347348 -py01_kernels_trends,python,hartmann3/exp/constant,pylibkriging,q2,0.857215 -py01_kernels_trends,python,hartmann3/exp/constant,pylibkriging,nlpd,0.352683 -py01_kernels_trends,python,hartmann3/exp/linear,pylibkriging,fit_s,0.00216174 -py01_kernels_trends,python,hartmann3/exp/linear,pylibkriging,rmse,0.347273 -py01_kernels_trends,python,hartmann3/exp/linear,pylibkriging,q2,0.857276 -py01_kernels_trends,python,hartmann3/exp/linear,pylibkriging,nlpd,0.343631 -py01_kernels_trends,python,hartmann3/exp/interactive,pylibkriging,fit_s,0.0024302 -py01_kernels_trends,python,hartmann3/exp/interactive,pylibkriging,rmse,0.378349 -py01_kernels_trends,python,hartmann3/exp/interactive,pylibkriging,q2,0.83059 -py01_kernels_trends,python,hartmann3/exp/interactive,pylibkriging,nlpd,0.385842 -py01_kernels_trends,python,hartmann3/exp/quadratic,pylibkriging,fit_s,0.0039556 -py01_kernels_trends,python,hartmann3/exp/quadratic,pylibkriging,rmse,0.355119 -py01_kernels_trends,python,hartmann3/exp/quadratic,pylibkriging,q2,0.850755 -py01_kernels_trends,python,hartmann3/exp/quadratic,pylibkriging,nlpd,0.334371 -py01_kernels_trends,python,hartmann3/matern3_2/constant,pylibkriging,fit_s,0.0110524 -py01_kernels_trends,python,hartmann3/matern3_2/constant,pylibkriging,rmse,0.227521 -py01_kernels_trends,python,hartmann3/matern3_2/constant,pylibkriging,q2,0.938737 -py01_kernels_trends,python,hartmann3/matern3_2/constant,pylibkriging,nlpd,-0.181207 -py01_kernels_trends,python,hartmann3/matern3_2/linear,pylibkriging,fit_s,0.00763154 -py01_kernels_trends,python,hartmann3/matern3_2/linear,pylibkriging,rmse,0.23038 -py01_kernels_trends,python,hartmann3/matern3_2/linear,pylibkriging,q2,0.937188 -py01_kernels_trends,python,hartmann3/matern3_2/linear,pylibkriging,nlpd,-0.162203 -py01_kernels_trends,python,hartmann3/matern3_2/interactive,pylibkriging,fit_s,0.00412679 -py01_kernels_trends,python,hartmann3/matern3_2/interactive,pylibkriging,rmse,0.315752 -py01_kernels_trends,python,hartmann3/matern3_2/interactive,pylibkriging,q2,0.88201 -py01_kernels_trends,python,hartmann3/matern3_2/interactive,pylibkriging,nlpd,0.898253 -py01_kernels_trends,python,hartmann3/matern3_2/quadratic,pylibkriging,fit_s,0.00620675 -py01_kernels_trends,python,hartmann3/matern3_2/quadratic,pylibkriging,rmse,0.291393 -py01_kernels_trends,python,hartmann3/matern3_2/quadratic,pylibkriging,q2,0.899513 -py01_kernels_trends,python,hartmann3/matern3_2/quadratic,pylibkriging,nlpd,0.53743 -py01_kernels_trends,python,hartmann3/matern5_2/constant,pylibkriging,fit_s,0.00997186 -py01_kernels_trends,python,hartmann3/matern5_2/constant,pylibkriging,rmse,0.190287 -py01_kernels_trends,python,hartmann3/matern5_2/constant,pylibkriging,q2,0.957148 -py01_kernels_trends,python,hartmann3/matern5_2/constant,pylibkriging,nlpd,-0.403054 -py01_kernels_trends,python,hartmann3/matern5_2/constant,pylibkriging BFGS10,rmse,0.190286 -py01_kernels_trends,python,hartmann3/matern5_2/constant,pylibkriging BFGS10,q2,0.957148 -py01_kernels_trends,python,hartmann3/matern5_2/constant,pylibkriging BFGS10,nlpd,-0.40306 -py01_kernels_trends,python,hartmann3/matern5_2/linear,pylibkriging,fit_s,0.0127335 -py01_kernels_trends,python,hartmann3/matern5_2/linear,pylibkriging,rmse,0.189633 -py01_kernels_trends,python,hartmann3/matern5_2/linear,pylibkriging,q2,0.957442 -py01_kernels_trends,python,hartmann3/matern5_2/linear,pylibkriging,nlpd,-0.410075 -py01_kernels_trends,python,hartmann3/matern5_2/interactive,pylibkriging,fit_s,0.0101018 -py01_kernels_trends,python,hartmann3/matern5_2/interactive,pylibkriging,rmse,0.307114 -py01_kernels_trends,python,hartmann3/matern5_2/interactive,pylibkriging,q2,0.888377 -py01_kernels_trends,python,hartmann3/matern5_2/interactive,pylibkriging,nlpd,1.50063 -py01_kernels_trends,python,hartmann3/matern5_2/quadratic,pylibkriging,fit_s,0.0101326 -py01_kernels_trends,python,hartmann3/matern5_2/quadratic,pylibkriging,rmse,0.285355 -py01_kernels_trends,python,hartmann3/matern5_2/quadratic,pylibkriging,q2,0.903634 -py01_kernels_trends,python,hartmann3/matern5_2/quadratic,pylibkriging,nlpd,0.898128 -py02_objectives_gradients,python,branin/LL,pylibkriging,rmse,0.833715 -py02_objectives_gradients,python,branin/LL,pylibkriging,q2,0.999729 -py02_objectives_gradients,python,branin/LL,pylibkriging,nlpd,0.697034 -py02_objectives_gradients,python,branin/LOO,pylibkriging,rmse,0.550995 -py02_objectives_gradients,python,branin/LOO,pylibkriging,q2,0.999882 -py02_objectives_gradients,python,branin/LOO,pylibkriging,nlpd,0.471613 -py02_objectives_gradients,python,branin/LMP,pylibkriging,rmse,0.66967 -py02_objectives_gradients,python,branin/LMP,pylibkriging,q2,0.999825 -py02_objectives_gradients,python,branin/LMP,pylibkriging,nlpd,0.617527 -py02_objectives_gradients,python,hartmann3/LL,pylibkriging,rmse,0.157276 -py02_objectives_gradients,python,hartmann3/LL,pylibkriging,q2,0.970945 -py02_objectives_gradients,python,hartmann3/LL,pylibkriging,nlpd,-0.497548 -py02_objectives_gradients,python,hartmann3/LOO,pylibkriging,rmse,0.144982 -py02_objectives_gradients,python,hartmann3/LOO,pylibkriging,q2,0.97531 -py02_objectives_gradients,python,hartmann3/LOO,pylibkriging,nlpd,-0.584885 -py02_objectives_gradients,python,hartmann3/LMP,pylibkriging,rmse,0.150075 -py02_objectives_gradients,python,hartmann3/LMP,pylibkriging,q2,0.973544 -py02_objectives_gradients,python,hartmann3/LMP,pylibkriging,nlpd,-0.528379 -py02_objectives_gradients,python,ishigami/LL,pylibkriging,rmse,3.41501 -py02_objectives_gradients,python,ishigami/LL,pylibkriging,q2,0.203357 -py02_objectives_gradients,python,ishigami/LL,pylibkriging,nlpd,4.86258 -py02_objectives_gradients,python,ishigami/LOO,pylibkriging,rmse,2.45853 -py02_objectives_gradients,python,ishigami/LOO,pylibkriging,q2,0.587112 -py02_objectives_gradients,python,ishigami/LOO,pylibkriging,nlpd,2.72077 -py02_objectives_gradients,python,ishigami/LMP,pylibkriging,rmse,3.47385 -py02_objectives_gradients,python,ishigami/LMP,pylibkriging,q2,0.175667 -py02_objectives_gradients,python,ishigami/LMP,pylibkriging,nlpd,5.27033 -py02_objectives_gradients,python,friedman/LL,pylibkriging,rmse,0.273164 -py02_objectives_gradients,python,friedman/LL,pylibkriging,q2,0.996855 -py02_objectives_gradients,python,friedman/LL,pylibkriging,nlpd,-0.118408 -py02_objectives_gradients,python,friedman/LOO,pylibkriging,rmse,0.233921 -py02_objectives_gradients,python,friedman/LOO,pylibkriging,q2,0.997694 -py02_objectives_gradients,python,friedman/LOO,pylibkriging,nlpd,-0.169369 -py02_objectives_gradients,python,friedman/LMP,pylibkriging,rmse,0.269913 -py02_objectives_gradients,python,friedman/LMP,pylibkriging,q2,0.996929 -py02_objectives_gradients,python,friedman/LMP,pylibkriging,nlpd,-0.122709 -py03_noise,python,forrester,nugget,rmse,0.198091 -py03_noise,python,forrester,nugget,q2,0.998087 -py03_noise,python,forrester,nugget,nlpd,0.286631 -py03_noise,python,forrester,noise-free fit,rmse,0.487916 -py03_noise,python,forrester,noise-free fit,q2,0.988392 -py03_noise,python,forrester,noise-free fit,nlpd,862.67 -py03_noise,python,friedman,nugget,rmse,0.467335 -py03_noise,python,friedman,nugget,q2,0.99089 -py03_noise,python,friedman,nugget,nlpd,1.07073 -py03_noise,python,friedman,noise-free fit,rmse,1.27157 -py03_noise,python,friedman,noise-free fit,q2,0.932558 -py03_noise,python,friedman,noise-free fit,nlpd,1.64497 -py03_noise,python,gramacy_lee_2012/hetero,known variances,rmse,0.168741 -py03_noise,python,gramacy_lee_2012/hetero,known variances,q2,0.982515 -py03_noise,python,gramacy_lee_2012/hetero,known variances,nlpd,-0.570574 -py03_noise,python,gramacy_lee_2012/hetero,nugget,rmse,0.170945 -py03_noise,python,gramacy_lee_2012/hetero,nugget,q2,0.982055 -py03_noise,python,gramacy_lee_2012/hetero,nugget,nlpd,-0.221568 -py05_warpkriging,python,gl2012,Kriging,rmse,0.167263 -py05_warpkriging,python,gl2012,Kriging,q2,0.981176 -py05_warpkriging,python,gl2012,Kriging,nlpd,-1.08917 -py05_warpkriging,python,gl2012,affine,rmse,0.167263 -py05_warpkriging,python,gl2012,affine,q2,0.981176 -py05_warpkriging,python,gl2012,affine,nlpd,-1.08917 -py05_warpkriging,python,gl2012,boxcox,rmse,0.16759 -py05_warpkriging,python,gl2012,boxcox,q2,0.981103 -py05_warpkriging,python,gl2012,boxcox,nlpd,282.422 -py05_warpkriging,python,gl2012,kumaraswamy,rmse,0.168286 -py05_warpkriging,python,gl2012,kumaraswamy,q2,0.980945 -py05_warpkriging,python,gl2012,kumaraswamy,nlpd,2.04577e+06 -py05_warpkriging,python,gl2012,knots(5),rmse,0.16791 -py05_warpkriging,python,gl2012,knots(5),q2,0.98103 -py05_warpkriging,python,gl2012,knots(5),nlpd,2.04376e+06 -py05_warpkriging,python,gl2012,neural_mono(8),rmse,0.170824 -py05_warpkriging,python,gl2012,neural_mono(8),q2,0.980366 -py05_warpkriging,python,gl2012,neural_mono(8),nlpd,-0.265141 -py05_warpkriging,python,gl2008,Kriging,rmse,0.0534649 -py05_warpkriging,python,gl2008,Kriging,q2,0.585636 -py05_warpkriging,python,gl2008,Kriging,nlpd,-1.74569 -py05_warpkriging,python,gl2008,kumaraswamy,rmse,0.0535298 -py05_warpkriging,python,gl2008,kumaraswamy,q2,0.58463 -py05_warpkriging,python,gl2008,kumaraswamy,nlpd,-1.74552 -py05_warpkriging,python,mixed,Kriging (level as number),rmse,16.7693 -py05_warpkriging,python,mixed,Kriging (level as number),q2,0.838452 -py05_warpkriging,python,mixed,Kriging (level as number),nlpd,3.81063 -py05_warpkriging,python,mixed,"categorical(5,2)",rmse,16.1438 -py05_warpkriging,python,mixed,"categorical(5,2)",q2,0.85028 -py05_warpkriging,python,mixed,"categorical(5,2)",nlpd,3.3206 -py05_warpkriging,python,ordinal,Kriging (rank as number),rmse,0.213845 -py05_warpkriging,python,ordinal,Kriging (rank as number),q2,0.953177 -py05_warpkriging,python,ordinal,Kriging (rank as number),nlpd,0.810969 -py05_warpkriging,python,ordinal,ordinal(6),rmse,0.213043 -py05_warpkriging,python,ordinal,ordinal(6),q2,0.953528 -py05_warpkriging,python,ordinal,ordinal(6),nlpd,0.78888 -py06_high_dimension_mlp,python,otl_circuit,unit cube,fit_s,0.00789523 -py06_high_dimension_mlp,python,otl_circuit,unit cube,rmse,0.0150514 -py06_high_dimension_mlp,python,otl_circuit,unit cube,q2,0.999836 -py06_high_dimension_mlp,python,otl_circuit,unit cube,nlpd,-2.90452 -py06_high_dimension_mlp,python,otl_circuit,native + normalize,rmse,0.0150514 -py06_high_dimension_mlp,python,otl_circuit,native + normalize,q2,0.999836 -py06_high_dimension_mlp,python,otl_circuit,native + normalize,nlpd,-2.90452 -py06_high_dimension_mlp,python,piston,unit cube,fit_s,0.0197392 -py06_high_dimension_mlp,python,piston,unit cube,rmse,0.0104258 -py06_high_dimension_mlp,python,piston,unit cube,q2,0.994462 -py06_high_dimension_mlp,python,piston,unit cube,nlpd,-3.49038 -py06_high_dimension_mlp,python,piston,native + normalize,rmse,0.0104258 -py06_high_dimension_mlp,python,piston,native + normalize,q2,0.994462 -py06_high_dimension_mlp,python,piston,native + normalize,nlpd,-3.49038 -py06_high_dimension_mlp,python,borehole,unit cube,fit_s,0.0140877 -py06_high_dimension_mlp,python,borehole,unit cube,rmse,0.893341 -py06_high_dimension_mlp,python,borehole,unit cube,q2,0.999599 -py06_high_dimension_mlp,python,borehole,unit cube,nlpd,1.08433 -py06_high_dimension_mlp,python,borehole,native + normalize,rmse,0.893341 -py06_high_dimension_mlp,python,borehole,native + normalize,q2,0.999599 -py06_high_dimension_mlp,python,borehole,native + normalize,nlpd,1.08433 -py06_high_dimension_mlp,python,robot_arm,unit cube,fit_s,0.167482 -py06_high_dimension_mlp,python,robot_arm,unit cube,rmse,0.358507 -py06_high_dimension_mlp,python,robot_arm,unit cube,q2,0.571154 -py06_high_dimension_mlp,python,robot_arm,unit cube,nlpd,0.355106 -py06_high_dimension_mlp,python,robot_arm,native + normalize,rmse,0.358507 -py06_high_dimension_mlp,python,robot_arm,native + normalize,q2,0.571154 -py06_high_dimension_mlp,python,robot_arm,native + normalize,nlpd,0.355106 -py06_high_dimension_mlp,python,dette_pepelyshev,unit cube,fit_s,0.0246282 -py06_high_dimension_mlp,python,dette_pepelyshev,unit cube,rmse,0.424908 -py06_high_dimension_mlp,python,dette_pepelyshev,unit cube,q2,0.997757 -py06_high_dimension_mlp,python,dette_pepelyshev,unit cube,nlpd,0.510276 -py06_high_dimension_mlp,python,dette_pepelyshev,native + normalize,rmse,0.424908 -py06_high_dimension_mlp,python,dette_pepelyshev,native + normalize,q2,0.997757 -py06_high_dimension_mlp,python,dette_pepelyshev,native + normalize,nlpd,0.510276 -py06_high_dimension_mlp,python,wing_weight,unit cube,fit_s,0.0245481 -py06_high_dimension_mlp,python,wing_weight,unit cube,rmse,0.809743 -py06_high_dimension_mlp,python,wing_weight,unit cube,q2,0.999716 -py06_high_dimension_mlp,python,wing_weight,unit cube,nlpd,1.07161 -py06_high_dimension_mlp,python,wing_weight,native + normalize,rmse,0.809743 -py06_high_dimension_mlp,python,wing_weight,native + normalize,q2,0.999716 -py06_high_dimension_mlp,python,wing_weight,native + normalize,nlpd,1.07161 -py06_high_dimension_mlp,python,welch,unit cube,fit_s,0.564758 -py06_high_dimension_mlp,python,welch,unit cube,rmse,0.167729 -py06_high_dimension_mlp,python,welch,unit cube,q2,0.993452 -py06_high_dimension_mlp,python,welch,unit cube,nlpd,-0.429138 -py06_high_dimension_mlp,python,welch,native + normalize,rmse,0.167692 -py06_high_dimension_mlp,python,welch,native + normalize,q2,0.993454 -py06_high_dimension_mlp,python,welch,native + normalize,nlpd,-0.429291 -py06_high_dimension_mlp,python,projected_branin_10d,Kriging,rmse,4.42284 -py06_high_dimension_mlp,python,projected_branin_10d,Kriging,q2,0.959122 -py06_high_dimension_mlp,python,projected_branin_10d,Kriging,nlpd,2.89997 -py06_high_dimension_mlp,python,projected_branin_10d,MLPKriging tanh BFGS,fit_s,0.130867 -py06_high_dimension_mlp,python,projected_branin_10d,MLPKriging tanh BFGS,rmse,57.402 -py06_high_dimension_mlp,python,projected_branin_10d,MLPKriging tanh BFGS,q2,-5.88553 -py06_high_dimension_mlp,python,projected_branin_10d,MLPKriging tanh BFGS,nlpd,136.771 -py06_high_dimension_mlp,python,projected_branin_10d,MLPKriging tanh BFGS10,fit_s,0.442791 -py06_high_dimension_mlp,python,projected_branin_10d,MLPKriging tanh BFGS10,rmse,4.28606 -py06_high_dimension_mlp,python,projected_branin_10d,MLPKriging tanh BFGS10,q2,0.961612 -py06_high_dimension_mlp,python,projected_branin_10d,MLPKriging tanh BFGS10,nlpd,29.0854 -py06_high_dimension_mlp,python,projected_branin_10d,MLPKriging selu BFGS,fit_s,0.139126 -py06_high_dimension_mlp,python,projected_branin_10d,MLPKriging selu BFGS,rmse,58.9913 -py06_high_dimension_mlp,python,projected_branin_10d,MLPKriging selu BFGS,q2,-6.2721 -py06_high_dimension_mlp,python,projected_branin_10d,MLPKriging selu BFGS,nlpd,46.9788 -py06_high_dimension_mlp,python,projected_branin_10d,MLPKriging selu BFGS10,fit_s,0.440179 -py06_high_dimension_mlp,python,projected_branin_10d,MLPKriging selu BFGS10,rmse,4.39195 -py06_high_dimension_mlp,python,projected_branin_10d,MLPKriging selu BFGS10,q2,0.959691 -py06_high_dimension_mlp,python,projected_branin_10d,MLPKriging selu BFGS10,nlpd,10.6053 -py07_scalability,python,hartmann3/n=1500,exact,fit_s,11.1133 -py07_scalability,python,hartmann3/n=1500,exact,rmse,0.000259233 -py07_scalability,python,hartmann3/n=1500,exact,q2,1 -py07_scalability,python,hartmann3/n=1500,exact,nlpd,-6.92184 -py07_scalability,python,hartmann3/n=1500,vecchia(30),fit_s,5.43768 -py07_scalability,python,hartmann3/n=1500,vecchia(30),rmse,0.00353225 -py07_scalability,python,hartmann3/n=1500,vecchia(30),q2,0.999987 -py07_scalability,python,hartmann3/n=1500,vecchia(30),nlpd,-4.6266 -py07_scalability,python,hartmann3/n=5000,vecchia(30),fit_s,16.2077 -py07_scalability,python,hartmann3/n=5000,vecchia(30),rmse,0.000851738 -py07_scalability,python,hartmann3/n=5000,vecchia(30),q2,0.999999 -py07_scalability,python,hartmann3/n=5000,vecchia(30),nlpd,-5.95078 -py07_scalability,python,hartmann3/n=1500,nystrom(100),fit_s,2.04779 -py07_scalability,python,hartmann3/n=1500,nystrom(100),rmse,1.51127 -py07_scalability,python,hartmann3/n=1500,nystrom(100),q2,-1.39237 -py07_scalability,python,hartmann3/n=1500,nystrom(100),nlpd,1.14197e+24 -py07_scalability,python,hartmann3/n=5000,nystrom(100),fit_s,3.25622 -py07_scalability,python,hartmann3/n=5000,nystrom(100),rmse,9.95398 -py07_scalability,python,hartmann3/n=5000,nystrom(100),q2,-102.786 -py07_scalability,python,hartmann3/n=5000,nystrom(100),nlpd,4.95409e+25 -py07_scalability,python,hartmann3/n=1500,nested NK (20 groups),fit_s,0.269932 -py07_scalability,python,hartmann3/n=1500,nested NK (20 groups),rmse,0.0029908 -py07_scalability,python,hartmann3/n=1500,nested NK (20 groups),q2,0.999991 -py07_scalability,python,hartmann3/n=1500,nested NK (20 groups),nlpd,-4.85946 -py07_scalability,python,hartmann3/n=5000,nested NK (20 groups),fit_s,2.87469 -py07_scalability,python,hartmann3/n=5000,nested NK (20 groups),rmse,0.000222655 -py07_scalability,python,hartmann3/n=5000,nested NK (20 groups),q2,1 -py07_scalability,python,hartmann3/n=5000,nested NK (20 groups),nlpd,-7.44453 -py07_scalability,python,hartmann3/n=1500,nested rBCM (20 groups),fit_s,0.25767 -py07_scalability,python,hartmann3/n=1500,nested rBCM (20 groups),rmse,0.00304431 -py07_scalability,python,hartmann3/n=1500,nested rBCM (20 groups),q2,0.99999 -py07_scalability,python,hartmann3/n=1500,nested rBCM (20 groups),nlpd,-4.50201 -py07_scalability,python,hartmann3/n=5000,nested rBCM (20 groups),fit_s,2.5369 -py07_scalability,python,hartmann3/n=5000,nested rBCM (20 groups),rmse,0.000224517 -py07_scalability,python,hartmann3/n=5000,nested rBCM (20 groups),q2,1 -py07_scalability,python,hartmann3/n=5000,nested rBCM (20 groups),nlpd,-7.18182 -py07_scalability,python,hartmann3/n=1500,subset of data (500),fit_s,0.712767 -py07_scalability,python,hartmann3/n=1500,subset of data (500),rmse,0.00269153 -py07_scalability,python,hartmann3/n=1500,subset of data (500),q2,0.999992 -py07_scalability,python,hartmann3/n=1500,subset of data (500),nlpd,-4.68641 -py07_scalability,python,hartmann3/n=5000,subset of data (500),fit_s,0.950298 -py07_scalability,python,hartmann3/n=5000,subset of data (500),rmse,0.00247552 -py07_scalability,python,hartmann3/n=5000,subset of data (500),q2,0.999994 -py07_scalability,python,hartmann3/n=5000,subset of data (500),nlpd,-4.71359 -py07_scalability,python,otl_circuit/n=1500,exact,fit_s,10.6103 -py07_scalability,python,otl_circuit/n=1500,exact,rmse,9.81598e-05 -py07_scalability,python,otl_circuit/n=1500,exact,q2,1 -py07_scalability,python,otl_circuit/n=1500,exact,nlpd,-8.21962 -py07_scalability,python,otl_circuit/n=1500,vecchia(30),fit_s,4.73217 -py07_scalability,python,otl_circuit/n=1500,vecchia(30),rmse,0.00629979 -py07_scalability,python,otl_circuit/n=1500,vecchia(30),q2,0.999969 -py07_scalability,python,otl_circuit/n=1500,vecchia(30),nlpd,-3.95617 -py07_scalability,python,otl_circuit/n=5000,vecchia(30),fit_s,14.5549 -py07_scalability,python,otl_circuit/n=5000,vecchia(30),rmse,0.00312888 -py07_scalability,python,otl_circuit/n=5000,vecchia(30),q2,0.999992 -py07_scalability,python,otl_circuit/n=5000,vecchia(30),nlpd,-4.57612 -py07_scalability,python,otl_circuit/n=1500,nystrom(100),fit_s,1.47345 -py07_scalability,python,otl_circuit/n=1500,nystrom(100),rmse,0.392811 -py07_scalability,python,otl_circuit/n=1500,nystrom(100),q2,0.880908 -py07_scalability,python,otl_circuit/n=1500,nystrom(100),nlpd,7.06156e+22 -py07_scalability,python,otl_circuit/n=5000,nystrom(100),fit_s,4.69579 -py07_scalability,python,otl_circuit/n=5000,nystrom(100),rmse,1.11526 -py07_scalability,python,otl_circuit/n=5000,nystrom(100),q2,0.0400057 -py07_scalability,python,otl_circuit/n=5000,nystrom(100),nlpd,5.31511e+23 -py07_scalability,python,otl_circuit/n=1500,nested NK (20 groups),fit_s,0.278129 -py07_scalability,python,otl_circuit/n=1500,nested NK (20 groups),rmse,0.00226873 -py07_scalability,python,otl_circuit/n=1500,nested NK (20 groups),q2,0.999996 -py07_scalability,python,otl_circuit/n=1500,nested NK (20 groups),nlpd,-4.90542 -py07_scalability,python,otl_circuit/n=5000,nested NK (20 groups),fit_s,3.62576 -py07_scalability,python,otl_circuit/n=5000,nested NK (20 groups),rmse,0.000261686 -py07_scalability,python,otl_circuit/n=5000,nested NK (20 groups),q2,1 -py07_scalability,python,otl_circuit/n=5000,nested NK (20 groups),nlpd,-7.20623 -py07_scalability,python,otl_circuit/n=1500,nested rBCM (20 groups),fit_s,0.315619 -py07_scalability,python,otl_circuit/n=1500,nested rBCM (20 groups),rmse,0.00291869 -py07_scalability,python,otl_circuit/n=1500,nested rBCM (20 groups),q2,0.999993 -py07_scalability,python,otl_circuit/n=1500,nested rBCM (20 groups),nlpd,-2.55512 -py07_scalability,python,otl_circuit/n=5000,nested rBCM (20 groups),fit_s,3.39037 -py07_scalability,python,otl_circuit/n=5000,nested rBCM (20 groups),rmse,0.000278149 -py07_scalability,python,otl_circuit/n=5000,nested rBCM (20 groups),q2,1 -py07_scalability,python,otl_circuit/n=5000,nested rBCM (20 groups),nlpd,-4.34927 -py07_scalability,python,otl_circuit/n=1500,subset of data (500),fit_s,0.672431 -py07_scalability,python,otl_circuit/n=1500,subset of data (500),rmse,0.0004286 -py07_scalability,python,otl_circuit/n=1500,subset of data (500),q2,1 -py07_scalability,python,otl_circuit/n=1500,subset of data (500),nlpd,-6.72678 -py07_scalability,python,otl_circuit/n=5000,subset of data (500),fit_s,0.692672 -py07_scalability,python,otl_circuit/n=5000,subset of data (500),rmse,0.000446879 -py07_scalability,python,otl_circuit/n=5000,subset of data (500),q2,1 -py07_scalability,python,otl_circuit/n=5000,subset of data (500),nlpd,-6.78844 -py07_scalability,python,branin/nested,NK,rmse,0.0343984 -py07_scalability,python,branin/nested,NK,q2,1 -py07_scalability,python,branin/nested,NK,nlpd,2.00082e+14 -py07_scalability,python,branin/nested,PoE,rmse,0.0354265 -py07_scalability,python,branin/nested,PoE,q2,1 -py07_scalability,python,branin/nested,PoE,nlpd,-1.64533 -py07_scalability,python,branin/nested,gPoE,rmse,0.0354265 -py07_scalability,python,branin/nested,gPoE,q2,1 -py07_scalability,python,branin/nested,gPoE,nlpd,-0.621836 -py07_scalability,python,branin/nested,BCM,rmse,0.0354196 -py07_scalability,python,branin/nested,BCM,q2,1 -py07_scalability,python,branin/nested,BCM,nlpd,-1.64533 -py07_scalability,python,branin/nested,rBCM,rmse,0.035287 -py07_scalability,python,branin/nested,rBCM,q2,1 -py07_scalability,python,branin/nested,rBCM,nlpd,-2.61369 -py10_compare_python_packages,python,branin,pylibkriging,q2,0.999944 -py10_compare_python_packages,python,branin,pylibkriging,nlpd,0.137225 -py10_compare_python_packages,python,branin,pylibkriging,fit_s,0.00261593 -py10_compare_python_packages,python,branin,pylibkriging BFGS10,q2,0.999945 -py10_compare_python_packages,python,branin,pylibkriging BFGS10,nlpd,0.137254 -py10_compare_python_packages,python,branin,pylibkriging BFGS10,fit_s,0.02301 -py10_compare_python_packages,python,branin,scikit-learn,q2,0.999597 -py10_compare_python_packages,python,branin,scikit-learn,nlpd,0.928508 -py10_compare_python_packages,python,branin,scikit-learn,fit_s,0.169693 -py10_compare_python_packages,python,branin,SMT,q2,0.999944 -py10_compare_python_packages,python,branin,SMT,nlpd,0.137259 -py10_compare_python_packages,python,branin,SMT,fit_s,0.833295 -py10_compare_python_packages,python,branin,OpenTURNS,q2,0.998221 -py10_compare_python_packages,python,branin,OpenTURNS,nlpd,1.55599 -py10_compare_python_packages,python,branin,OpenTURNS,fit_s,0.00490117 -py10_compare_python_packages,python,six_hump_camel,pylibkriging,q2,0.999418 -py10_compare_python_packages,python,six_hump_camel,pylibkriging,nlpd,0.0429342 -py10_compare_python_packages,python,six_hump_camel,pylibkriging,fit_s,0.00324678 -py10_compare_python_packages,python,six_hump_camel,pylibkriging BFGS10,q2,0.999418 -py10_compare_python_packages,python,six_hump_camel,pylibkriging BFGS10,nlpd,0.0429386 -py10_compare_python_packages,python,six_hump_camel,pylibkriging BFGS10,fit_s,0.060586 -py10_compare_python_packages,python,six_hump_camel,scikit-learn,q2,0.0163311 -py10_compare_python_packages,python,six_hump_camel,scikit-learn,nlpd,4.63763 -py10_compare_python_packages,python,six_hump_camel,scikit-learn,fit_s,0.206083 -py10_compare_python_packages,python,six_hump_camel,SMT,q2,0.999594 -py10_compare_python_packages,python,six_hump_camel,SMT,nlpd,-0.121007 -py10_compare_python_packages,python,six_hump_camel,SMT,fit_s,1.03758 -py10_compare_python_packages,python,six_hump_camel,OpenTURNS,q2,0.956713 -py10_compare_python_packages,python,six_hump_camel,OpenTURNS,nlpd,2.55446 -py10_compare_python_packages,python,six_hump_camel,OpenTURNS,fit_s,0.00741315 -py10_compare_python_packages,python,hartmann3,pylibkriging,q2,0.986148 -py10_compare_python_packages,python,hartmann3,pylibkriging,nlpd,-0.847247 -py10_compare_python_packages,python,hartmann3,pylibkriging,fit_s,0.0169713 -py10_compare_python_packages,python,hartmann3,pylibkriging BFGS10,q2,0.986148 -py10_compare_python_packages,python,hartmann3,pylibkriging BFGS10,nlpd,-0.847249 -py10_compare_python_packages,python,hartmann3,pylibkriging BFGS10,fit_s,0.139588 -py10_compare_python_packages,python,hartmann3,scikit-learn,q2,0.983889 -py10_compare_python_packages,python,hartmann3,scikit-learn,nlpd,-0.782697 -py10_compare_python_packages,python,hartmann3,scikit-learn,fit_s,0.188796 -py10_compare_python_packages,python,hartmann3,SMT,q2,0.986148 -py10_compare_python_packages,python,hartmann3,SMT,nlpd,-0.847249 -py10_compare_python_packages,python,hartmann3,SMT,fit_s,1.02907 -py10_compare_python_packages,python,hartmann3,OpenTURNS,q2,0.984712 -py10_compare_python_packages,python,hartmann3,OpenTURNS,nlpd,-0.809851 -py10_compare_python_packages,python,hartmann3,OpenTURNS,fit_s,0.0153861 -py10_compare_python_packages,python,ishigami,pylibkriging,q2,0.259556 -py10_compare_python_packages,python,ishigami,pylibkriging,nlpd,4.38991 -py10_compare_python_packages,python,ishigami,pylibkriging,fit_s,0.0256422 -py10_compare_python_packages,python,ishigami,pylibkriging BFGS10,q2,0.935722 -py10_compare_python_packages,python,ishigami,pylibkriging BFGS10,nlpd,0.857443 -py10_compare_python_packages,python,ishigami,pylibkriging BFGS10,fit_s,0.308839 -py10_compare_python_packages,python,ishigami,scikit-learn,q2,0.831142 -py10_compare_python_packages,python,ishigami,scikit-learn,nlpd,1.5455 -py10_compare_python_packages,python,ishigami,scikit-learn,fit_s,0.51733 -py10_compare_python_packages,python,ishigami,SMT,q2,0.51196 -py10_compare_python_packages,python,ishigami,SMT,nlpd,3.31349 -py10_compare_python_packages,python,ishigami,SMT,fit_s,2.77225 -py10_compare_python_packages,python,ishigami,OpenTURNS,q2,0.887251 -py10_compare_python_packages,python,ishigami,OpenTURNS,nlpd,1.39348 -py10_compare_python_packages,python,ishigami,OpenTURNS,fit_s,0.0499659 -py10_compare_python_packages,python,friedman,pylibkriging,q2,0.999771 -py10_compare_python_packages,python,friedman,pylibkriging,nlpd,-1.29726 -py10_compare_python_packages,python,friedman,pylibkriging,fit_s,0.0181558 -py10_compare_python_packages,python,friedman,pylibkriging BFGS10,q2,0.999771 -py10_compare_python_packages,python,friedman,pylibkriging BFGS10,nlpd,-1.29727 -py10_compare_python_packages,python,friedman,pylibkriging BFGS10,fit_s,0.215106 -py10_compare_python_packages,python,friedman,scikit-learn,q2,0.999754 -py10_compare_python_packages,python,friedman,scikit-learn,nlpd,-1.37702 -py10_compare_python_packages,python,friedman,scikit-learn,fit_s,0.283978 -py10_compare_python_packages,python,friedman,SMT,q2,0.999899 -py10_compare_python_packages,python,friedman,SMT,nlpd,-1.871 -py10_compare_python_packages,python,friedman,SMT,fit_s,4.3249 -py10_compare_python_packages,python,friedman,OpenTURNS,q2,0.990862 -py10_compare_python_packages,python,friedman,OpenTURNS,nlpd,0.469828 -py10_compare_python_packages,python,friedman,OpenTURNS,fit_s,0.0293689 -py10_compare_python_packages,python,otl_circuit,pylibkriging,q2,0.999922 -py10_compare_python_packages,python,otl_circuit,pylibkriging,nlpd,-3.46228 -py10_compare_python_packages,python,otl_circuit,pylibkriging,fit_s,0.013788 -py10_compare_python_packages,python,otl_circuit,pylibkriging BFGS10,q2,0.999922 -py10_compare_python_packages,python,otl_circuit,pylibkriging BFGS10,nlpd,-3.46227 -py10_compare_python_packages,python,otl_circuit,pylibkriging BFGS10,fit_s,0.141071 -py10_compare_python_packages,python,otl_circuit,scikit-learn,q2,0.999906 -py10_compare_python_packages,python,otl_circuit,scikit-learn,nlpd,-3.43062 -py10_compare_python_packages,python,otl_circuit,scikit-learn,fit_s,0.51218 -py10_compare_python_packages,python,otl_circuit,SMT,q2,0.999954 -py10_compare_python_packages,python,otl_circuit,SMT,nlpd,-3.35016 -py10_compare_python_packages,python,otl_circuit,SMT,fit_s,7.1224 -py10_compare_python_packages,python,otl_circuit,OpenTURNS,q2,0.99777 -py10_compare_python_packages,python,otl_circuit,OpenTURNS,nlpd,-1.61723 -py10_compare_python_packages,python,otl_circuit,OpenTURNS,fit_s,0.0603108 -py10_compare_python_packages,python,piston,pylibkriging,q2,0.997498 -py10_compare_python_packages,python,piston,pylibkriging,nlpd,-3.80667 -py10_compare_python_packages,python,piston,pylibkriging,fit_s,0.0395648 -py10_compare_python_packages,python,piston,pylibkriging BFGS10,q2,0.997498 -py10_compare_python_packages,python,piston,pylibkriging BFGS10,nlpd,-3.8067 -py10_compare_python_packages,python,piston,pylibkriging BFGS10,fit_s,0.327989 -py10_compare_python_packages,python,piston,scikit-learn,q2,0.998684 -py10_compare_python_packages,python,piston,scikit-learn,nlpd,-4.10497 -py10_compare_python_packages,python,piston,scikit-learn,fit_s,1.11386 -py10_compare_python_packages,python,piston,SMT,q2,0.997511 -py10_compare_python_packages,python,piston,SMT,nlpd,-3.86618 -py10_compare_python_packages,python,piston,SMT,fit_s,12.649 -py10_compare_python_packages,python,piston,OpenTURNS,q2,0.990631 -py10_compare_python_packages,python,piston,OpenTURNS,nlpd,-2.9906 -py10_compare_python_packages,python,piston,OpenTURNS,fit_s,0.11631 -py10_compare_python_packages,python,borehole,pylibkriging,q2,0.999694 -py10_compare_python_packages,python,borehole,pylibkriging,nlpd,1.00118 -py10_compare_python_packages,python,borehole,pylibkriging,fit_s,0.0149446 -py10_compare_python_packages,python,borehole,pylibkriging BFGS10,q2,0.999694 -py10_compare_python_packages,python,borehole,pylibkriging BFGS10,nlpd,1.00121 -py10_compare_python_packages,python,borehole,pylibkriging BFGS10,fit_s,0.164608 -py10_compare_python_packages,python,borehole,scikit-learn,q2,0.999929 -py10_compare_python_packages,python,borehole,scikit-learn,nlpd,0.194455 -py10_compare_python_packages,python,borehole,scikit-learn,fit_s,1.04563 -py10_compare_python_packages,python,borehole,SMT,q2,0.999925 -py10_compare_python_packages,python,borehole,SMT,nlpd,0.279122 -py10_compare_python_packages,python,borehole,SMT,fit_s,11.8531 -py10_compare_python_packages,python,borehole,OpenTURNS,q2,0.996206 -py10_compare_python_packages,python,borehole,OpenTURNS,nlpd,2.55431 -py10_compare_python_packages,python,borehole,OpenTURNS,fit_s,0.12395 -py10_compare_python_packages,python,wing_weight,pylibkriging,q2,0.99972 -py10_compare_python_packages,python,wing_weight,pylibkriging,nlpd,1.11517 -py10_compare_python_packages,python,wing_weight,pylibkriging,fit_s,0.0290489 -py10_compare_python_packages,python,wing_weight,pylibkriging BFGS10,q2,0.99972 -py10_compare_python_packages,python,wing_weight,pylibkriging BFGS10,nlpd,1.11512 -py10_compare_python_packages,python,wing_weight,pylibkriging BFGS10,fit_s,0.314707 -py10_compare_python_packages,python,wing_weight,scikit-learn,q2,0.998048 -py10_compare_python_packages,python,wing_weight,scikit-learn,nlpd,2.57304 -py10_compare_python_packages,python,wing_weight,scikit-learn,fit_s,0.729278 -py10_compare_python_packages,python,wing_weight,SMT,q2,0.999898 -py10_compare_python_packages,python,wing_weight,SMT,nlpd,0.604023 -py10_compare_python_packages,python,wing_weight,SMT,fit_s,25.8262 -py10_compare_python_packages,python,wing_weight,OpenTURNS,q2,0.993145 -py10_compare_python_packages,python,wing_weight,OpenTURNS,nlpd,2.83627 -py10_compare_python_packages,python,wing_weight,OpenTURNS,fit_s,0.280434 -py10_compare_python_packages,python,hartmann6,pylibkriging,q2,0.422938 -py10_compare_python_packages,python,hartmann6,pylibkriging,nlpd,0.0965167 -py10_compare_python_packages,python,hartmann6,pylibkriging,fit_s,0.0562823 -py10_compare_python_packages,python,hartmann6,pylibkriging BFGS10,q2,0.72352 -py10_compare_python_packages,python,hartmann6,pylibkriging BFGS10,nlpd,-0.156898 -py10_compare_python_packages,python,hartmann6,pylibkriging BFGS10,fit_s,0.822745 -py10_compare_python_packages,python,hartmann6,scikit-learn,q2,0.69049 -py10_compare_python_packages,python,hartmann6,scikit-learn,nlpd,-0.137845 -py10_compare_python_packages,python,hartmann6,scikit-learn,fit_s,1.3155 -py10_compare_python_packages,python,hartmann6,SMT,q2,0.708463 -py10_compare_python_packages,python,hartmann6,SMT,nlpd,-0.0177081 -py10_compare_python_packages,python,hartmann6,SMT,fit_s,12.0578 -py10_compare_python_packages,python,hartmann6,OpenTURNS,q2,0.696177 -py10_compare_python_packages,python,hartmann6,OpenTURNS,nlpd,-0.132838 -py10_compare_python_packages,python,hartmann6,OpenTURNS,fit_s,0.133332 diff --git a/baseline/metrics-release.csv b/baseline/metrics-release.csv deleted file mode 100644 index 0a25355..0000000 --- a/baseline/metrics-release.csv +++ /dev/null @@ -1,916 +0,0 @@ -suite,lang,case,package,metric,value -py01_kernels_trends,python,forrester/gauss/constant,pylibkriging,fit_s,0.0039537 -py01_kernels_trends,python,forrester/gauss/constant,pylibkriging,rmse,0.0848271 -py01_kernels_trends,python,forrester/gauss/constant,pylibkriging,q2,0.999634 -py01_kernels_trends,python,forrester/gauss/constant,pylibkriging,nlpd,-3.06486 -py01_kernels_trends,python,forrester/gauss/linear,pylibkriging,fit_s,0.0008564 -py01_kernels_trends,python,forrester/gauss/linear,pylibkriging,rmse,0.0855468 -py01_kernels_trends,python,forrester/gauss/linear,pylibkriging,q2,0.999628 -py01_kernels_trends,python,forrester/gauss/linear,pylibkriging,nlpd,-3.11029 -py01_kernels_trends,python,forrester/gauss/interactive,pylibkriging,fit_s,0.000818729 -py01_kernels_trends,python,forrester/gauss/interactive,pylibkriging,rmse,0.0855468 -py01_kernels_trends,python,forrester/gauss/interactive,pylibkriging,q2,0.999628 -py01_kernels_trends,python,forrester/gauss/interactive,pylibkriging,nlpd,-3.11029 -py01_kernels_trends,python,forrester/gauss/quadratic,pylibkriging,fit_s,0.000473261 -py01_kernels_trends,python,forrester/gauss/quadratic,pylibkriging,rmse,1.57249 -py01_kernels_trends,python,forrester/gauss/quadratic,pylibkriging,q2,0.874394 -py01_kernels_trends,python,forrester/gauss/quadratic,pylibkriging,nlpd,5.33879 -py01_kernels_trends,python,forrester/exp/constant,pylibkriging,fit_s,0.00042963 -py01_kernels_trends,python,forrester/exp/constant,pylibkriging,rmse,0.721582 -py01_kernels_trends,python,forrester/exp/constant,pylibkriging,q2,0.973551 -py01_kernels_trends,python,forrester/exp/constant,pylibkriging,nlpd,1.54551 -py01_kernels_trends,python,forrester/exp/linear,pylibkriging,fit_s,0.000402451 -py01_kernels_trends,python,forrester/exp/linear,pylibkriging,rmse,0.793413 -py01_kernels_trends,python,forrester/exp/linear,pylibkriging,q2,0.968024 -py01_kernels_trends,python,forrester/exp/linear,pylibkriging,nlpd,1.50639 -py01_kernels_trends,python,forrester/exp/interactive,pylibkriging,fit_s,0.000428438 -py01_kernels_trends,python,forrester/exp/interactive,pylibkriging,rmse,0.793413 -py01_kernels_trends,python,forrester/exp/interactive,pylibkriging,q2,0.968024 -py01_kernels_trends,python,forrester/exp/interactive,pylibkriging,nlpd,1.50639 -py01_kernels_trends,python,forrester/exp/quadratic,pylibkriging,fit_s,0.000418425 -py01_kernels_trends,python,forrester/exp/quadratic,pylibkriging,rmse,0.723267 -py01_kernels_trends,python,forrester/exp/quadratic,pylibkriging,q2,0.973428 -py01_kernels_trends,python,forrester/exp/quadratic,pylibkriging,nlpd,1.28782 -py01_kernels_trends,python,forrester/matern3_2/constant,pylibkriging,fit_s,0.000519753 -py01_kernels_trends,python,forrester/matern3_2/constant,pylibkriging,rmse,0.334007 -py01_kernels_trends,python,forrester/matern3_2/constant,pylibkriging,q2,0.994333 -py01_kernels_trends,python,forrester/matern3_2/constant,pylibkriging,nlpd,-0.227715 -py01_kernels_trends,python,forrester/matern3_2/linear,pylibkriging,fit_s,0.000559092 -py01_kernels_trends,python,forrester/matern3_2/linear,pylibkriging,rmse,0.364477 -py01_kernels_trends,python,forrester/matern3_2/linear,pylibkriging,q2,0.993252 -py01_kernels_trends,python,forrester/matern3_2/linear,pylibkriging,nlpd,-0.187262 -py01_kernels_trends,python,forrester/matern3_2/interactive,pylibkriging,fit_s,0.00060463 -py01_kernels_trends,python,forrester/matern3_2/interactive,pylibkriging,rmse,0.364477 -py01_kernels_trends,python,forrester/matern3_2/interactive,pylibkriging,q2,0.993252 -py01_kernels_trends,python,forrester/matern3_2/interactive,pylibkriging,nlpd,-0.187262 -py01_kernels_trends,python,forrester/matern3_2/quadratic,pylibkriging,fit_s,0.000493765 -py01_kernels_trends,python,forrester/matern3_2/quadratic,pylibkriging,rmse,0.435609 -py01_kernels_trends,python,forrester/matern3_2/quadratic,pylibkriging,q2,0.990361 -py01_kernels_trends,python,forrester/matern3_2/quadratic,pylibkriging,nlpd,-0.183076 -py01_kernels_trends,python,forrester/matern5_2/constant,pylibkriging,fit_s,0.000758886 -py01_kernels_trends,python,forrester/matern5_2/constant,pylibkriging,rmse,0.281945 -py01_kernels_trends,python,forrester/matern5_2/constant,pylibkriging,q2,0.995962 -py01_kernels_trends,python,forrester/matern5_2/constant,pylibkriging,nlpd,-1.01945 -py01_kernels_trends,python,forrester/matern5_2/constant,pylibkriging BFGS10,rmse,0.281945 -py01_kernels_trends,python,forrester/matern5_2/constant,pylibkriging BFGS10,q2,0.995962 -py01_kernels_trends,python,forrester/matern5_2/constant,pylibkriging BFGS10,nlpd,-1.01945 -py01_kernels_trends,python,forrester/matern5_2/linear,pylibkriging,fit_s,0.000576973 -py01_kernels_trends,python,forrester/matern5_2/linear,pylibkriging,rmse,0.298959 -py01_kernels_trends,python,forrester/matern5_2/linear,pylibkriging,q2,0.99546 -py01_kernels_trends,python,forrester/matern5_2/linear,pylibkriging,nlpd,-0.95551 -py01_kernels_trends,python,forrester/matern5_2/interactive,pylibkriging,fit_s,0.00054884 -py01_kernels_trends,python,forrester/matern5_2/interactive,pylibkriging,rmse,0.298959 -py01_kernels_trends,python,forrester/matern5_2/interactive,pylibkriging,q2,0.99546 -py01_kernels_trends,python,forrester/matern5_2/interactive,pylibkriging,nlpd,-0.95551 -py01_kernels_trends,python,forrester/matern5_2/quadratic,pylibkriging,fit_s,0.000477076 -py01_kernels_trends,python,forrester/matern5_2/quadratic,pylibkriging,rmse,0.365039 -py01_kernels_trends,python,forrester/matern5_2/quadratic,pylibkriging,q2,0.993231 -py01_kernels_trends,python,forrester/matern5_2/quadratic,pylibkriging,nlpd,-0.790032 -py01_kernels_trends,python,gramacy_lee_2012/gauss/constant,pylibkriging,fit_s,0.0016222 -py01_kernels_trends,python,gramacy_lee_2012/gauss/constant,pylibkriging,rmse,0.191274 -py01_kernels_trends,python,gramacy_lee_2012/gauss/constant,pylibkriging,q2,0.978742 -py01_kernels_trends,python,gramacy_lee_2012/gauss/constant,pylibkriging,nlpd,-0.704948 -py01_kernels_trends,python,gramacy_lee_2012/gauss/linear,pylibkriging,fit_s,0.00240469 -py01_kernels_trends,python,gramacy_lee_2012/gauss/linear,pylibkriging,rmse,0.236028 -py01_kernels_trends,python,gramacy_lee_2012/gauss/linear,pylibkriging,q2,0.96763 -py01_kernels_trends,python,gramacy_lee_2012/gauss/linear,pylibkriging,nlpd,-0.580993 -py01_kernels_trends,python,gramacy_lee_2012/gauss/interactive,pylibkriging,fit_s,0.00205922 -py01_kernels_trends,python,gramacy_lee_2012/gauss/interactive,pylibkriging,rmse,0.236028 -py01_kernels_trends,python,gramacy_lee_2012/gauss/interactive,pylibkriging,q2,0.96763 -py01_kernels_trends,python,gramacy_lee_2012/gauss/interactive,pylibkriging,nlpd,-0.580993 -py01_kernels_trends,python,gramacy_lee_2012/gauss/quadratic,pylibkriging,fit_s,0.00200701 -py01_kernels_trends,python,gramacy_lee_2012/gauss/quadratic,pylibkriging,rmse,0.150914 -py01_kernels_trends,python,gramacy_lee_2012/gauss/quadratic,pylibkriging,q2,0.986767 -py01_kernels_trends,python,gramacy_lee_2012/gauss/quadratic,pylibkriging,nlpd,-0.458431 -py01_kernels_trends,python,gramacy_lee_2012/exp/constant,pylibkriging,fit_s,0.00123 -py01_kernels_trends,python,gramacy_lee_2012/exp/constant,pylibkriging,rmse,0.196586 -py01_kernels_trends,python,gramacy_lee_2012/exp/constant,pylibkriging,q2,0.977545 -py01_kernels_trends,python,gramacy_lee_2012/exp/constant,pylibkriging,nlpd,-0.0621192 -py01_kernels_trends,python,gramacy_lee_2012/exp/linear,pylibkriging,fit_s,0.00172782 -py01_kernels_trends,python,gramacy_lee_2012/exp/linear,pylibkriging,rmse,0.20979 -py01_kernels_trends,python,gramacy_lee_2012/exp/linear,pylibkriging,q2,0.974427 -py01_kernels_trends,python,gramacy_lee_2012/exp/linear,pylibkriging,nlpd,-0.0466568 -py01_kernels_trends,python,gramacy_lee_2012/exp/interactive,pylibkriging,fit_s,0.0017736 -py01_kernels_trends,python,gramacy_lee_2012/exp/interactive,pylibkriging,rmse,0.20979 -py01_kernels_trends,python,gramacy_lee_2012/exp/interactive,pylibkriging,q2,0.974427 -py01_kernels_trends,python,gramacy_lee_2012/exp/interactive,pylibkriging,nlpd,-0.0466568 -py01_kernels_trends,python,gramacy_lee_2012/exp/quadratic,pylibkriging,fit_s,0.00394821 -py01_kernels_trends,python,gramacy_lee_2012/exp/quadratic,pylibkriging,rmse,0.252565 -py01_kernels_trends,python,gramacy_lee_2012/exp/quadratic,pylibkriging,q2,0.962935 -py01_kernels_trends,python,gramacy_lee_2012/exp/quadratic,pylibkriging,nlpd,0.220609 -py01_kernels_trends,python,gramacy_lee_2012/matern3_2/constant,pylibkriging,fit_s,0.000825167 -py01_kernels_trends,python,gramacy_lee_2012/matern3_2/constant,pylibkriging,rmse,0.177253 -py01_kernels_trends,python,gramacy_lee_2012/matern3_2/constant,pylibkriging,q2,0.981744 -py01_kernels_trends,python,gramacy_lee_2012/matern3_2/constant,pylibkriging,nlpd,-0.482212 -py01_kernels_trends,python,gramacy_lee_2012/matern3_2/linear,pylibkriging,fit_s,0.000857115 -py01_kernels_trends,python,gramacy_lee_2012/matern3_2/linear,pylibkriging,rmse,0.211499 -py01_kernels_trends,python,gramacy_lee_2012/matern3_2/linear,pylibkriging,q2,0.974009 -py01_kernels_trends,python,gramacy_lee_2012/matern3_2/linear,pylibkriging,nlpd,-0.375844 -py01_kernels_trends,python,gramacy_lee_2012/matern3_2/interactive,pylibkriging,fit_s,0.00114012 -py01_kernels_trends,python,gramacy_lee_2012/matern3_2/interactive,pylibkriging,rmse,0.211499 -py01_kernels_trends,python,gramacy_lee_2012/matern3_2/interactive,pylibkriging,q2,0.974009 -py01_kernels_trends,python,gramacy_lee_2012/matern3_2/interactive,pylibkriging,nlpd,-0.375844 -py01_kernels_trends,python,gramacy_lee_2012/matern3_2/quadratic,pylibkriging,fit_s,0.00237894 -py01_kernels_trends,python,gramacy_lee_2012/matern3_2/quadratic,pylibkriging,rmse,0.184528 -py01_kernels_trends,python,gramacy_lee_2012/matern3_2/quadratic,pylibkriging,q2,0.980215 -py01_kernels_trends,python,gramacy_lee_2012/matern3_2/quadratic,pylibkriging,nlpd,-0.171795 -py01_kernels_trends,python,gramacy_lee_2012/matern5_2/constant,pylibkriging,fit_s,0.000836849 -py01_kernels_trends,python,gramacy_lee_2012/matern5_2/constant,pylibkriging,rmse,0.177638 -py01_kernels_trends,python,gramacy_lee_2012/matern5_2/constant,pylibkriging,q2,0.981665 -py01_kernels_trends,python,gramacy_lee_2012/matern5_2/constant,pylibkriging,nlpd,-0.575421 -py01_kernels_trends,python,gramacy_lee_2012/matern5_2/constant,pylibkriging BFGS10,rmse,0.177638 -py01_kernels_trends,python,gramacy_lee_2012/matern5_2/constant,pylibkriging BFGS10,q2,0.981665 -py01_kernels_trends,python,gramacy_lee_2012/matern5_2/constant,pylibkriging BFGS10,nlpd,-0.575422 -py01_kernels_trends,python,gramacy_lee_2012/matern5_2/linear,pylibkriging,fit_s,0.000859499 -py01_kernels_trends,python,gramacy_lee_2012/matern5_2/linear,pylibkriging,rmse,0.219759 -py01_kernels_trends,python,gramacy_lee_2012/matern5_2/linear,pylibkriging,q2,0.971939 -py01_kernels_trends,python,gramacy_lee_2012/matern5_2/linear,pylibkriging,nlpd,-0.458933 -py01_kernels_trends,python,gramacy_lee_2012/matern5_2/interactive,pylibkriging,fit_s,0.00080061 -py01_kernels_trends,python,gramacy_lee_2012/matern5_2/interactive,pylibkriging,rmse,0.219759 -py01_kernels_trends,python,gramacy_lee_2012/matern5_2/interactive,pylibkriging,q2,0.971939 -py01_kernels_trends,python,gramacy_lee_2012/matern5_2/interactive,pylibkriging,nlpd,-0.458933 -py01_kernels_trends,python,gramacy_lee_2012/matern5_2/quadratic,pylibkriging,fit_s,0.00249147 -py01_kernels_trends,python,gramacy_lee_2012/matern5_2/quadratic,pylibkriging,rmse,0.169941 -py01_kernels_trends,python,gramacy_lee_2012/matern5_2/quadratic,pylibkriging,q2,0.983219 -py01_kernels_trends,python,gramacy_lee_2012/matern5_2/quadratic,pylibkriging,nlpd,-0.28938 -py01_kernels_trends,python,branin/gauss/constant,pylibkriging,fit_s,0.00172019 -py01_kernels_trends,python,branin/gauss/constant,pylibkriging,rmse,7.2629 -py01_kernels_trends,python,branin/gauss/constant,pylibkriging,q2,0.980655 -py01_kernels_trends,python,branin/gauss/constant,pylibkriging,nlpd,4.3152 -py01_kernels_trends,python,branin/gauss/linear,pylibkriging,fit_s,0.00188136 -py01_kernels_trends,python,branin/gauss/linear,pylibkriging,rmse,7.25733 -py01_kernels_trends,python,branin/gauss/linear,pylibkriging,q2,0.980685 -py01_kernels_trends,python,branin/gauss/linear,pylibkriging,nlpd,4.30813 -py01_kernels_trends,python,branin/gauss/interactive,pylibkriging,fit_s,0.00456238 -py01_kernels_trends,python,branin/gauss/interactive,pylibkriging,rmse,7.2576 -py01_kernels_trends,python,branin/gauss/interactive,pylibkriging,q2,0.980683 -py01_kernels_trends,python,branin/gauss/interactive,pylibkriging,nlpd,4.30176 -py01_kernels_trends,python,branin/gauss/quadratic,pylibkriging,fit_s,0.00182772 -py01_kernels_trends,python,branin/gauss/quadratic,pylibkriging,rmse,7.27808 -py01_kernels_trends,python,branin/gauss/quadratic,pylibkriging,q2,0.980574 -py01_kernels_trends,python,branin/gauss/quadratic,pylibkriging,nlpd,4.29399 -py01_kernels_trends,python,branin/exp/constant,pylibkriging,fit_s,0.00215006 -py01_kernels_trends,python,branin/exp/constant,pylibkriging,rmse,14.5887 -py01_kernels_trends,python,branin/exp/constant,pylibkriging,q2,0.921947 -py01_kernels_trends,python,branin/exp/constant,pylibkriging,nlpd,4.09457 -py01_kernels_trends,python,branin/exp/linear,pylibkriging,fit_s,0.00210071 -py01_kernels_trends,python,branin/exp/linear,pylibkriging,rmse,14.9622 -py01_kernels_trends,python,branin/exp/linear,pylibkriging,q2,0.9179 -py01_kernels_trends,python,branin/exp/linear,pylibkriging,nlpd,4.0909 -py01_kernels_trends,python,branin/exp/interactive,pylibkriging,fit_s,0.00220799 -py01_kernels_trends,python,branin/exp/interactive,pylibkriging,rmse,11.6963 -py01_kernels_trends,python,branin/exp/interactive,pylibkriging,q2,0.94983 -py01_kernels_trends,python,branin/exp/interactive,pylibkriging,nlpd,3.79671 -py01_kernels_trends,python,branin/exp/quadratic,pylibkriging,fit_s,0.00237751 -py01_kernels_trends,python,branin/exp/quadratic,pylibkriging,rmse,9.33909 -py01_kernels_trends,python,branin/exp/quadratic,pylibkriging,q2,0.968014 -py01_kernels_trends,python,branin/exp/quadratic,pylibkriging,nlpd,3.62362 -py01_kernels_trends,python,branin/matern3_2/constant,pylibkriging,fit_s,0.00252843 -py01_kernels_trends,python,branin/matern3_2/constant,pylibkriging,rmse,3.24796 -py01_kernels_trends,python,branin/matern3_2/constant,pylibkriging,q2,0.996131 -py01_kernels_trends,python,branin/matern3_2/constant,pylibkriging,nlpd,2.30444 -py01_kernels_trends,python,branin/matern3_2/linear,pylibkriging,fit_s,0.00200725 -py01_kernels_trends,python,branin/matern3_2/linear,pylibkriging,rmse,3.14402 -py01_kernels_trends,python,branin/matern3_2/linear,pylibkriging,q2,0.996375 -py01_kernels_trends,python,branin/matern3_2/linear,pylibkriging,nlpd,2.28067 -py01_kernels_trends,python,branin/matern3_2/interactive,pylibkriging,fit_s,0.00414944 -py01_kernels_trends,python,branin/matern3_2/interactive,pylibkriging,rmse,3.19226 -py01_kernels_trends,python,branin/matern3_2/interactive,pylibkriging,q2,0.996263 -py01_kernels_trends,python,branin/matern3_2/interactive,pylibkriging,nlpd,2.20359 -py01_kernels_trends,python,branin/matern3_2/quadratic,pylibkriging,fit_s,0.00253224 -py01_kernels_trends,python,branin/matern3_2/quadratic,pylibkriging,rmse,3.01232 -py01_kernels_trends,python,branin/matern3_2/quadratic,pylibkriging,q2,0.996672 -py01_kernels_trends,python,branin/matern3_2/quadratic,pylibkriging,nlpd,2.11627 -py01_kernels_trends,python,branin/matern5_2/constant,pylibkriging,fit_s,0.00171685 -py01_kernels_trends,python,branin/matern5_2/constant,pylibkriging,rmse,0.52012 -py01_kernels_trends,python,branin/matern5_2/constant,pylibkriging,q2,0.999901 -py01_kernels_trends,python,branin/matern5_2/constant,pylibkriging,nlpd,0.283399 -py01_kernels_trends,python,branin/matern5_2/constant,pylibkriging BFGS10,rmse,0.52012 -py01_kernels_trends,python,branin/matern5_2/constant,pylibkriging BFGS10,q2,0.999901 -py01_kernels_trends,python,branin/matern5_2/constant,pylibkriging BFGS10,nlpd,0.283395 -py01_kernels_trends,python,branin/matern5_2/linear,pylibkriging,fit_s,0.0021255 -py01_kernels_trends,python,branin/matern5_2/linear,pylibkriging,rmse,0.460868 -py01_kernels_trends,python,branin/matern5_2/linear,pylibkriging,q2,0.999922 -py01_kernels_trends,python,branin/matern5_2/linear,pylibkriging,nlpd,0.209741 -py01_kernels_trends,python,branin/matern5_2/interactive,pylibkriging,fit_s,0.00307012 -py01_kernels_trends,python,branin/matern5_2/interactive,pylibkriging,rmse,0.498484 -py01_kernels_trends,python,branin/matern5_2/interactive,pylibkriging,q2,0.999909 -py01_kernels_trends,python,branin/matern5_2/interactive,pylibkriging,nlpd,0.239079 -py01_kernels_trends,python,branin/matern5_2/quadratic,pylibkriging,fit_s,0.0092895 -py01_kernels_trends,python,branin/matern5_2/quadratic,pylibkriging,rmse,0.67472 -py01_kernels_trends,python,branin/matern5_2/quadratic,pylibkriging,q2,0.999833 -py01_kernels_trends,python,branin/matern5_2/quadratic,pylibkriging,nlpd,0.419266 -py01_kernels_trends,python,six_hump_camel/gauss/constant,pylibkriging,fit_s,0.00194049 -py01_kernels_trends,python,six_hump_camel/gauss/constant,pylibkriging,rmse,18.9501 -py01_kernels_trends,python,six_hump_camel/gauss/constant,pylibkriging,q2,0.536354 -py01_kernels_trends,python,six_hump_camel/gauss/constant,pylibkriging,nlpd,84.151 -py01_kernels_trends,python,six_hump_camel/gauss/linear,pylibkriging,fit_s,0.00170755 -py01_kernels_trends,python,six_hump_camel/gauss/linear,pylibkriging,rmse,18.9579 -py01_kernels_trends,python,six_hump_camel/gauss/linear,pylibkriging,q2,0.535976 -py01_kernels_trends,python,six_hump_camel/gauss/linear,pylibkriging,nlpd,44.8696 -py01_kernels_trends,python,six_hump_camel/gauss/interactive,pylibkriging,fit_s,0.00180864 -py01_kernels_trends,python,six_hump_camel/gauss/interactive,pylibkriging,rmse,18.9819 -py01_kernels_trends,python,six_hump_camel/gauss/interactive,pylibkriging,q2,0.5348 -py01_kernels_trends,python,six_hump_camel/gauss/interactive,pylibkriging,nlpd,35.2765 -py01_kernels_trends,python,six_hump_camel/gauss/quadratic,pylibkriging,fit_s,0.00178218 -py01_kernels_trends,python,six_hump_camel/gauss/quadratic,pylibkriging,rmse,10.9058 -py01_kernels_trends,python,six_hump_camel/gauss/quadratic,pylibkriging,q2,0.846441 -py01_kernels_trends,python,six_hump_camel/gauss/quadratic,pylibkriging,nlpd,4.92017 -py01_kernels_trends,python,six_hump_camel/exp/constant,pylibkriging,fit_s,0.000977039 -py01_kernels_trends,python,six_hump_camel/exp/constant,pylibkriging,rmse,16.9653 -py01_kernels_trends,python,six_hump_camel/exp/constant,pylibkriging,q2,0.628393 -py01_kernels_trends,python,six_hump_camel/exp/constant,pylibkriging,nlpd,4.48595 -py01_kernels_trends,python,six_hump_camel/exp/linear,pylibkriging,fit_s,0.00107837 -py01_kernels_trends,python,six_hump_camel/exp/linear,pylibkriging,rmse,17.1781 -py01_kernels_trends,python,six_hump_camel/exp/linear,pylibkriging,q2,0.619012 -py01_kernels_trends,python,six_hump_camel/exp/linear,pylibkriging,nlpd,4.47061 -py01_kernels_trends,python,six_hump_camel/exp/interactive,pylibkriging,fit_s,0.00123405 -py01_kernels_trends,python,six_hump_camel/exp/interactive,pylibkriging,rmse,17.1856 -py01_kernels_trends,python,six_hump_camel/exp/interactive,pylibkriging,q2,0.618679 -py01_kernels_trends,python,six_hump_camel/exp/interactive,pylibkriging,nlpd,4.43133 -py01_kernels_trends,python,six_hump_camel/exp/quadratic,pylibkriging,fit_s,0.00120115 -py01_kernels_trends,python,six_hump_camel/exp/quadratic,pylibkriging,rmse,7.99484 -py01_kernels_trends,python,six_hump_camel/exp/quadratic,pylibkriging,q2,0.917476 -py01_kernels_trends,python,six_hump_camel/exp/quadratic,pylibkriging,nlpd,3.35612 -py01_kernels_trends,python,six_hump_camel/matern3_2/constant,pylibkriging,fit_s,0.00264788 -py01_kernels_trends,python,six_hump_camel/matern3_2/constant,pylibkriging,rmse,18.4958 -py01_kernels_trends,python,six_hump_camel/matern3_2/constant,pylibkriging,q2,0.558321 -py01_kernels_trends,python,six_hump_camel/matern3_2/constant,pylibkriging,nlpd,18.8349 -py01_kernels_trends,python,six_hump_camel/matern3_2/linear,pylibkriging,fit_s,0.00251079 -py01_kernels_trends,python,six_hump_camel/matern3_2/linear,pylibkriging,rmse,18.5562 -py01_kernels_trends,python,six_hump_camel/matern3_2/linear,pylibkriging,q2,0.555433 -py01_kernels_trends,python,six_hump_camel/matern3_2/linear,pylibkriging,nlpd,14.9494 -py01_kernels_trends,python,six_hump_camel/matern3_2/interactive,pylibkriging,fit_s,0.00148797 -py01_kernels_trends,python,six_hump_camel/matern3_2/interactive,pylibkriging,rmse,18.561 -py01_kernels_trends,python,six_hump_camel/matern3_2/interactive,pylibkriging,q2,0.555202 -py01_kernels_trends,python,six_hump_camel/matern3_2/interactive,pylibkriging,nlpd,12.9923 -py01_kernels_trends,python,six_hump_camel/matern3_2/quadratic,pylibkriging,fit_s,0.00289869 -py01_kernels_trends,python,six_hump_camel/matern3_2/quadratic,pylibkriging,rmse,8.91645 -py01_kernels_trends,python,six_hump_camel/matern3_2/quadratic,pylibkriging,q2,0.897353 -py01_kernels_trends,python,six_hump_camel/matern3_2/quadratic,pylibkriging,nlpd,3.94939 -py01_kernels_trends,python,six_hump_camel/matern5_2/constant,pylibkriging,fit_s,0.00238967 -py01_kernels_trends,python,six_hump_camel/matern5_2/constant,pylibkriging,rmse,18.3236 -py01_kernels_trends,python,six_hump_camel/matern5_2/constant,pylibkriging,q2,0.566507 -py01_kernels_trends,python,six_hump_camel/matern5_2/constant,pylibkriging,nlpd,18.4322 -py01_kernels_trends,python,six_hump_camel/matern5_2/constant,pylibkriging BFGS10,rmse,1.36208 -py01_kernels_trends,python,six_hump_camel/matern5_2/constant,pylibkriging BFGS10,q2,0.997605 -py01_kernels_trends,python,six_hump_camel/matern5_2/constant,pylibkriging BFGS10,nlpd,1.03215 -py01_kernels_trends,python,six_hump_camel/matern5_2/linear,pylibkriging,fit_s,0.00207353 -py01_kernels_trends,python,six_hump_camel/matern5_2/linear,pylibkriging,rmse,18.7127 -py01_kernels_trends,python,six_hump_camel/matern5_2/linear,pylibkriging,q2,0.547901 -py01_kernels_trends,python,six_hump_camel/matern5_2/linear,pylibkriging,nlpd,24.5771 -py01_kernels_trends,python,six_hump_camel/matern5_2/interactive,pylibkriging,fit_s,0.0020268 -py01_kernels_trends,python,six_hump_camel/matern5_2/interactive,pylibkriging,rmse,18.7194 -py01_kernels_trends,python,six_hump_camel/matern5_2/interactive,pylibkriging,q2,0.547577 -py01_kernels_trends,python,six_hump_camel/matern5_2/interactive,pylibkriging,nlpd,20.4255 -py01_kernels_trends,python,six_hump_camel/matern5_2/quadratic,pylibkriging,fit_s,0.00144219 -py01_kernels_trends,python,six_hump_camel/matern5_2/quadratic,pylibkriging,rmse,10.2878 -py01_kernels_trends,python,six_hump_camel/matern5_2/quadratic,pylibkriging,q2,0.863351 -py01_kernels_trends,python,six_hump_camel/matern5_2/quadratic,pylibkriging,nlpd,4.46344 -py01_kernels_trends,python,log_goldstein_price/gauss/constant,pylibkriging,fit_s,0.00526643 -py01_kernels_trends,python,log_goldstein_price/gauss/constant,pylibkriging,rmse,0.295934 -py01_kernels_trends,python,log_goldstein_price/gauss/constant,pylibkriging,q2,0.920537 -py01_kernels_trends,python,log_goldstein_price/gauss/constant,pylibkriging,nlpd,1.58666 -py01_kernels_trends,python,log_goldstein_price/gauss/linear,pylibkriging,fit_s,0.00606179 -py01_kernels_trends,python,log_goldstein_price/gauss/linear,pylibkriging,rmse,0.296355 -py01_kernels_trends,python,log_goldstein_price/gauss/linear,pylibkriging,q2,0.92031 -py01_kernels_trends,python,log_goldstein_price/gauss/linear,pylibkriging,nlpd,1.2962 -py01_kernels_trends,python,log_goldstein_price/gauss/interactive,pylibkriging,fit_s,0.00633311 -py01_kernels_trends,python,log_goldstein_price/gauss/interactive,pylibkriging,rmse,0.285136 -py01_kernels_trends,python,log_goldstein_price/gauss/interactive,pylibkriging,q2,0.926229 -py01_kernels_trends,python,log_goldstein_price/gauss/interactive,pylibkriging,nlpd,1.04122 -py01_kernels_trends,python,log_goldstein_price/gauss/quadratic,pylibkriging,fit_s,0.00615239 -py01_kernels_trends,python,log_goldstein_price/gauss/quadratic,pylibkriging,rmse,0.237308 -py01_kernels_trends,python,log_goldstein_price/gauss/quadratic,pylibkriging,q2,0.948902 -py01_kernels_trends,python,log_goldstein_price/gauss/quadratic,pylibkriging,nlpd,0.146484 -py01_kernels_trends,python,log_goldstein_price/exp/constant,pylibkriging,fit_s,0.00271893 -py01_kernels_trends,python,log_goldstein_price/exp/constant,pylibkriging,rmse,0.334876 -py01_kernels_trends,python,log_goldstein_price/exp/constant,pylibkriging,q2,0.898247 -py01_kernels_trends,python,log_goldstein_price/exp/constant,pylibkriging,nlpd,0.339235 -py01_kernels_trends,python,log_goldstein_price/exp/linear,pylibkriging,fit_s,0.00223827 -py01_kernels_trends,python,log_goldstein_price/exp/linear,pylibkriging,rmse,0.333447 -py01_kernels_trends,python,log_goldstein_price/exp/linear,pylibkriging,q2,0.899114 -py01_kernels_trends,python,log_goldstein_price/exp/linear,pylibkriging,nlpd,0.336515 -py01_kernels_trends,python,log_goldstein_price/exp/interactive,pylibkriging,fit_s,0.00192547 -py01_kernels_trends,python,log_goldstein_price/exp/interactive,pylibkriging,rmse,0.336819 -py01_kernels_trends,python,log_goldstein_price/exp/interactive,pylibkriging,q2,0.897063 -py01_kernels_trends,python,log_goldstein_price/exp/interactive,pylibkriging,nlpd,0.329566 -py01_kernels_trends,python,log_goldstein_price/exp/quadratic,pylibkriging,fit_s,0.00333118 -py01_kernels_trends,python,log_goldstein_price/exp/quadratic,pylibkriging,rmse,0.327816 -py01_kernels_trends,python,log_goldstein_price/exp/quadratic,pylibkriging,q2,0.902492 -py01_kernels_trends,python,log_goldstein_price/exp/quadratic,pylibkriging,nlpd,0.322056 -py01_kernels_trends,python,log_goldstein_price/matern3_2/constant,pylibkriging,fit_s,0.00606227 -py01_kernels_trends,python,log_goldstein_price/matern3_2/constant,pylibkriging,rmse,0.684362 -py01_kernels_trends,python,log_goldstein_price/matern3_2/constant,pylibkriging,q2,0.575038 -py01_kernels_trends,python,log_goldstein_price/matern3_2/constant,pylibkriging,nlpd,0.972803 -py01_kernels_trends,python,log_goldstein_price/matern3_2/linear,pylibkriging,fit_s,0.00382805 -py01_kernels_trends,python,log_goldstein_price/matern3_2/linear,pylibkriging,rmse,0.650201 -py01_kernels_trends,python,log_goldstein_price/matern3_2/linear,pylibkriging,q2,0.616404 -py01_kernels_trends,python,log_goldstein_price/matern3_2/linear,pylibkriging,nlpd,0.921455 -py01_kernels_trends,python,log_goldstein_price/matern3_2/interactive,pylibkriging,fit_s,0.00707507 -py01_kernels_trends,python,log_goldstein_price/matern3_2/interactive,pylibkriging,rmse,0.274326 -py01_kernels_trends,python,log_goldstein_price/matern3_2/interactive,pylibkriging,q2,0.931717 -py01_kernels_trends,python,log_goldstein_price/matern3_2/interactive,pylibkriging,nlpd,0.0549679 -py01_kernels_trends,python,log_goldstein_price/matern3_2/quadratic,pylibkriging,fit_s,0.00617123 -py01_kernels_trends,python,log_goldstein_price/matern3_2/quadratic,pylibkriging,rmse,0.263745 -py01_kernels_trends,python,log_goldstein_price/matern3_2/quadratic,pylibkriging,q2,0.936883 -py01_kernels_trends,python,log_goldstein_price/matern3_2/quadratic,pylibkriging,nlpd,0.0446267 -py01_kernels_trends,python,log_goldstein_price/matern5_2/constant,pylibkriging,fit_s,0.00426936 -py01_kernels_trends,python,log_goldstein_price/matern5_2/constant,pylibkriging,rmse,0.674391 -py01_kernels_trends,python,log_goldstein_price/matern5_2/constant,pylibkriging,q2,0.58733 -py01_kernels_trends,python,log_goldstein_price/matern5_2/constant,pylibkriging,nlpd,0.944765 -py01_kernels_trends,python,log_goldstein_price/matern5_2/constant,pylibkriging BFGS10,rmse,0.268216 -py01_kernels_trends,python,log_goldstein_price/matern5_2/constant,pylibkriging BFGS10,q2,0.934725 -py01_kernels_trends,python,log_goldstein_price/matern5_2/constant,pylibkriging BFGS10,nlpd,0.169585 -py01_kernels_trends,python,log_goldstein_price/matern5_2/linear,pylibkriging,fit_s,0.00730562 -py01_kernels_trends,python,log_goldstein_price/matern5_2/linear,pylibkriging,rmse,0.268808 -py01_kernels_trends,python,log_goldstein_price/matern5_2/linear,pylibkriging,q2,0.934436 -py01_kernels_trends,python,log_goldstein_price/matern5_2/linear,pylibkriging,nlpd,0.142382 -py01_kernels_trends,python,log_goldstein_price/matern5_2/interactive,pylibkriging,fit_s,0.00520349 -py01_kernels_trends,python,log_goldstein_price/matern5_2/interactive,pylibkriging,rmse,0.271088 -py01_kernels_trends,python,log_goldstein_price/matern5_2/interactive,pylibkriging,q2,0.933319 -py01_kernels_trends,python,log_goldstein_price/matern5_2/interactive,pylibkriging,nlpd,0.128788 -py01_kernels_trends,python,log_goldstein_price/matern5_2/quadratic,pylibkriging,fit_s,0.00757623 -py01_kernels_trends,python,log_goldstein_price/matern5_2/quadratic,pylibkriging,rmse,0.253093 -py01_kernels_trends,python,log_goldstein_price/matern5_2/quadratic,pylibkriging,q2,0.941878 -py01_kernels_trends,python,log_goldstein_price/matern5_2/quadratic,pylibkriging,nlpd,0.025469 -py01_kernels_trends,python,lim_polynomial/gauss/constant,pylibkriging,fit_s,0.000713825 -py01_kernels_trends,python,lim_polynomial/gauss/constant,pylibkriging,rmse,0.026127 -py01_kernels_trends,python,lim_polynomial/gauss/constant,pylibkriging,q2,0.999737 -py01_kernels_trends,python,lim_polynomial/gauss/constant,pylibkriging,nlpd,-2.69817 -py01_kernels_trends,python,lim_polynomial/gauss/linear,pylibkriging,fit_s,0.000786066 -py01_kernels_trends,python,lim_polynomial/gauss/linear,pylibkriging,rmse,1.06243 -py01_kernels_trends,python,lim_polynomial/gauss/linear,pylibkriging,q2,0.564691 -py01_kernels_trends,python,lim_polynomial/gauss/linear,pylibkriging,nlpd,2.7394 -py01_kernels_trends,python,lim_polynomial/gauss/interactive,pylibkriging,fit_s,0.000851393 -py01_kernels_trends,python,lim_polynomial/gauss/interactive,pylibkriging,rmse,1.06504 -py01_kernels_trends,python,lim_polynomial/gauss/interactive,pylibkriging,q2,0.562551 -py01_kernels_trends,python,lim_polynomial/gauss/interactive,pylibkriging,nlpd,2.33575 -py01_kernels_trends,python,lim_polynomial/gauss/quadratic,pylibkriging,fit_s,0.0012238 -py01_kernels_trends,python,lim_polynomial/gauss/quadratic,pylibkriging,rmse,0.281816 -py01_kernels_trends,python,lim_polynomial/gauss/quadratic,pylibkriging,q2,0.969371 -py01_kernels_trends,python,lim_polynomial/gauss/quadratic,pylibkriging,nlpd,0.71682 -py01_kernels_trends,python,lim_polynomial/exp/constant,pylibkriging,fit_s,0.000771999 -py01_kernels_trends,python,lim_polynomial/exp/constant,pylibkriging,rmse,0.249023 -py01_kernels_trends,python,lim_polynomial/exp/constant,pylibkriging,q2,0.976085 -py01_kernels_trends,python,lim_polynomial/exp/constant,pylibkriging,nlpd,0.394159 -py01_kernels_trends,python,lim_polynomial/exp/linear,pylibkriging,fit_s,0.0006814 -py01_kernels_trends,python,lim_polynomial/exp/linear,pylibkriging,rmse,0.245558 -py01_kernels_trends,python,lim_polynomial/exp/linear,pylibkriging,q2,0.976746 -py01_kernels_trends,python,lim_polynomial/exp/linear,pylibkriging,nlpd,0.224593 -py01_kernels_trends,python,lim_polynomial/exp/interactive,pylibkriging,fit_s,0.000779867 -py01_kernels_trends,python,lim_polynomial/exp/interactive,pylibkriging,rmse,0.239032 -py01_kernels_trends,python,lim_polynomial/exp/interactive,pylibkriging,q2,0.977965 -py01_kernels_trends,python,lim_polynomial/exp/interactive,pylibkriging,nlpd,0.226133 -py01_kernels_trends,python,lim_polynomial/exp/quadratic,pylibkriging,fit_s,0.00164485 -py01_kernels_trends,python,lim_polynomial/exp/quadratic,pylibkriging,rmse,0.237713 -py01_kernels_trends,python,lim_polynomial/exp/quadratic,pylibkriging,q2,0.978208 -py01_kernels_trends,python,lim_polynomial/exp/quadratic,pylibkriging,nlpd,-0.139582 -py01_kernels_trends,python,lim_polynomial/matern3_2/constant,pylibkriging,fit_s,0.000581741 -py01_kernels_trends,python,lim_polynomial/matern3_2/constant,pylibkriging,rmse,0.0600911 -py01_kernels_trends,python,lim_polynomial/matern3_2/constant,pylibkriging,q2,0.998607 -py01_kernels_trends,python,lim_polynomial/matern3_2/constant,pylibkriging,nlpd,-1.34658 -py01_kernels_trends,python,lim_polynomial/matern3_2/linear,pylibkriging,fit_s,0.000698566 -py01_kernels_trends,python,lim_polynomial/matern3_2/linear,pylibkriging,rmse,0.0584376 -py01_kernels_trends,python,lim_polynomial/matern3_2/linear,pylibkriging,q2,0.998683 -py01_kernels_trends,python,lim_polynomial/matern3_2/linear,pylibkriging,nlpd,-1.34912 -py01_kernels_trends,python,lim_polynomial/matern3_2/interactive,pylibkriging,fit_s,0.000763416 -py01_kernels_trends,python,lim_polynomial/matern3_2/interactive,pylibkriging,rmse,0.0576096 -py01_kernels_trends,python,lim_polynomial/matern3_2/interactive,pylibkriging,q2,0.99872 -py01_kernels_trends,python,lim_polynomial/matern3_2/interactive,pylibkriging,nlpd,-1.35376 -py01_kernels_trends,python,lim_polynomial/matern3_2/quadratic,pylibkriging,fit_s,0.00313497 -py01_kernels_trends,python,lim_polynomial/matern3_2/quadratic,pylibkriging,rmse,0.215336 -py01_kernels_trends,python,lim_polynomial/matern3_2/quadratic,pylibkriging,q2,0.982117 -py01_kernels_trends,python,lim_polynomial/matern3_2/quadratic,pylibkriging,nlpd,-0.350837 -py01_kernels_trends,python,lim_polynomial/matern5_2/constant,pylibkriging,fit_s,0.000642776 -py01_kernels_trends,python,lim_polynomial/matern5_2/constant,pylibkriging,rmse,0.0246348 -py01_kernels_trends,python,lim_polynomial/matern5_2/constant,pylibkriging,q2,0.999766 -py01_kernels_trends,python,lim_polynomial/matern5_2/constant,pylibkriging,nlpd,-2.59598 -py01_kernels_trends,python,lim_polynomial/matern5_2/constant,pylibkriging BFGS10,rmse,0.0246338 -py01_kernels_trends,python,lim_polynomial/matern5_2/constant,pylibkriging BFGS10,q2,0.999766 -py01_kernels_trends,python,lim_polynomial/matern5_2/constant,pylibkriging BFGS10,nlpd,-2.59598 -py01_kernels_trends,python,lim_polynomial/matern5_2/linear,pylibkriging,fit_s,0.000878334 -py01_kernels_trends,python,lim_polynomial/matern5_2/linear,pylibkriging,rmse,0.0232483 -py01_kernels_trends,python,lim_polynomial/matern5_2/linear,pylibkriging,q2,0.999792 -py01_kernels_trends,python,lim_polynomial/matern5_2/linear,pylibkriging,nlpd,-2.58843 -py01_kernels_trends,python,lim_polynomial/matern5_2/interactive,pylibkriging,fit_s,0.000845909 -py01_kernels_trends,python,lim_polynomial/matern5_2/interactive,pylibkriging,rmse,0.0230248 -py01_kernels_trends,python,lim_polynomial/matern5_2/interactive,pylibkriging,q2,0.999796 -py01_kernels_trends,python,lim_polynomial/matern5_2/interactive,pylibkriging,nlpd,-2.58839 -py01_kernels_trends,python,lim_polynomial/matern5_2/quadratic,pylibkriging,fit_s,0.00176644 -py01_kernels_trends,python,lim_polynomial/matern5_2/quadratic,pylibkriging,rmse,0.209802 -py01_kernels_trends,python,lim_polynomial/matern5_2/quadratic,pylibkriging,q2,0.983025 -py01_kernels_trends,python,lim_polynomial/matern5_2/quadratic,pylibkriging,nlpd,-0.403998 -py01_kernels_trends,python,ishigami/gauss/constant,pylibkriging,fit_s,0.0820026 -py01_kernels_trends,python,ishigami/gauss/constant,pylibkriging,rmse,1.41975 -py01_kernels_trends,python,ishigami/gauss/constant,pylibkriging,q2,0.844642 -py01_kernels_trends,python,ishigami/gauss/constant,pylibkriging,nlpd,2.25894 -py01_kernels_trends,python,ishigami/gauss/linear,pylibkriging,fit_s,0.0247335 -py01_kernels_trends,python,ishigami/gauss/linear,pylibkriging,rmse,2.84579 -py01_kernels_trends,python,ishigami/gauss/linear,pylibkriging,q2,0.375811 -py01_kernels_trends,python,ishigami/gauss/linear,pylibkriging,nlpd,3.41805 -py01_kernels_trends,python,ishigami/gauss/interactive,pylibkriging,fit_s,0.0169995 -py01_kernels_trends,python,ishigami/gauss/interactive,pylibkriging,rmse,2.85366 -py01_kernels_trends,python,ishigami/gauss/interactive,pylibkriging,q2,0.372353 -py01_kernels_trends,python,ishigami/gauss/interactive,pylibkriging,nlpd,3.28643 -py01_kernels_trends,python,ishigami/gauss/quadratic,pylibkriging,fit_s,0.00751019 -py01_kernels_trends,python,ishigami/gauss/quadratic,pylibkriging,rmse,2.08982 -py01_kernels_trends,python,ishigami/gauss/quadratic,pylibkriging,q2,0.66339 -py01_kernels_trends,python,ishigami/gauss/quadratic,pylibkriging,nlpd,2.80099 -py01_kernels_trends,python,ishigami/exp/constant,pylibkriging,fit_s,0.00825953 -py01_kernels_trends,python,ishigami/exp/constant,pylibkriging,rmse,1.41632 -py01_kernels_trends,python,ishigami/exp/constant,pylibkriging,q2,0.845391 -py01_kernels_trends,python,ishigami/exp/constant,pylibkriging,nlpd,1.75239 -py01_kernels_trends,python,ishigami/exp/linear,pylibkriging,fit_s,0.00724149 -py01_kernels_trends,python,ishigami/exp/linear,pylibkriging,rmse,1.4095 -py01_kernels_trends,python,ishigami/exp/linear,pylibkriging,q2,0.846877 -py01_kernels_trends,python,ishigami/exp/linear,pylibkriging,nlpd,1.74779 -py01_kernels_trends,python,ishigami/exp/interactive,pylibkriging,fit_s,0.00750446 -py01_kernels_trends,python,ishigami/exp/interactive,pylibkriging,rmse,1.41511 -py01_kernels_trends,python,ishigami/exp/interactive,pylibkriging,q2,0.845657 -py01_kernels_trends,python,ishigami/exp/interactive,pylibkriging,nlpd,1.75024 -py01_kernels_trends,python,ishigami/exp/quadratic,pylibkriging,fit_s,0.00750661 -py01_kernels_trends,python,ishigami/exp/quadratic,pylibkriging,rmse,1.40329 -py01_kernels_trends,python,ishigami/exp/quadratic,pylibkriging,q2,0.848223 -py01_kernels_trends,python,ishigami/exp/quadratic,pylibkriging,nlpd,1.73254 -py01_kernels_trends,python,ishigami/matern3_2/constant,pylibkriging,fit_s,0.0483243 -py01_kernels_trends,python,ishigami/matern3_2/constant,pylibkriging,rmse,1.32974 -py01_kernels_trends,python,ishigami/matern3_2/constant,pylibkriging,q2,0.863717 -py01_kernels_trends,python,ishigami/matern3_2/constant,pylibkriging,nlpd,1.50266 -py01_kernels_trends,python,ishigami/matern3_2/linear,pylibkriging,fit_s,0.0462224 -py01_kernels_trends,python,ishigami/matern3_2/linear,pylibkriging,rmse,1.30244 -py01_kernels_trends,python,ishigami/matern3_2/linear,pylibkriging,q2,0.869255 -py01_kernels_trends,python,ishigami/matern3_2/linear,pylibkriging,nlpd,1.47906 -py01_kernels_trends,python,ishigami/matern3_2/interactive,pylibkriging,fit_s,0.0462408 -py01_kernels_trends,python,ishigami/matern3_2/interactive,pylibkriging,rmse,1.30591 -py01_kernels_trends,python,ishigami/matern3_2/interactive,pylibkriging,q2,0.868558 -py01_kernels_trends,python,ishigami/matern3_2/interactive,pylibkriging,nlpd,1.47384 -py01_kernels_trends,python,ishigami/matern3_2/quadratic,pylibkriging,fit_s,0.0412879 -py01_kernels_trends,python,ishigami/matern3_2/quadratic,pylibkriging,rmse,1.32197 -py01_kernels_trends,python,ishigami/matern3_2/quadratic,pylibkriging,q2,0.865305 -py01_kernels_trends,python,ishigami/matern3_2/quadratic,pylibkriging,nlpd,1.46827 -py01_kernels_trends,python,ishigami/matern5_2/constant,pylibkriging,fit_s,0.02212 -py01_kernels_trends,python,ishigami/matern5_2/constant,pylibkriging,rmse,3.00566 -py01_kernels_trends,python,ishigami/matern5_2/constant,pylibkriging,q2,0.303709 -py01_kernels_trends,python,ishigami/matern5_2/constant,pylibkriging,nlpd,3.42085 -py01_kernels_trends,python,ishigami/matern5_2/constant,pylibkriging BFGS10,rmse,1.33903 -py01_kernels_trends,python,ishigami/matern5_2/constant,pylibkriging BFGS10,q2,0.861806 -py01_kernels_trends,python,ishigami/matern5_2/constant,pylibkriging BFGS10,nlpd,1.66132 -py01_kernels_trends,python,ishigami/matern5_2/linear,pylibkriging,fit_s,0.0472796 -py01_kernels_trends,python,ishigami/matern5_2/linear,pylibkriging,rmse,1.3434 -py01_kernels_trends,python,ishigami/matern5_2/linear,pylibkriging,q2,0.860901 -py01_kernels_trends,python,ishigami/matern5_2/linear,pylibkriging,nlpd,1.65103 -py01_kernels_trends,python,ishigami/matern5_2/interactive,pylibkriging,fit_s,0.0477624 -py01_kernels_trends,python,ishigami/matern5_2/interactive,pylibkriging,rmse,1.34479 -py01_kernels_trends,python,ishigami/matern5_2/interactive,pylibkriging,q2,0.860615 -py01_kernels_trends,python,ishigami/matern5_2/interactive,pylibkriging,nlpd,1.64153 -py01_kernels_trends,python,ishigami/matern5_2/quadratic,pylibkriging,fit_s,0.0483804 -py01_kernels_trends,python,ishigami/matern5_2/quadratic,pylibkriging,rmse,1.37082 -py01_kernels_trends,python,ishigami/matern5_2/quadratic,pylibkriging,q2,0.855166 -py01_kernels_trends,python,ishigami/matern5_2/quadratic,pylibkriging,nlpd,1.62228 -py01_kernels_trends,python,hartmann3/gauss/constant,pylibkriging,fit_s,0.00703788 -py01_kernels_trends,python,hartmann3/gauss/constant,pylibkriging,rmse,0.156226 -py01_kernels_trends,python,hartmann3/gauss/constant,pylibkriging,q2,0.971116 -py01_kernels_trends,python,hartmann3/gauss/constant,pylibkriging,nlpd,-0.555623 -py01_kernels_trends,python,hartmann3/gauss/linear,pylibkriging,fit_s,0.0116901 -py01_kernels_trends,python,hartmann3/gauss/linear,pylibkriging,rmse,0.153297 -py01_kernels_trends,python,hartmann3/gauss/linear,pylibkriging,q2,0.972189 -py01_kernels_trends,python,hartmann3/gauss/linear,pylibkriging,nlpd,-0.568404 -py01_kernels_trends,python,hartmann3/gauss/interactive,pylibkriging,fit_s,0.00962472 -py01_kernels_trends,python,hartmann3/gauss/interactive,pylibkriging,rmse,0.712947 -py01_kernels_trends,python,hartmann3/gauss/interactive,pylibkriging,q2,0.398455 -py01_kernels_trends,python,hartmann3/gauss/interactive,pylibkriging,nlpd,1.74554 -py01_kernels_trends,python,hartmann3/gauss/quadratic,pylibkriging,fit_s,0.00845146 -py01_kernels_trends,python,hartmann3/gauss/quadratic,pylibkriging,rmse,0.73238 -py01_kernels_trends,python,hartmann3/gauss/quadratic,pylibkriging,q2,0.365216 -py01_kernels_trends,python,hartmann3/gauss/quadratic,pylibkriging,nlpd,2.14588 -py01_kernels_trends,python,hartmann3/exp/constant,pylibkriging,fit_s,0.00484872 -py01_kernels_trends,python,hartmann3/exp/constant,pylibkriging,rmse,0.347348 -py01_kernels_trends,python,hartmann3/exp/constant,pylibkriging,q2,0.857215 -py01_kernels_trends,python,hartmann3/exp/constant,pylibkriging,nlpd,0.352683 -py01_kernels_trends,python,hartmann3/exp/linear,pylibkriging,fit_s,0.00318956 -py01_kernels_trends,python,hartmann3/exp/linear,pylibkriging,rmse,0.347273 -py01_kernels_trends,python,hartmann3/exp/linear,pylibkriging,q2,0.857276 -py01_kernels_trends,python,hartmann3/exp/linear,pylibkriging,nlpd,0.343631 -py01_kernels_trends,python,hartmann3/exp/interactive,pylibkriging,fit_s,0.00330567 -py01_kernels_trends,python,hartmann3/exp/interactive,pylibkriging,rmse,0.378349 -py01_kernels_trends,python,hartmann3/exp/interactive,pylibkriging,q2,0.83059 -py01_kernels_trends,python,hartmann3/exp/interactive,pylibkriging,nlpd,0.385842 -py01_kernels_trends,python,hartmann3/exp/quadratic,pylibkriging,fit_s,0.00355411 -py01_kernels_trends,python,hartmann3/exp/quadratic,pylibkriging,rmse,0.355119 -py01_kernels_trends,python,hartmann3/exp/quadratic,pylibkriging,q2,0.850755 -py01_kernels_trends,python,hartmann3/exp/quadratic,pylibkriging,nlpd,0.334371 -py01_kernels_trends,python,hartmann3/matern3_2/constant,pylibkriging,fit_s,0.0107734 -py01_kernels_trends,python,hartmann3/matern3_2/constant,pylibkriging,rmse,0.227521 -py01_kernels_trends,python,hartmann3/matern3_2/constant,pylibkriging,q2,0.938737 -py01_kernels_trends,python,hartmann3/matern3_2/constant,pylibkriging,nlpd,-0.181207 -py01_kernels_trends,python,hartmann3/matern3_2/linear,pylibkriging,fit_s,0.0118749 -py01_kernels_trends,python,hartmann3/matern3_2/linear,pylibkriging,rmse,0.23038 -py01_kernels_trends,python,hartmann3/matern3_2/linear,pylibkriging,q2,0.937188 -py01_kernels_trends,python,hartmann3/matern3_2/linear,pylibkriging,nlpd,-0.162203 -py01_kernels_trends,python,hartmann3/matern3_2/interactive,pylibkriging,fit_s,0.00637007 -py01_kernels_trends,python,hartmann3/matern3_2/interactive,pylibkriging,rmse,0.315752 -py01_kernels_trends,python,hartmann3/matern3_2/interactive,pylibkriging,q2,0.88201 -py01_kernels_trends,python,hartmann3/matern3_2/interactive,pylibkriging,nlpd,0.898253 -py01_kernels_trends,python,hartmann3/matern3_2/quadratic,pylibkriging,fit_s,0.00598764 -py01_kernels_trends,python,hartmann3/matern3_2/quadratic,pylibkriging,rmse,0.291393 -py01_kernels_trends,python,hartmann3/matern3_2/quadratic,pylibkriging,q2,0.899513 -py01_kernels_trends,python,hartmann3/matern3_2/quadratic,pylibkriging,nlpd,0.53743 -py01_kernels_trends,python,hartmann3/matern5_2/constant,pylibkriging,fit_s,0.0100911 -py01_kernels_trends,python,hartmann3/matern5_2/constant,pylibkriging,rmse,0.190287 -py01_kernels_trends,python,hartmann3/matern5_2/constant,pylibkriging,q2,0.957148 -py01_kernels_trends,python,hartmann3/matern5_2/constant,pylibkriging,nlpd,-0.403054 -py01_kernels_trends,python,hartmann3/matern5_2/constant,pylibkriging BFGS10,rmse,0.190286 -py01_kernels_trends,python,hartmann3/matern5_2/constant,pylibkriging BFGS10,q2,0.957148 -py01_kernels_trends,python,hartmann3/matern5_2/constant,pylibkriging BFGS10,nlpd,-0.40306 -py01_kernels_trends,python,hartmann3/matern5_2/linear,pylibkriging,fit_s,0.0106812 -py01_kernels_trends,python,hartmann3/matern5_2/linear,pylibkriging,rmse,0.189633 -py01_kernels_trends,python,hartmann3/matern5_2/linear,pylibkriging,q2,0.957442 -py01_kernels_trends,python,hartmann3/matern5_2/linear,pylibkriging,nlpd,-0.410075 -py01_kernels_trends,python,hartmann3/matern5_2/interactive,pylibkriging,fit_s,0.00839925 -py01_kernels_trends,python,hartmann3/matern5_2/interactive,pylibkriging,rmse,0.307114 -py01_kernels_trends,python,hartmann3/matern5_2/interactive,pylibkriging,q2,0.888377 -py01_kernels_trends,python,hartmann3/matern5_2/interactive,pylibkriging,nlpd,1.50063 -py01_kernels_trends,python,hartmann3/matern5_2/quadratic,pylibkriging,fit_s,0.00724792 -py01_kernels_trends,python,hartmann3/matern5_2/quadratic,pylibkriging,rmse,0.285355 -py01_kernels_trends,python,hartmann3/matern5_2/quadratic,pylibkriging,q2,0.903634 -py01_kernels_trends,python,hartmann3/matern5_2/quadratic,pylibkriging,nlpd,0.898128 -py02_objectives_gradients,python,branin/LL,pylibkriging,rmse,71.4801 -py02_objectives_gradients,python,branin/LL,pylibkriging,q2,-1991.62 -py02_objectives_gradients,python,branin/LL,pylibkriging,nlpd,13603.3 -py02_objectives_gradients,python,branin/LOO,pylibkriging,rmse,71.5631 -py02_objectives_gradients,python,branin/LOO,pylibkriging,q2,-1996.25 -py02_objectives_gradients,python,branin/LOO,pylibkriging,nlpd,18836.2 -py02_objectives_gradients,python,branin/LMP,pylibkriging,rmse,71.5295 -py02_objectives_gradients,python,branin/LMP,pylibkriging,q2,-1994.38 -py02_objectives_gradients,python,branin/LMP,pylibkriging,nlpd,14603.3 -py02_objectives_gradients,python,hartmann3/LL,pylibkriging,rmse,1.30952 -py02_objectives_gradients,python,hartmann3/LL,pylibkriging,q2,-2013.29 -py02_objectives_gradients,python,hartmann3/LL,pylibkriging,nlpd,41.7128 -py02_objectives_gradients,python,hartmann3/LOO,pylibkriging,rmse,1.31436 -py02_objectives_gradients,python,hartmann3/LOO,pylibkriging,q2,-2028.22 -py02_objectives_gradients,python,hartmann3/LOO,pylibkriging,nlpd,49.7135 -py02_objectives_gradients,python,hartmann3/LMP,pylibkriging,rmse,1.31148 -py02_objectives_gradients,python,hartmann3/LMP,pylibkriging,q2,-2019.34 -py02_objectives_gradients,python,hartmann3/LMP,pylibkriging,nlpd,43.397 -py02_objectives_gradients,python,ishigami/LL,pylibkriging,rmse,5.11103 -py02_objectives_gradients,python,ishigami/LL,pylibkriging,q2,-1783.42 -py02_objectives_gradients,python,ishigami/LL,pylibkriging,nlpd,10.4546 -py02_objectives_gradients,python,ishigami/LOO,pylibkriging,rmse,5.81789 -py02_objectives_gradients,python,ishigami/LOO,pylibkriging,q2,-2311.12 -py02_objectives_gradients,python,ishigami/LOO,pylibkriging,nlpd,40.8323 -py02_objectives_gradients,python,ishigami/LMP,pylibkriging,rmse,5.18495 -py02_objectives_gradients,python,ishigami/LMP,pylibkriging,q2,-1835.41 -py02_objectives_gradients,python,ishigami/LMP,pylibkriging,nlpd,11.5894 -py02_objectives_gradients,python,friedman/LL,pylibkriging,rmse,6.87533 -py02_objectives_gradients,python,friedman/LL,pylibkriging,q2,-1991.33 -py02_objectives_gradients,python,friedman/LL,pylibkriging,nlpd,651.706 -py02_objectives_gradients,python,friedman/LOO,pylibkriging,rmse,6.87323 -py02_objectives_gradients,python,friedman/LOO,pylibkriging,q2,-1990.12 -py02_objectives_gradients,python,friedman/LOO,pylibkriging,nlpd,484.115 -py02_objectives_gradients,python,friedman/LMP,pylibkriging,rmse,6.87647 -py02_objectives_gradients,python,friedman/LMP,pylibkriging,q2,-1991.99 -py02_objectives_gradients,python,friedman/LMP,pylibkriging,nlpd,658.808 -py03_noise,python,forrester,nugget,rmse,0.198091 -py03_noise,python,forrester,nugget,q2,0.998087 -py03_noise,python,forrester,nugget,nlpd,0.286631 -py03_noise,python,forrester,noise-free fit,rmse,0.487916 -py03_noise,python,forrester,noise-free fit,q2,0.988392 -py03_noise,python,forrester,noise-free fit,nlpd,862.687 -py03_noise,python,friedman,nugget,rmse,0.467335 -py03_noise,python,friedman,nugget,q2,0.99089 -py03_noise,python,friedman,nugget,nlpd,1.07073 -py03_noise,python,friedman,noise-free fit,rmse,1.27156 -py03_noise,python,friedman,noise-free fit,q2,0.932559 -py03_noise,python,friedman,noise-free fit,nlpd,1.64495 -py03_noise,python,gramacy_lee_2012/hetero,known variances,rmse,0.168741 -py03_noise,python,gramacy_lee_2012/hetero,known variances,q2,0.982515 -py03_noise,python,gramacy_lee_2012/hetero,known variances,nlpd,-0.570574 -py03_noise,python,gramacy_lee_2012/hetero,nugget,rmse,0.170945 -py03_noise,python,gramacy_lee_2012/hetero,nugget,q2,0.982055 -py03_noise,python,gramacy_lee_2012/hetero,nugget,nlpd,-0.221568 -py05_warpkriging,python,gl2012,Kriging,rmse,0.167263 -py05_warpkriging,python,gl2012,Kriging,q2,0.981176 -py05_warpkriging,python,gl2012,Kriging,nlpd,-1.08917 -py05_warpkriging,python,gl2012,affine,rmse,0.167263 -py05_warpkriging,python,gl2012,affine,q2,0.981176 -py05_warpkriging,python,gl2012,affine,nlpd,-1.08917 -py05_warpkriging,python,gl2012,boxcox,rmse,0.16759 -py05_warpkriging,python,gl2012,boxcox,q2,0.981103 -py05_warpkriging,python,gl2012,boxcox,nlpd,282.422 -py05_warpkriging,python,gl2012,kumaraswamy,rmse,0.168286 -py05_warpkriging,python,gl2012,kumaraswamy,q2,0.980945 -py05_warpkriging,python,gl2012,kumaraswamy,nlpd,2.04577e+06 -py05_warpkriging,python,gl2012,knots(5),rmse,0.16791 -py05_warpkriging,python,gl2012,knots(5),q2,0.98103 -py05_warpkriging,python,gl2012,knots(5),nlpd,2.04376e+06 -py05_warpkriging,python,gl2012,neural_mono(8),rmse,0.170824 -py05_warpkriging,python,gl2012,neural_mono(8),q2,0.980366 -py05_warpkriging,python,gl2012,neural_mono(8),nlpd,-0.265141 -py05_warpkriging,python,gl2008,Kriging,rmse,0.0534649 -py05_warpkriging,python,gl2008,Kriging,q2,0.585636 -py05_warpkriging,python,gl2008,Kriging,nlpd,-1.74569 -py05_warpkriging,python,gl2008,kumaraswamy,rmse,0.0535298 -py05_warpkriging,python,gl2008,kumaraswamy,q2,0.58463 -py05_warpkriging,python,gl2008,kumaraswamy,nlpd,-1.74552 -py05_warpkriging,python,mixed,Kriging (level as number),rmse,16.7693 -py05_warpkriging,python,mixed,Kriging (level as number),q2,0.838453 -py05_warpkriging,python,mixed,Kriging (level as number),nlpd,3.81063 -py05_warpkriging,python,mixed,"categorical(5,2)",rmse,16.1438 -py05_warpkriging,python,mixed,"categorical(5,2)",q2,0.85028 -py05_warpkriging,python,mixed,"categorical(5,2)",nlpd,3.3206 -py05_warpkriging,python,ordinal,Kriging (rank as number),rmse,0.213845 -py05_warpkriging,python,ordinal,Kriging (rank as number),q2,0.953177 -py05_warpkriging,python,ordinal,Kriging (rank as number),nlpd,0.810969 -py05_warpkriging,python,ordinal,ordinal(6),rmse,0.213043 -py05_warpkriging,python,ordinal,ordinal(6),q2,0.953528 -py05_warpkriging,python,ordinal,ordinal(6),nlpd,0.78888 -py06_high_dimension_mlp,python,otl_circuit,unit cube,fit_s,0.00796986 -py06_high_dimension_mlp,python,otl_circuit,unit cube,rmse,0.0150514 -py06_high_dimension_mlp,python,otl_circuit,unit cube,q2,0.999836 -py06_high_dimension_mlp,python,otl_circuit,unit cube,nlpd,-2.90452 -py06_high_dimension_mlp,python,otl_circuit,native + normalize,rmse,0.0150514 -py06_high_dimension_mlp,python,otl_circuit,native + normalize,q2,0.999836 -py06_high_dimension_mlp,python,otl_circuit,native + normalize,nlpd,-2.90452 -py06_high_dimension_mlp,python,piston,unit cube,fit_s,0.023371 -py06_high_dimension_mlp,python,piston,unit cube,rmse,0.0104258 -py06_high_dimension_mlp,python,piston,unit cube,q2,0.994462 -py06_high_dimension_mlp,python,piston,unit cube,nlpd,-3.49038 -py06_high_dimension_mlp,python,piston,native + normalize,rmse,0.0104258 -py06_high_dimension_mlp,python,piston,native + normalize,q2,0.994462 -py06_high_dimension_mlp,python,piston,native + normalize,nlpd,-3.49038 -py06_high_dimension_mlp,python,borehole,unit cube,fit_s,0.0214174 -py06_high_dimension_mlp,python,borehole,unit cube,rmse,0.893341 -py06_high_dimension_mlp,python,borehole,unit cube,q2,0.999599 -py06_high_dimension_mlp,python,borehole,unit cube,nlpd,1.08433 -py06_high_dimension_mlp,python,borehole,native + normalize,rmse,0.893341 -py06_high_dimension_mlp,python,borehole,native + normalize,q2,0.999599 -py06_high_dimension_mlp,python,borehole,native + normalize,nlpd,1.08433 -py06_high_dimension_mlp,python,robot_arm,unit cube,fit_s,0.187081 -py06_high_dimension_mlp,python,robot_arm,unit cube,rmse,0.358507 -py06_high_dimension_mlp,python,robot_arm,unit cube,q2,0.571154 -py06_high_dimension_mlp,python,robot_arm,unit cube,nlpd,0.355106 -py06_high_dimension_mlp,python,robot_arm,native + normalize,rmse,0.358507 -py06_high_dimension_mlp,python,robot_arm,native + normalize,q2,0.571154 -py06_high_dimension_mlp,python,robot_arm,native + normalize,nlpd,0.355106 -py06_high_dimension_mlp,python,dette_pepelyshev,unit cube,fit_s,0.0181198 -py06_high_dimension_mlp,python,dette_pepelyshev,unit cube,rmse,0.424908 -py06_high_dimension_mlp,python,dette_pepelyshev,unit cube,q2,0.997757 -py06_high_dimension_mlp,python,dette_pepelyshev,unit cube,nlpd,0.510276 -py06_high_dimension_mlp,python,dette_pepelyshev,native + normalize,rmse,0.424908 -py06_high_dimension_mlp,python,dette_pepelyshev,native + normalize,q2,0.997757 -py06_high_dimension_mlp,python,dette_pepelyshev,native + normalize,nlpd,0.510276 -py06_high_dimension_mlp,python,wing_weight,unit cube,fit_s,0.0303686 -py06_high_dimension_mlp,python,wing_weight,unit cube,rmse,0.809743 -py06_high_dimension_mlp,python,wing_weight,unit cube,q2,0.999716 -py06_high_dimension_mlp,python,wing_weight,unit cube,nlpd,1.07161 -py06_high_dimension_mlp,python,wing_weight,native + normalize,rmse,0.809743 -py06_high_dimension_mlp,python,wing_weight,native + normalize,q2,0.999716 -py06_high_dimension_mlp,python,wing_weight,native + normalize,nlpd,1.07161 -py06_high_dimension_mlp,python,welch,unit cube,fit_s,0.500499 -py06_high_dimension_mlp,python,welch,unit cube,rmse,0.167729 -py06_high_dimension_mlp,python,welch,unit cube,q2,0.993452 -py06_high_dimension_mlp,python,welch,unit cube,nlpd,-0.429138 -py06_high_dimension_mlp,python,welch,native + normalize,rmse,0.167692 -py06_high_dimension_mlp,python,welch,native + normalize,q2,0.993454 -py06_high_dimension_mlp,python,welch,native + normalize,nlpd,-0.429291 -py06_high_dimension_mlp,python,projected_branin_10d,Kriging,rmse,4.42284 -py06_high_dimension_mlp,python,projected_branin_10d,Kriging,q2,0.959122 -py06_high_dimension_mlp,python,projected_branin_10d,Kriging,nlpd,2.89997 -py06_high_dimension_mlp,python,projected_branin_10d,MLPKriging tanh BFGS,fit_s,0.171914 -py06_high_dimension_mlp,python,projected_branin_10d,MLPKriging tanh BFGS,rmse,59.5068 -py06_high_dimension_mlp,python,projected_branin_10d,MLPKriging tanh BFGS,q2,-6.39973 -py06_high_dimension_mlp,python,projected_branin_10d,MLPKriging tanh BFGS,nlpd,100.745 -py06_high_dimension_mlp,python,projected_branin_10d,MLPKriging tanh BFGS10,fit_s,0.53444 -py06_high_dimension_mlp,python,projected_branin_10d,MLPKriging tanh BFGS10,rmse,4.41161 -py06_high_dimension_mlp,python,projected_branin_10d,MLPKriging tanh BFGS10,q2,0.95933 -py06_high_dimension_mlp,python,projected_branin_10d,MLPKriging tanh BFGS10,nlpd,28.4236 -py06_high_dimension_mlp,python,projected_branin_10d,MLPKriging selu BFGS,fit_s,0.145419 -py06_high_dimension_mlp,python,projected_branin_10d,MLPKriging selu BFGS,rmse,58.9844 -py06_high_dimension_mlp,python,projected_branin_10d,MLPKriging selu BFGS,q2,-6.27038 -py06_high_dimension_mlp,python,projected_branin_10d,MLPKriging selu BFGS,nlpd,46.9347 -py06_high_dimension_mlp,python,projected_branin_10d,MLPKriging selu BFGS10,fit_s,0.518803 -py06_high_dimension_mlp,python,projected_branin_10d,MLPKriging selu BFGS10,rmse,3.79093 -py06_high_dimension_mlp,python,projected_branin_10d,MLPKriging selu BFGS10,q2,0.969969 -py06_high_dimension_mlp,python,projected_branin_10d,MLPKriging selu BFGS10,nlpd,12.6814 -py07_scalability,python,hartmann3/n=1500,exact,fit_s,11.2644 -py07_scalability,python,hartmann3/n=1500,exact,rmse,0.000259233 -py07_scalability,python,hartmann3/n=1500,exact,q2,1 -py07_scalability,python,hartmann3/n=1500,exact,nlpd,-6.92184 -py07_scalability,python,hartmann3/n=1500,nested NK (20 groups),fit_s,0.324342 -py07_scalability,python,hartmann3/n=1500,nested NK (20 groups),rmse,0.0029908 -py07_scalability,python,hartmann3/n=1500,nested NK (20 groups),q2,0.999991 -py07_scalability,python,hartmann3/n=1500,nested NK (20 groups),nlpd,-4.85946 -py07_scalability,python,hartmann3/n=5000,nested NK (20 groups),fit_s,3.25297 -py07_scalability,python,hartmann3/n=5000,nested NK (20 groups),rmse,0.000222655 -py07_scalability,python,hartmann3/n=5000,nested NK (20 groups),q2,1 -py07_scalability,python,hartmann3/n=5000,nested NK (20 groups),nlpd,-7.44453 -py07_scalability,python,hartmann3/n=1500,nested rBCM (20 groups),fit_s,0.273419 -py07_scalability,python,hartmann3/n=1500,nested rBCM (20 groups),rmse,0.00304431 -py07_scalability,python,hartmann3/n=1500,nested rBCM (20 groups),q2,0.99999 -py07_scalability,python,hartmann3/n=1500,nested rBCM (20 groups),nlpd,-4.50201 -py07_scalability,python,hartmann3/n=5000,nested rBCM (20 groups),fit_s,2.67811 -py07_scalability,python,hartmann3/n=5000,nested rBCM (20 groups),rmse,0.000224517 -py07_scalability,python,hartmann3/n=5000,nested rBCM (20 groups),q2,1 -py07_scalability,python,hartmann3/n=5000,nested rBCM (20 groups),nlpd,-7.18182 -py07_scalability,python,hartmann3/n=1500,subset of data (500),fit_s,0.685586 -py07_scalability,python,hartmann3/n=1500,subset of data (500),rmse,0.00269153 -py07_scalability,python,hartmann3/n=1500,subset of data (500),q2,0.999992 -py07_scalability,python,hartmann3/n=1500,subset of data (500),nlpd,-4.68641 -py07_scalability,python,hartmann3/n=5000,subset of data (500),fit_s,0.906366 -py07_scalability,python,hartmann3/n=5000,subset of data (500),rmse,0.00247552 -py07_scalability,python,hartmann3/n=5000,subset of data (500),q2,0.999994 -py07_scalability,python,hartmann3/n=5000,subset of data (500),nlpd,-4.71359 -py07_scalability,python,otl_circuit/n=1500,exact,fit_s,10.9405 -py07_scalability,python,otl_circuit/n=1500,exact,rmse,9.81598e-05 -py07_scalability,python,otl_circuit/n=1500,exact,q2,1 -py07_scalability,python,otl_circuit/n=1500,exact,nlpd,-8.21962 -py07_scalability,python,otl_circuit/n=1500,nested NK (20 groups),fit_s,0.279359 -py07_scalability,python,otl_circuit/n=1500,nested NK (20 groups),rmse,0.00226873 -py07_scalability,python,otl_circuit/n=1500,nested NK (20 groups),q2,0.999996 -py07_scalability,python,otl_circuit/n=1500,nested NK (20 groups),nlpd,-4.90542 -py07_scalability,python,otl_circuit/n=5000,nested NK (20 groups),fit_s,3.08712 -py07_scalability,python,otl_circuit/n=5000,nested NK (20 groups),rmse,0.000261686 -py07_scalability,python,otl_circuit/n=5000,nested NK (20 groups),q2,1 -py07_scalability,python,otl_circuit/n=5000,nested NK (20 groups),nlpd,-7.20623 -py07_scalability,python,otl_circuit/n=1500,nested rBCM (20 groups),fit_s,0.251989 -py07_scalability,python,otl_circuit/n=1500,nested rBCM (20 groups),rmse,0.00291869 -py07_scalability,python,otl_circuit/n=1500,nested rBCM (20 groups),q2,0.999993 -py07_scalability,python,otl_circuit/n=1500,nested rBCM (20 groups),nlpd,-2.55512 -py07_scalability,python,otl_circuit/n=5000,nested rBCM (20 groups),fit_s,3.67768 -py07_scalability,python,otl_circuit/n=5000,nested rBCM (20 groups),rmse,0.000278149 -py07_scalability,python,otl_circuit/n=5000,nested rBCM (20 groups),q2,1 -py07_scalability,python,otl_circuit/n=5000,nested rBCM (20 groups),nlpd,-4.34927 -py07_scalability,python,otl_circuit/n=1500,subset of data (500),fit_s,0.709236 -py07_scalability,python,otl_circuit/n=1500,subset of data (500),rmse,0.0004286 -py07_scalability,python,otl_circuit/n=1500,subset of data (500),q2,1 -py07_scalability,python,otl_circuit/n=1500,subset of data (500),nlpd,-6.72678 -py07_scalability,python,otl_circuit/n=5000,subset of data (500),fit_s,0.905534 -py07_scalability,python,otl_circuit/n=5000,subset of data (500),rmse,0.000446879 -py07_scalability,python,otl_circuit/n=5000,subset of data (500),q2,1 -py07_scalability,python,otl_circuit/n=5000,subset of data (500),nlpd,-6.78844 -py07_scalability,python,branin/nested,NK,rmse,0.0343984 -py07_scalability,python,branin/nested,NK,q2,1 -py07_scalability,python,branin/nested,NK,nlpd,2.00083e+14 -py07_scalability,python,branin/nested,PoE,rmse,0.0354265 -py07_scalability,python,branin/nested,PoE,q2,1 -py07_scalability,python,branin/nested,PoE,nlpd,-1.64533 -py07_scalability,python,branin/nested,gPoE,rmse,0.0354265 -py07_scalability,python,branin/nested,gPoE,q2,1 -py07_scalability,python,branin/nested,gPoE,nlpd,-0.621836 -py07_scalability,python,branin/nested,BCM,rmse,0.0354196 -py07_scalability,python,branin/nested,BCM,q2,1 -py07_scalability,python,branin/nested,BCM,nlpd,-1.64533 -py07_scalability,python,branin/nested,rBCM,rmse,0.035287 -py07_scalability,python,branin/nested,rBCM,q2,1 -py07_scalability,python,branin/nested,rBCM,nlpd,-2.61369 -py10_compare_python_packages,python,branin,pylibkriging,q2,0.999944 -py10_compare_python_packages,python,branin,pylibkriging,nlpd,0.137225 -py10_compare_python_packages,python,branin,pylibkriging,fit_s,0.00151396 -py10_compare_python_packages,python,branin,pylibkriging BFGS10,q2,0.999945 -py10_compare_python_packages,python,branin,pylibkriging BFGS10,nlpd,0.137254 -py10_compare_python_packages,python,branin,pylibkriging BFGS10,fit_s,0.0161211 -py10_compare_python_packages,python,branin,scikit-learn,q2,0.999597 -py10_compare_python_packages,python,branin,scikit-learn,nlpd,0.928508 -py10_compare_python_packages,python,branin,scikit-learn,fit_s,0.147773 -py10_compare_python_packages,python,branin,SMT,q2,0.999944 -py10_compare_python_packages,python,branin,SMT,nlpd,0.137259 -py10_compare_python_packages,python,branin,SMT,fit_s,0.792991 -py10_compare_python_packages,python,branin,OpenTURNS,q2,0.998221 -py10_compare_python_packages,python,branin,OpenTURNS,nlpd,1.55599 -py10_compare_python_packages,python,branin,OpenTURNS,fit_s,0.00512815 -py10_compare_python_packages,python,six_hump_camel,pylibkriging,q2,0.999418 -py10_compare_python_packages,python,six_hump_camel,pylibkriging,nlpd,0.0429346 -py10_compare_python_packages,python,six_hump_camel,pylibkriging,fit_s,0.00358009 -py10_compare_python_packages,python,six_hump_camel,pylibkriging BFGS10,q2,0.999418 -py10_compare_python_packages,python,six_hump_camel,pylibkriging BFGS10,nlpd,0.0429347 -py10_compare_python_packages,python,six_hump_camel,pylibkriging BFGS10,fit_s,0.0500369 -py10_compare_python_packages,python,six_hump_camel,scikit-learn,q2,0.0163311 -py10_compare_python_packages,python,six_hump_camel,scikit-learn,nlpd,4.63763 -py10_compare_python_packages,python,six_hump_camel,scikit-learn,fit_s,0.156482 -py10_compare_python_packages,python,six_hump_camel,SMT,q2,0.999594 -py10_compare_python_packages,python,six_hump_camel,SMT,nlpd,-0.121007 -py10_compare_python_packages,python,six_hump_camel,SMT,fit_s,0.941661 -py10_compare_python_packages,python,six_hump_camel,OpenTURNS,q2,0.956713 -py10_compare_python_packages,python,six_hump_camel,OpenTURNS,nlpd,2.55446 -py10_compare_python_packages,python,six_hump_camel,OpenTURNS,fit_s,0.00930476 -py10_compare_python_packages,python,hartmann3,pylibkriging,q2,0.986148 -py10_compare_python_packages,python,hartmann3,pylibkriging,nlpd,-0.847247 -py10_compare_python_packages,python,hartmann3,pylibkriging,fit_s,0.023098 -py10_compare_python_packages,python,hartmann3,pylibkriging BFGS10,q2,0.986148 -py10_compare_python_packages,python,hartmann3,pylibkriging BFGS10,nlpd,-0.847249 -py10_compare_python_packages,python,hartmann3,pylibkriging BFGS10,fit_s,0.113887 -py10_compare_python_packages,python,hartmann3,scikit-learn,q2,0.983889 -py10_compare_python_packages,python,hartmann3,scikit-learn,nlpd,-0.782697 -py10_compare_python_packages,python,hartmann3,scikit-learn,fit_s,0.208279 -py10_compare_python_packages,python,hartmann3,SMT,q2,0.986148 -py10_compare_python_packages,python,hartmann3,SMT,nlpd,-0.847249 -py10_compare_python_packages,python,hartmann3,SMT,fit_s,1.35887 -py10_compare_python_packages,python,hartmann3,OpenTURNS,q2,0.984712 -py10_compare_python_packages,python,hartmann3,OpenTURNS,nlpd,-0.809851 -py10_compare_python_packages,python,hartmann3,OpenTURNS,fit_s,0.0169773 -py10_compare_python_packages,python,ishigami,pylibkriging,q2,0.259556 -py10_compare_python_packages,python,ishigami,pylibkriging,nlpd,4.38991 -py10_compare_python_packages,python,ishigami,pylibkriging,fit_s,0.0264924 -py10_compare_python_packages,python,ishigami,pylibkriging BFGS10,q2,0.935722 -py10_compare_python_packages,python,ishigami,pylibkriging BFGS10,nlpd,0.857443 -py10_compare_python_packages,python,ishigami,pylibkriging BFGS10,fit_s,0.322963 -py10_compare_python_packages,python,ishigami,scikit-learn,q2,0.831142 -py10_compare_python_packages,python,ishigami,scikit-learn,nlpd,1.5455 -py10_compare_python_packages,python,ishigami,scikit-learn,fit_s,0.512728 -py10_compare_python_packages,python,ishigami,SMT,q2,0.51196 -py10_compare_python_packages,python,ishigami,SMT,nlpd,3.31349 -py10_compare_python_packages,python,ishigami,SMT,fit_s,2.70852 -py10_compare_python_packages,python,ishigami,OpenTURNS,q2,0.887251 -py10_compare_python_packages,python,ishigami,OpenTURNS,nlpd,1.39348 -py10_compare_python_packages,python,ishigami,OpenTURNS,fit_s,0.0510182 -py10_compare_python_packages,python,friedman,pylibkriging,q2,0.999771 -py10_compare_python_packages,python,friedman,pylibkriging,nlpd,-1.29726 -py10_compare_python_packages,python,friedman,pylibkriging,fit_s,0.0165708 -py10_compare_python_packages,python,friedman,pylibkriging BFGS10,q2,0.999771 -py10_compare_python_packages,python,friedman,pylibkriging BFGS10,nlpd,-1.29727 -py10_compare_python_packages,python,friedman,pylibkriging BFGS10,fit_s,0.279774 -py10_compare_python_packages,python,friedman,scikit-learn,q2,0.999754 -py10_compare_python_packages,python,friedman,scikit-learn,nlpd,-1.37702 -py10_compare_python_packages,python,friedman,scikit-learn,fit_s,0.308127 -py10_compare_python_packages,python,friedman,SMT,q2,0.999899 -py10_compare_python_packages,python,friedman,SMT,nlpd,-1.871 -py10_compare_python_packages,python,friedman,SMT,fit_s,4.71948 -py10_compare_python_packages,python,friedman,OpenTURNS,q2,0.990862 -py10_compare_python_packages,python,friedman,OpenTURNS,nlpd,0.469828 -py10_compare_python_packages,python,friedman,OpenTURNS,fit_s,0.0511642 -py10_compare_python_packages,python,otl_circuit,pylibkriging,q2,0.999922 -py10_compare_python_packages,python,otl_circuit,pylibkriging,nlpd,-3.46228 -py10_compare_python_packages,python,otl_circuit,pylibkriging,fit_s,0.0136361 -py10_compare_python_packages,python,otl_circuit,pylibkriging BFGS10,q2,0.999922 -py10_compare_python_packages,python,otl_circuit,pylibkriging BFGS10,nlpd,-3.46227 -py10_compare_python_packages,python,otl_circuit,pylibkriging BFGS10,fit_s,0.168652 -py10_compare_python_packages,python,otl_circuit,scikit-learn,q2,0.999906 -py10_compare_python_packages,python,otl_circuit,scikit-learn,nlpd,-3.43062 -py10_compare_python_packages,python,otl_circuit,scikit-learn,fit_s,0.64982 -py10_compare_python_packages,python,otl_circuit,SMT,q2,0.999954 -py10_compare_python_packages,python,otl_circuit,SMT,nlpd,-3.35016 -py10_compare_python_packages,python,otl_circuit,SMT,fit_s,6.837 -py10_compare_python_packages,python,otl_circuit,OpenTURNS,q2,0.99777 -py10_compare_python_packages,python,otl_circuit,OpenTURNS,nlpd,-1.61723 -py10_compare_python_packages,python,otl_circuit,OpenTURNS,fit_s,0.0662544 -py10_compare_python_packages,python,piston,pylibkriging,q2,0.997498 -py10_compare_python_packages,python,piston,pylibkriging,nlpd,-3.80667 -py10_compare_python_packages,python,piston,pylibkriging,fit_s,0.03055 -py10_compare_python_packages,python,piston,pylibkriging BFGS10,q2,0.997498 -py10_compare_python_packages,python,piston,pylibkriging BFGS10,nlpd,-3.8067 -py10_compare_python_packages,python,piston,pylibkriging BFGS10,fit_s,0.326413 -py10_compare_python_packages,python,piston,scikit-learn,q2,0.998684 -py10_compare_python_packages,python,piston,scikit-learn,nlpd,-4.10497 -py10_compare_python_packages,python,piston,scikit-learn,fit_s,0.856336 -py10_compare_python_packages,python,piston,SMT,q2,0.997511 -py10_compare_python_packages,python,piston,SMT,nlpd,-3.86618 -py10_compare_python_packages,python,piston,SMT,fit_s,12.0067 -py10_compare_python_packages,python,piston,OpenTURNS,q2,0.990631 -py10_compare_python_packages,python,piston,OpenTURNS,nlpd,-2.9906 -py10_compare_python_packages,python,piston,OpenTURNS,fit_s,0.137117 -py10_compare_python_packages,python,borehole,pylibkriging,q2,0.999694 -py10_compare_python_packages,python,borehole,pylibkriging,nlpd,1.00118 -py10_compare_python_packages,python,borehole,pylibkriging,fit_s,0.0166521 -py10_compare_python_packages,python,borehole,pylibkriging BFGS10,q2,0.999694 -py10_compare_python_packages,python,borehole,pylibkriging BFGS10,nlpd,1.00121 -py10_compare_python_packages,python,borehole,pylibkriging BFGS10,fit_s,0.200685 -py10_compare_python_packages,python,borehole,scikit-learn,q2,0.999929 -py10_compare_python_packages,python,borehole,scikit-learn,nlpd,0.194455 -py10_compare_python_packages,python,borehole,scikit-learn,fit_s,1.18352 -py10_compare_python_packages,python,borehole,SMT,q2,0.999925 -py10_compare_python_packages,python,borehole,SMT,nlpd,0.279122 -py10_compare_python_packages,python,borehole,SMT,fit_s,12.3314 -py10_compare_python_packages,python,borehole,OpenTURNS,q2,0.996206 -py10_compare_python_packages,python,borehole,OpenTURNS,nlpd,2.55431 -py10_compare_python_packages,python,borehole,OpenTURNS,fit_s,0.121907 -py10_compare_python_packages,python,wing_weight,pylibkriging,q2,0.99972 -py10_compare_python_packages,python,wing_weight,pylibkriging,nlpd,1.11517 -py10_compare_python_packages,python,wing_weight,pylibkriging,fit_s,0.0297599 -py10_compare_python_packages,python,wing_weight,pylibkriging BFGS10,q2,0.99972 -py10_compare_python_packages,python,wing_weight,pylibkriging BFGS10,nlpd,1.11512 -py10_compare_python_packages,python,wing_weight,pylibkriging BFGS10,fit_s,0.335854 -py10_compare_python_packages,python,wing_weight,scikit-learn,q2,0.998048 -py10_compare_python_packages,python,wing_weight,scikit-learn,nlpd,2.57304 -py10_compare_python_packages,python,wing_weight,scikit-learn,fit_s,0.833252 -py10_compare_python_packages,python,wing_weight,SMT,q2,0.999898 -py10_compare_python_packages,python,wing_weight,SMT,nlpd,0.604023 -py10_compare_python_packages,python,wing_weight,SMT,fit_s,26.7889 -py10_compare_python_packages,python,wing_weight,OpenTURNS,q2,0.993145 -py10_compare_python_packages,python,wing_weight,OpenTURNS,nlpd,2.83627 -py10_compare_python_packages,python,wing_weight,OpenTURNS,fit_s,0.253929 -py10_compare_python_packages,python,hartmann6,pylibkriging,q2,0.422939 -py10_compare_python_packages,python,hartmann6,pylibkriging,nlpd,0.0965158 -py10_compare_python_packages,python,hartmann6,pylibkriging,fit_s,0.068651 -py10_compare_python_packages,python,hartmann6,pylibkriging BFGS10,q2,0.72352 -py10_compare_python_packages,python,hartmann6,pylibkriging BFGS10,nlpd,-0.156898 -py10_compare_python_packages,python,hartmann6,pylibkriging BFGS10,fit_s,0.877097 -py10_compare_python_packages,python,hartmann6,scikit-learn,q2,0.69049 -py10_compare_python_packages,python,hartmann6,scikit-learn,nlpd,-0.137845 -py10_compare_python_packages,python,hartmann6,scikit-learn,fit_s,1.43153 -py10_compare_python_packages,python,hartmann6,SMT,q2,0.708463 -py10_compare_python_packages,python,hartmann6,SMT,nlpd,-0.0177081 -py10_compare_python_packages,python,hartmann6,SMT,fit_s,11.2226 -py10_compare_python_packages,python,hartmann6,OpenTURNS,q2,0.696177 -py10_compare_python_packages,python,hartmann6,OpenTURNS,nlpd,-0.132838 -py10_compare_python_packages,python,hartmann6,OpenTURNS,fit_s,0.141383 diff --git a/julia_Kriging_cross-binding.ipynb b/julia_Kriging_cross-binding.ipynb new file mode 100644 index 0000000..68997e7 --- /dev/null +++ b/julia_Kriging_cross-binding.ipynb @@ -0,0 +1,95 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "c000", + "metadata": {}, + "source": [ + "# Julia · Kriging — cross-binding consistency: the Julia side\n", + "\n", + "Same protocol as `python_Kriging_cross-binding-reference.ipynb` (see there), on\n", + "the shared datasets of `data/`; writes `results/xbind_julia.csv`, compared with\n", + "the Python values by `tools/report.py`.\n", + "\n", + "The module is `JLibKriging` (the packaged binding, [JLibKriging.jl](https://github.com/libKriging/JLibKriging.jl)),\n", + "imported under a short name so that its exported accessors (`X`, `y`, `theta`, ...)\n", + "do not clash with the variables of the notebook." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c001", + "metadata": {}, + "outputs": [], + "source": [ + "import JLibKriging as LK\n", + "include(\"tools/julia/checks.jl\")\n", + "s = Suite(\"julia_Kriging_cross-binding\")\n", + "rows = Vector{Vector{String}}()\n", + "record!(dataset, quantity, values) =\n", + " for (i, v) in enumerate(vec(Float64.(collect(values))))\n", + " push!(rows, [\"julia\", dataset, quantity, string(i), @sprintf(\"%.17g\", v)])\n", + " end" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c002", + "metadata": {}, + "outputs": [], + "source": [ + "for name in (\"branin\", \"hartmann3\", \"borehole\")\n", + " Xd = read_data(name, \"X\"); yd = vec(read_data(name, \"y\")); Xt = read_data(name, \"Xt\")[1:50, :]\n", + " d = size(Xd, 2)\n", + " theta0 = fill(0.3, d)\n", + " kf = LK.Kriging(yd, Xd, \"matern5_2\"; regmodel=\"constant\", optim=\"none\", theta=theta0, is_theta_estim=false)\n", + " record!(name, \"fixed_sigma2\", LK.sigma2(kf))\n", + " record!(name, \"fixed_beta\", LK.beta(kf))\n", + " record!(name, \"fixed_ll\", LK.log_likelihood(kf))\n", + " record!(name, \"fixed_loo\", LK.leave_one_out(kf))\n", + " p = LK.predict(kf, Xt; return_stdev=true)\n", + " record!(name, \"fixed_mean\", p.mean)\n", + " record!(name, \"fixed_stdev\", p.stdev)\n", + " llf = LK.log_likelihood_fun(kf, theta0; return_grad=true)\n", + " record!(name, \"llfun\", llf.ll)\n", + " record!(name, \"llfun_grad\", llf.grad)\n", + " check_close!(s, \"$name: log_likelihood() == log_likelihood_fun(theta0)\", LK.log_likelihood(kf), llf.ll; rtol=1e-10)\n", + " k = LK.Kriging(yd, Xd, \"matern5_2\")\n", + " record!(name, \"fit_theta\", LK.theta(k))\n", + " record!(name, \"fit_ll\", LK.log_likelihood(k))\n", + " record!(name, \"fit_mean\", LK.predict(k, Xt; return_stdev=false).mean)\n", + " record!(name, \"sim\", LK.simulate(kf, 3, 123, Xt[1:5, :]))\n", + "end\n", + "open(joinpath(RESULTS_DIR, \"xbind_julia.csv\"), \"w\") do io\n", + " println(io, \"lang,dataset,quantity,index,value\")\n", + " for r in rows; println(io, join(r, \",\")); end\n", + "end\n", + "println(length(rows), \" values written\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c003", + "metadata": {}, + "outputs": [], + "source": [ + "finalize!(s)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Julia 1.11", + "language": "julia", + "name": "julia-1.11" + }, + "language_info": { + "name": "julia" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/julia/01_cross_binding.ipynb b/julia_Kriging_vs-GaussianProcesses.ipynb similarity index 54% rename from notebooks/julia/01_cross_binding.ipynb rename to julia_Kriging_vs-GaussianProcesses.ipynb index 62b6668..1fbfb40 100644 --- a/notebooks/julia/01_cross_binding.ipynb +++ b/julia_Kriging_vs-GaussianProcesses.ipynb @@ -5,17 +5,16 @@ "id": "c000", "metadata": {}, "source": [ - "# Julia 01 — Cross-binding consistency, and a comparison with GaussianProcesses.jl\n", + "# Julia · Kriging vs GaussianProcesses.jl\n", "\n", - "1. Same protocol as `notebooks/python/11_cross_binding.ipynb` (see there), on\n", - " the shared datasets of `data/`; writes `results/xbind_julia.csv`, compared\n", - " with the Python values by `tools/report.py`.\n", - "2. Default fits against [GaussianProcesses.jl](https://github.com/STOR-i/GaussianProcesses.jl)\n", - " (Matérn 5/2 ARD, constant mean, MLE), when it is installed (warn-level).\n", + "Default fits on the shared datasets of `data/` (Branin, Hartmann 3, Borehole):\n", + "JLibKriging (`matern5_2`, constant trend, single start and 10 restarts) against\n", + "[GaussianProcesses.jl](https://github.com/STOR-i/GaussianProcesses.jl)\n", + "(Matérn 5/2 ARD, constant mean, MLE), warn-level comparison of Q².\n", "\n", "The module is `JLibKriging` (the packaged binding, [JLibKriging.jl](https://github.com/libKriging/JLibKriging.jl)),\n", - "imported under a short name so that its exported accessors (`X`, `y`, `theta`, ...)\n", - "do not clash with the variables of the notebook." + "imported under a short name (as GaussianProcesses.jl): both export `predict`,\n", + "`optimize!`, ... so every call is qualified." ] }, { @@ -26,73 +25,55 @@ "outputs": [], "source": [ "import JLibKriging as LK\n", - "include(\"checks.jl\")\n", - "s = Suite(\"jl01_cross_binding\")\n", - "rows = Vector{Vector{String}}()\n", - "record!(dataset, quantity, values) =\n", - " for (i, v) in enumerate(vec(Float64.(collect(values))))\n", - " push!(rows, [\"julia\", dataset, quantity, string(i), @sprintf(\"%.17g\", v)])\n", - " end" + "include(\"tools/julia/checks.jl\")\n", + "s = Suite(\"julia_Kriging_vs-GaussianProcesses\")" + ] + }, + { + "cell_type": "markdown", + "id": "c002", + "metadata": {}, + "source": [ + "## Installing GaussianProcesses.jl\n", + "\n", + "Added to the active Julia environment (with Optim.jl, its optimizer). Without\n", + "it, the comparison is skipped (warn-level)." ] }, { "cell_type": "code", "execution_count": null, - "id": "c002", + "id": "c003", "metadata": {}, "outputs": [], "source": [ - "for name in (\"branin\", \"hartmann3\", \"borehole\")\n", - " Xd = read_data(name, \"X\"); yd = vec(read_data(name, \"y\")); Xt = read_data(name, \"Xt\")[1:50, :]\n", - " d = size(Xd, 2)\n", - " theta0 = fill(0.3, d)\n", - " kf = LK.Kriging(yd, Xd, \"matern5_2\"; regmodel=\"constant\", optim=\"none\", theta=theta0, is_theta_estim=false)\n", - " record!(name, \"fixed_sigma2\", LK.sigma2(kf))\n", - " record!(name, \"fixed_beta\", LK.beta(kf))\n", - " record!(name, \"fixed_ll\", LK.log_likelihood(kf))\n", - " record!(name, \"fixed_loo\", LK.leave_one_out(kf))\n", - " p = LK.predict(kf, Xt; return_stdev=true)\n", - " record!(name, \"fixed_mean\", p.mean)\n", - " record!(name, \"fixed_stdev\", p.stdev)\n", - " llf = LK.log_likelihood_fun(kf, theta0; return_grad=true)\n", - " record!(name, \"llfun\", llf.ll)\n", - " record!(name, \"llfun_grad\", llf.grad)\n", - " check_close!(s, \"$name: log_likelihood() == log_likelihood_fun(theta0)\", LK.log_likelihood(kf), llf.ll; rtol=1e-10)\n", - " k = LK.Kriging(yd, Xd, \"matern5_2\")\n", - " record!(name, \"fit_theta\", LK.theta(k))\n", - " record!(name, \"fit_ll\", LK.log_likelihood(k))\n", - " record!(name, \"fit_mean\", LK.predict(k, Xt; return_stdev=false).mean)\n", - " record!(name, \"sim\", LK.simulate(kf, 3, 123, Xt[1:5, :]))\n", - "end\n", - "open(joinpath(RESULTS_DIR, \"xbind_julia.csv\"), \"w\") do io\n", - " println(io, \"lang,dataset,quantity,index,value\")\n", - " for r in rows; println(io, join(r, \",\")); end\n", - "end\n", - "println(length(rows), \" values written\")" + "import Pkg\n", + "have_gp = try\n", + " Pkg.add([\"GaussianProcesses\", \"Optim\"])\n", + " @eval import GaussianProcesses as GP\n", + " @eval import Optim\n", + " true\n", + "catch e\n", + " println(\"GaussianProcesses.jl could not be installed: \", sprint(showerror, e))\n", + " false\n", + "end" ] }, { "cell_type": "markdown", - "id": "c003", + "id": "c004", "metadata": {}, "source": [ - "## Default fits vs GaussianProcesses.jl" + "## Default fits" ] }, { "cell_type": "code", "execution_count": null, - "id": "c004", + "id": "c005", "metadata": {}, "outputs": [], "source": [ - "have_gp = try\n", - " @eval import GaussianProcesses as GP\n", - " @eval import Optim\n", - " true\n", - "catch\n", - " false\n", - "end\n", "for name in (\"branin\", \"hartmann3\", \"borehole\")\n", " Xd = read_data(name, \"X\"); yd = vec(read_data(name, \"y\")); Xt = read_data(name, \"Xt\"); yt = vec(read_data(name, \"yt\"))\n", " t = @elapsed k = LK.Kriging(yd, Xd, \"matern5_2\")\n", @@ -127,7 +108,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c005", + "id": "c006", "metadata": {}, "outputs": [], "source": [ diff --git a/julia_all-classes_api-tour.ipynb b/julia_all-classes_api-tour.ipynb new file mode 100644 index 0000000..84b51c4 --- /dev/null +++ b/julia_all-classes_api-tour.ipynb @@ -0,0 +1,313 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "c000", + "metadata": {}, + "source": [ + "# Julia · all classes — every libKriging model class and fit method through JLibKriging\n", + "\n", + "Same tour as `R_all-classes_api-tour.ipynb` (see there for the checks): Branin\n", + "on [0,1]², every class and fit method of the Julia binding, each with the\n", + "checks that hold for it whatever the data.\n", + "\n", + "The module is `JLibKriging` (the packaged binding, [JLibKriging.jl](https://github.com/libKriging/JLibKriging.jl)),\n", + "imported under a short name so that its exported accessors (`X`, `y`, `theta`, ...)\n", + "do not clash with the variables of the notebook." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c001", + "metadata": {}, + "outputs": [], + "source": [ + "import JLibKriging as LK\n", + "using Random\n", + "include(\"tools/julia/checks.jl\")\n", + "s = Suite(\"julia_all-classes_api-tour\")\n", + "\n", + "branin(U) = begin\n", + " x1 = 15 .* U[:, 1] .- 5; x2 = 15 .* U[:, 2]\n", + " (x2 .- 5.1 / (4π^2) .* x1 .^ 2 .+ 5 / π .* x1 .- 6) .^ 2 .+ 10 * (1 - 1 / (8π)) .* cos.(x1) .+ 10\n", + "end\n", + "rng = MersenneTwister(1)\n", + "lhs(n, d) = hcat([(randperm(rng, n) .- rand(rng, n)) ./ n for _ in 1:d]...)\n", + "X = lhs(30, 2); y = branin(X)\n", + "Xt = rand(rng, 500, 2); yt = branin(Xt)\n", + "sd(v) = sqrt(sum((v .- sum(v) / length(v)) .^ 2) / (length(v) - 1))\n", + "finite_count(v...) = count(!isfinite, vcat(v...))\n", + "\n", + "function check_runs!(f, s, check)\n", + " try\n", + " f()\n", + " catch e\n", + " add_check!(s, check, false; value=string(typeof(e)), expected=\"no error\", detail=first(sprint(showerror, e), 200))\n", + " end\n", + "end\n", + "\n", + "function roundtrip!(s, label, k, Xn)\n", + " check_runs!(s, \"$label: save + load\") do\n", + " f = joinpath(mktempdir(), \"k.json\")\n", + " LK.save(k, f)\n", + " k2 = LK.load(f)\n", + " check_close!(s, \"$label: load(save(k)) predicts the same mean\",\n", + " LK.predict(k2, Xn; return_stdev=false).mean, LK.predict(k, Xn; return_stdev=false).mean; rtol=1e-10)\n", + " end\n", + "end" + ] + }, + { + "cell_type": "markdown", + "id": "c002", + "metadata": {}, + "source": [ + "## `Kriging`, noise-free" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c003", + "metadata": {}, + "outputs": [], + "source": [ + "k = LK.Kriging(y, X, \"matern5_2\")\n", + "p = LK.predict(k, Xt; return_stdev=true)\n", + "add_metric!(s, \"branin\", \"Kriging\"; q2=q2(yt, p.mean))\n", + "add_check!(s, \"Kriging: Q2\", q2(yt, p.mean) > 0.95; value=q2(yt, p.mean), expected=\"> 0.95\")\n", + "e = maximum(abs.(LK.predict(k, X; return_stdev=false).mean .- y)) / sd(y)\n", + "add_check!(s, \"Kriging: interpolates (max |m(X) - y| / sd(y))\", e < 1e-6; value=e, expected=\"< 1e-6\")\n", + "S = LK.simulate(k, 5, 2, X)\n", + "e = maximum(abs.(S .- y)) / sqrt(LK.sigma2(k))\n", + "add_check!(s, \"Kriging: conditional simulations go through the data (max |sim(X) - y| / sigma)\", e < 1e-4;\n", + " value=e, expected=\"< 1e-4\")\n", + "add_check!(s, \"Kriging: stdev > 0 off the design\", minimum(p.stdev) > 0; value=minimum(p.stdev), expected=\"> 0\")\n", + "Xu = lhs(5, 2); yu = branin(Xu)\n", + "ku = copy(k)\n", + "LK.update!(ku, yu, Xu; refit=false)\n", + "k35 = LK.Kriging(vcat(y, yu), vcat(X, Xu), \"matern5_2\"; optim=\"none\",\n", + " theta=vec(LK.theta(k)), is_theta_estim=false, sigma2=LK.sigma2(k), is_sigma2_estim=false)\n", + "check_close!(s, \"Kriging: update!(refit=false) == fit on all points (same theta, sigma2)\",\n", + " LK.predict(ku, Xt; return_stdev=false).mean, LK.predict(k35, Xt; return_stdev=false).mean;\n", + " rtol=1e-6, atol=1e-8 * sd(y))\n", + "roundtrip!(s, \"Kriging\", k, Xt)" + ] + }, + { + "cell_type": "markdown", + "id": "c004", + "metadata": {}, + "source": [ + "## `Kriging` with noise: estimated nugget, known variances\n", + "\n", + "Noise of standard deviation 5% of sd(y). The nugget model is fitted with 10\n", + "restarts (`optim=\"BFGS10\"`); the default single start is compared with it\n", + "(warn-level: a lower likelihood means a local optimum). As DiceKriging's\n", + "nugget, the predictor returns y exactly *at* the design points (the nugget is\n", + "a micro-scale variation): smoothing is checked next to them." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c005", + "metadata": {}, + "outputs": [], + "source": [ + "sd_noise = 0.05 * sd(y)\n", + "yn = y .+ sd_noise .* randn(rng, length(y))\n", + "kn = LK.Kriging(yn, X, \"matern5_2\"; noise=\"nugget\", optim=\"BFGS10\")\n", + "mn = LK.predict(kn, Xt; return_stdev=false).mean\n", + "add_metric!(s, \"branin noisy\", \"Kriging nugget BFGS10\"; q2=q2(yt, mn))\n", + "add_check!(s, \"Kriging(nugget): estimated nugget > 0\", LK.nugget(kn) > 0; value=LK.nugget(kn), expected=\"> 0\")\n", + "add_check!(s, \"Kriging(nugget): Q2\", q2(yt, mn) > 0.95; value=q2(yt, mn), expected=\"> 0.95\")\n", + "r = sqrt(sum((LK.predict(kn, min.(X .+ 1e-7, 1.0); return_stdev=false).mean .- yn) .^ 2) / length(yn)) / sd_noise\n", + "add_check!(s, \"Kriging(nugget): smooths the noisy data next to the design points (rms residual / noise sd)\", r > 0.3;\n", + " value=r, expected=\"> 0.3\")\n", + "kn1 = LK.Kriging(yn, X, \"matern5_2\"; noise=\"nugget\")\n", + "add_metric!(s, \"branin noisy\", \"Kriging nugget BFGS\"; q2=q2(yt, LK.predict(kn1, Xt; return_stdev=false).mean))\n", + "dll = LK.log_likelihood(kn1) - LK.log_likelihood(kn)\n", + "add_check!(s, \"Kriging(nugget): LL(single start) - LL(10 restarts)\", dll > -1; value=dll, expected=\"> -1\", severity=\"warn\")\n", + "roundtrip!(s, \"Kriging(nugget)\", kn, Xt)\n", + "v = fill(sd_noise^2, length(yn))\n", + "kh = LK.Kriging(yn, X, \"matern5_2\"; noise=v)\n", + "check_close!(s, \"Kriging(noise = variances): noise() returns them\", LK.noise(kh), v; rtol=1e-12)\n", + "ph = LK.predict(kh, Xt; return_stdev=true)\n", + "add_check!(s, \"Kriging(noise = variances): finite predictions\", finite_count(ph.mean, ph.stdev) == 0;\n", + " value=finite_count(ph.mean, ph.stdev), expected=\"0 non-finite\")\n", + "add_metric!(s, \"branin noisy\", \"Kriging known noise\"; q2=q2(yt, ph.mean))" + ] + }, + { + "cell_type": "markdown", + "id": "c006", + "metadata": {}, + "source": [ + "## `Kriging` with an approximate fit: Vecchia, Nyström (n = 400)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c007", + "metadata": {}, + "outputs": [], + "source": [ + "Xl = lhs(400, 2); yl = branin(Xl)\n", + "for m in (\"vecchia(20)\", \"nystrom(60)\")\n", + " short = split(m, \"(\")[1]\n", + " label = \"Kriging(method = '$m')\"\n", + " check_runs!(s, \"$label: fit + predict\") do\n", + " ka = LK.Kriging(yl, Xl, \"matern5_2\"; method=m)\n", + " add_check!(s, \"$label: method()\", LK.method(ka) == m; value=LK.method(ka), expected=m)\n", + " pa = LK.predict(ka, Xt; return_stdev=true, method=String(short))\n", + " add_check!(s, \"$label: finite predictions\", finite_count(pa.mean, pa.stdev) == 0;\n", + " value=finite_count(pa.mean, pa.stdev), expected=\"0 non-finite\")\n", + " add_metric!(s, \"branin n=400\", \"Kriging $m\"; q2=q2(yt, pa.mean))\n", + " add_check!(s, \"$label: Q2\", q2(yt, pa.mean) > 0.95; value=q2(yt, pa.mean), expected=\"> 0.95\", severity=\"warn\")\n", + " end\n", + "end" + ] + }, + { + "cell_type": "markdown", + "id": "c008", + "metadata": {}, + "source": [ + "## `WarpKriging`: continuous warpings, then a categorical input\n", + "\n", + "The categorical case: x₂ restricted to 5 levels (coded 0, …, 4), embedded in ℝ²." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c009", + "metadata": {}, + "outputs": [], + "source": [ + "check_runs!(s, \"WarpKriging(kumaraswamy, kumaraswamy): fit + predict\") do\n", + " kw = LK.WarpKriging(y, X, [\"kumaraswamy\", \"kumaraswamy\"], \"matern5_2\")\n", + " pw = LK.predict(kw, Xt; return_stdev=true)\n", + " add_check!(s, \"WarpKriging(kumaraswamy): finite predictions\", finite_count(pw.mean, pw.stdev) == 0;\n", + " value=finite_count(pw.mean, pw.stdev), expected=\"0 non-finite\")\n", + " add_metric!(s, \"branin\", \"WarpKriging kumaraswamy\"; q2=q2(yt, pw.mean))\n", + " add_check!(s, \"WarpKriging(kumaraswamy): Q2\", q2(yt, pw.mean) > 0.9; value=q2(yt, pw.mean), expected=\"> 0.9\",\n", + " severity=\"warn\")\n", + " roundtrip!(s, \"WarpKriging(kumaraswamy)\", kw, Xt)\n", + "end\n", + "lev(U) = hcat(U[:, 1], round.(U[:, 2] .* 4))\n", + "unlev(L) = hcat(L[:, 1], L[:, 2] ./ 4)\n", + "Xc = lev(lhs(40, 2)); yc = branin(unlev(Xc))\n", + "Xtc = lev(Xt); ytc = branin(unlev(Xtc))\n", + "check_runs!(s, \"WarpKriging(none, categorical(5,2)): fit + predict\") do\n", + " kc = LK.WarpKriging(yc, Xc, [\"none\", \"categorical(5,2)\"], \"matern5_2\")\n", + " pc = LK.predict(kc, Xtc; return_stdev=true)\n", + " add_check!(s, \"WarpKriging(categorical): finite predictions\", finite_count(pc.mean, pc.stdev) == 0;\n", + " value=finite_count(pc.mean, pc.stdev), expected=\"0 non-finite\")\n", + " add_metric!(s, \"branin, x2 categorical\", \"WarpKriging categorical\"; q2=q2(ytc, pc.mean))\n", + " add_check!(s, \"WarpKriging(categorical): Q2\", q2(ytc, pc.mean) > 0.8; value=q2(ytc, pc.mean), expected=\"> 0.8\",\n", + " severity=\"warn\")\n", + " roundtrip!(s, \"WarpKriging(categorical)\", kc, Xtc)\n", + "end" + ] + }, + { + "cell_type": "markdown", + "id": "c010", + "metadata": {}, + "source": [ + "## `MLPKriging`: a joint feature map ℝ² → ℝ² before the kernel" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c011", + "metadata": {}, + "outputs": [], + "source": [ + "check_runs!(s, \"MLPKriging: fit + predict\") do\n", + " km = LK.MLPKriging(y, X, [8], 2; activation=\"tanh\", kernel=\"matern5_2\", optim=\"BFGS10\")\n", + " pm = LK.predict(km, Xt; return_stdev=true)\n", + " add_check!(s, \"MLPKriging: finite predictions\", finite_count(pm.mean, pm.stdev) == 0;\n", + " value=finite_count(pm.mean, pm.stdev), expected=\"0 non-finite\")\n", + " add_metric!(s, \"branin\", \"MLPKriging tanh [8] -> 2, BFGS10\"; q2=q2(yt, pm.mean))\n", + " add_check!(s, \"MLPKriging: Q2\", q2(yt, pm.mean) > 0.9; value=q2(yt, pm.mean), expected=\"> 0.9\", severity=\"warn\")\n", + " roundtrip!(s, \"MLPKriging\", km, Xt)\n", + " kd = LK.MLPKriging(y, X, [8], 2; activation=\"tanh\", kernel=\"matern5_2\") # default optimizer\n", + " qd = q2(yt, LK.predict(kd, Xt; return_stdev=false).mean)\n", + " add_metric!(s, \"branin\", \"MLPKriging tanh [8] -> 2, default optim\"; q2=qd)\n", + " add_check!(s, \"MLPKriging: Q2(default optim) - Q2(BFGS10)\", qd - q2(yt, pm.mean) > -0.05;\n", + " value=qd - q2(yt, pm.mean), expected=\"> -0.05\", severity=\"warn\")\n", + "end" + ] + }, + { + "cell_type": "markdown", + "id": "c012", + "metadata": {}, + "source": [ + "## `NestedKriging`: 4 groups of 100 points, `NK` and `PoE` aggregations" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c013", + "metadata": {}, + "outputs": [], + "source": [ + "for agg in (\"NK\", \"PoE\")\n", + " label = \"NestedKriging($agg)\"\n", + " check_runs!(s, \"$label: fit + predict\") do\n", + " kk = LK.NestedKriging(yl, Xl, \"matern5_2\", 4; aggregation=agg, seed=3)\n", + " pk = LK.predict(kk, Xt; return_stdev=true)\n", + " add_check!(s, \"$label: stdev >= 0\", all(pk.stdev .>= 0); value=minimum(pk.stdev), expected=\">= 0\")\n", + " add_metric!(s, \"branin n=400\", \"NestedKriging $agg\"; q2=q2(yt, pk.mean))\n", + " add_check!(s, \"$label: Q2\", q2(yt, pk.mean) > 0.95; value=q2(yt, pk.mean), expected=\"> 0.95\", severity=\"warn\")\n", + " if agg == \"NK\"\n", + " e = maximum(abs.(LK.predict(kk, Xl; return_stdev=false).mean .- yl)) / sd(yl)\n", + " add_check!(s, \"NestedKriging(NK): interpolates (max |m(X) - y| / sd(y))\", e < 1e-3; value=e, expected=\"< 1e-3\")\n", + " end\n", + " roundtrip!(s, label, kk, Xt)\n", + " end\n", + "end" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c014", + "metadata": {}, + "outputs": [], + "source": [ + "foreach(r -> r[5] == \"q2\" && println(rpad(r[3], 24), rpad(r[4], 28), r[6]), s.metrics)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c015", + "metadata": {}, + "outputs": [], + "source": [ + "finalize!(s)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Julia 1.11", + "language": "julia", + "name": "julia-1.11" + }, + "language_info": { + "name": "julia" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/known_issues.csv b/known_issues.csv deleted file mode 100644 index 3fcf230..0000000 --- a/known_issues.csv +++ /dev/null @@ -1,11 +0,0 @@ -suite,check,issue -py07_scalability,vecchia(n-1) LL == exact LL (same theta),"With m >= n-1 neighbours the Vecchia likelihood should be the exact one; it matches for n=20 but not for n=40 (-111.349 vs -110.834)" -py07_scalability,hartmann3/n=1500/nystrom(100): Q2,"Nystrom predictor degrades as n grows at fixed k (theta estimates are fine: the exact predictor at the Nystrom theta has Q2 ~ 1)" -py07_scalability,hartmann3/n=5000/nystrom(100): Q2,"Nystrom predictor degrades as n grows at fixed k (theta estimates are fine: the exact predictor at the Nystrom theta has Q2 ~ 1)" -py07_scalability,otl_circuit/n=1500/nystrom(100): Q2,"Nystrom predictor degrades as n grows at fixed k (theta estimates are fine: the exact predictor at the Nystrom theta has Q2 ~ 1)" -py07_scalability,otl_circuit/n=5000/nystrom(100): Q2,"Nystrom predictor degrades as n grows at fixed k (theta estimates are fine: the exact predictor at the Nystrom theta has Q2 ~ 1)" -py09_robustness,NaN in y is rejected,"Kriging accepts NaN in y without raising" -py09_robustness,Inf in y is rejected,"Kriging accepts Inf in y without raising" -py09_robustness,WarpKriging with 1 spec for 2 columns is rejected,"WarpKriging accepts fewer warping specs than columns of X" -py09_robustness,predict(stdev) time growth exponent in q (8000 vs 4000),"predict(return_stdev=True, return_cov=False) is O(q^2) in time and memory (10 000 points: 1.6 GB; 30 000: out of memory)" -py10_compare_python_packages,sklearn RBF == libKriging gauss: simple kriging stdev (beta fixed),"With beta fixed (is_beta_estim=False) predict still adds the trend-estimation variance (universal kriging) instead of the simple kriging variance" diff --git a/notebooks/julia/src/01_cross_binding.jl b/notebooks/julia/src/01_cross_binding.jl deleted file mode 100644 index 17740e4..0000000 --- a/notebooks/julia/src/01_cross_binding.jl +++ /dev/null @@ -1,95 +0,0 @@ -# %% [markdown] -# # Julia 01 — Cross-binding consistency, and a comparison with GaussianProcesses.jl -# -# 1. Same protocol as `notebooks/python/11_cross_binding.ipynb` (see there), on -# the shared datasets of `data/`; writes `results/xbind_julia.csv`, compared -# with the Python values by `tools/report.py`. -# 2. Default fits against [GaussianProcesses.jl](https://github.com/STOR-i/GaussianProcesses.jl) -# (Matérn 5/2 ARD, constant mean, MLE), when it is installed (warn-level). -# -# The module is `JLibKriging` (the packaged binding, [JLibKriging.jl](https://github.com/libKriging/JLibKriging.jl)), -# imported under a short name so that its exported accessors (`X`, `y`, `theta`, ...) -# do not clash with the variables of the notebook. - -# %% -import JLibKriging as LK -include("checks.jl") -s = Suite("jl01_cross_binding") -rows = Vector{Vector{String}}() -record!(dataset, quantity, values) = - for (i, v) in enumerate(vec(Float64.(collect(values)))) - push!(rows, ["julia", dataset, quantity, string(i), @sprintf("%.17g", v)]) - end - -# %% -for name in ("branin", "hartmann3", "borehole") - Xd = read_data(name, "X"); yd = vec(read_data(name, "y")); Xt = read_data(name, "Xt")[1:50, :] - d = size(Xd, 2) - theta0 = fill(0.3, d) - kf = LK.Kriging(yd, Xd, "matern5_2"; regmodel="constant", optim="none", theta=theta0, is_theta_estim=false) - record!(name, "fixed_sigma2", LK.sigma2(kf)) - record!(name, "fixed_beta", LK.beta(kf)) - record!(name, "fixed_ll", LK.log_likelihood(kf)) - record!(name, "fixed_loo", LK.leave_one_out(kf)) - p = LK.predict(kf, Xt; return_stdev=true) - record!(name, "fixed_mean", p.mean) - record!(name, "fixed_stdev", p.stdev) - llf = LK.log_likelihood_fun(kf, theta0; return_grad=true) - record!(name, "llfun", llf.ll) - record!(name, "llfun_grad", llf.grad) - check_close!(s, "$name: log_likelihood() == log_likelihood_fun(theta0)", LK.log_likelihood(kf), llf.ll; rtol=1e-10) - k = LK.Kriging(yd, Xd, "matern5_2") - record!(name, "fit_theta", LK.theta(k)) - record!(name, "fit_ll", LK.log_likelihood(k)) - record!(name, "fit_mean", LK.predict(k, Xt; return_stdev=false).mean) - record!(name, "sim", LK.simulate(kf, 3, 123, Xt[1:5, :])) -end -open(joinpath(RESULTS_DIR, "xbind_julia.csv"), "w") do io - println(io, "lang,dataset,quantity,index,value") - for r in rows; println(io, join(r, ",")); end -end -println(length(rows), " values written") - -# %% [markdown] -# ## Default fits vs GaussianProcesses.jl - -# %% -have_gp = try - @eval import GaussianProcesses as GP - @eval import Optim - true -catch - false -end -for name in ("branin", "hartmann3", "borehole") - Xd = read_data(name, "X"); yd = vec(read_data(name, "y")); Xt = read_data(name, "Xt"); yt = vec(read_data(name, "yt")) - t = @elapsed k = LK.Kriging(yd, Xd, "matern5_2") - q_lk = q2(yt, LK.predict(k, Xt; return_stdev=false).mean) - add_metric!(s, name, "JLibKriging"; q2=q_lk, fit_s=t) - t = @elapsed k10 = LK.Kriging(yd, Xd, "matern5_2"; optim="BFGS10") - q_lk10 = q2(yt, LK.predict(k10, Xt; return_stdev=false).mean) - add_metric!(s, name, "JLibKriging BFGS10"; q2=q_lk10, fit_s=t) - add_check!(s, "$name: Q2(JLibKriging BFGS10) > 0.9", q_lk10 > 0.9; value=q_lk10, expected="> 0.9") - if have_gp - try - d = size(Xd, 2) - t = @elapsed begin - gp = GP.GP(permutedims(Xd), yd, GP.MeanConst(sum(yd) / length(yd)), - GP.Mat52Ard(zeros(d), log(sum((yd .- sum(yd) / length(yd)) .^ 2) / length(yd)) / 2), -8.0) - GP.optimize!(gp; noise=false) - end - mg, _ = GP.predict_f(gp, permutedims(Xt)) - q_gp = q2(yt, mg) - add_metric!(s, name, "GaussianProcesses.jl"; q2=q_gp, fit_s=t) - add_check!(s, "$name: Q2(JLibKriging) - Q2(GaussianProcesses.jl)", q_lk - q_gp > -0.02; - value=q_lk - q_gp, expected="> -0.02", severity="warn") - catch e - add_check!(s, "$name: GaussianProcesses.jl fit", false; value=string(typeof(e)), severity="warn") - end - end -end -have_gp || add_check!(s, "GaussianProcesses.jl available", false; value="absent", severity="warn") -foreach(r -> println(join(r[3:6], " ")), s.metrics) - -# %% -finalize!(s) diff --git a/notebooks/octave/lkb_suite.m b/notebooks/octave/lkb_suite.m deleted file mode 100644 index a26ae2f..0000000 --- a/notebooks/octave/lkb_suite.m +++ /dev/null @@ -1,20 +0,0 @@ -function s = lkb_suite(name) - % Same CSV outputs as lkbench/checks.py (see there), for the Octave notebooks. - % A suite is a struct passed around by value: s = lkb_close(s, ...). - s.name = name; - s.checks = {}; - s.metrics = {}; - s.t0 = tic(); - s.known = {}; - f = fullfile("..", "..", "known_issues.csv"); - if exist(f, "file") - fid = fopen(f, "r"); fgetl(fid); - while true - l = fgetl(fid); - if ~ischar(l), break; end - p = strsplit(l, ","); - if numel(p) >= 3, s.known(end + 1, :) = {p{1}, p{2}, strjoin(p(3:end), ",")}; end - end - fclose(fid); - end -end diff --git a/notebooks/octave/read_data.m b/notebooks/octave/read_data.m deleted file mode 100644 index 1796dc7..0000000 --- a/notebooks/octave/read_data.m +++ /dev/null @@ -1,3 +0,0 @@ -function A = read_data(name, stem) - A = dlmread(fullfile("..", "..", "data", [name "_" stem ".csv"]), ","); -end diff --git a/notebooks/octave/01_cross_binding.ipynb b/octave_Kriging_cross-binding.ipynb similarity index 89% rename from notebooks/octave/01_cross_binding.ipynb rename to octave_Kriging_cross-binding.ipynb index ea5e878..6369816 100644 --- a/notebooks/octave/01_cross_binding.ipynb +++ b/octave_Kriging_cross-binding.ipynb @@ -5,9 +5,9 @@ "id": "c000", "metadata": {}, "source": [ - "# Octave 01 — Cross-binding consistency: the Octave side\n", + "# Octave · Kriging — cross-binding consistency: the Octave side\n", "\n", - "Same protocol as `notebooks/python/11_cross_binding.ipynb` (see there), on the\n", + "Same protocol as `python_Kriging_cross-binding-reference.ipynb` (see there), on the\n", "shared datasets of `data/`; writes `results/xbind_octave.csv`, compared with\n", "the Python values by `tools/report.py`. mLibKriging must be on the path\n", "(`OCTAVE_PATH` or `addpath`)." @@ -20,7 +20,8 @@ "metadata": {}, "outputs": [], "source": [ - "s = lkb_suite(\"oct01_cross_binding\");\n", + "addpath(fullfile(pwd, \"tools\", \"octave\"));\n", + "s = lkb_suite(\"octave_Kriging_cross-binding\");\n", "out = {};\n", "function out = put(out, dataset, quantity, values)\n", " v = double(values(:));\n", @@ -65,7 +66,7 @@ " S = kf.simulate(int32(3), int32(123), Xt(1:5, :), false);\n", " out = put(out, name, \"sim\", S);\n", "end\n", - "d = getenv(\"LKBENCH_RESULTS\"); if isempty(d), d = fullfile(\"..\", \"..\", \"results\"); end\n", + "d = getenv(\"LKBENCH_RESULTS\"); if isempty(d), d = \"results\"; end\n", "if ~exist(d, \"dir\"), mkdir(d); end\n", "fid = fopen(fullfile(d, \"xbind_octave.csv\"), \"w\");\n", "fprintf(fid, \"lang,dataset,quantity,index,value\\n\");\n", diff --git a/notebooks/octave/02_vs_stk.ipynb b/octave_Kriging_vs-STK.ipynb similarity index 76% rename from notebooks/octave/02_vs_stk.ipynb rename to octave_Kriging_vs-STK.ipynb index ee92492..198080d 100644 --- a/notebooks/octave/02_vs_stk.ipynb +++ b/octave_Kriging_vs-STK.ipynb @@ -5,7 +5,7 @@ "id": "c000", "metadata": {}, "source": [ - "# Octave 02 — mLibKriging vs STK\n", + "# Octave · Kriging vs STK\n", "\n", "[STK](https://github.com/stk-kriging/stk) (Small Toolbox for Kriging) is the\n", "reference kriging toolbox for Octave/MATLAB.\n", @@ -20,7 +20,8 @@ "2. **Default fits** on the shared datasets of `data/`: Matérn 5/2, each\n", " toolbox's own estimation (warn-level comparison of Q²).\n", "\n", - "STK's location: the `STK_PATH` environment variable (skipped if unset)." + "mLibKriging must be on the path (`OCTAVE_PATH` or `addpath`), see\n", + "`tools/install/octave.sh`." ] }, { @@ -30,14 +31,43 @@ "metadata": {}, "outputs": [], "source": [ - "s = lkb_suite(\"oct02_vs_stk\");\n", + "addpath(fullfile(pwd, \"tools\", \"octave\"));\n", + "s = lkb_suite(\"octave_Kriging_vs-STK\");" + ] + }, + { + "cell_type": "markdown", + "id": "c002", + "metadata": {}, + "source": [ + "## Installing STK\n", + "\n", + "STK is fetched here (shallow clone of its repository into a temporary\n", + "directory), unless the `STK_PATH` environment variable already points to a copy.\n", + "Without STK, the comparison is skipped (warn-level)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c003", + "metadata": {}, + "outputs": [], + "source": [ "stk_path = getenv(\"STK_PATH\");\n", - "have_stk = ~isempty(stk_path) && exist(fullfile(stk_path, \"stk_init.m\"), \"file\");\n", + "if isempty(stk_path) || ~exist(fullfile(stk_path, \"stk_init.m\"), \"file\")\n", + " stk_path = fullfile(tempdir(), \"stk\");\n", + " if ~exist(fullfile(stk_path, \"stk_init.m\"), \"file\")\n", + " [status, out] = system(sprintf(\"git clone --depth 1 https://github.com/stk-kriging/stk.git %s 2>&1\", stk_path));\n", + " printf(\"git clone stk: status %d\\n%s\", status, out);\n", + " end\n", + "end\n", + "have_stk = exist(fullfile(stk_path, \"stk_init.m\"), \"file\") == 2;\n", "if have_stk\n", " run(fullfile(stk_path, \"stk_init.m\"));\n", - " printf(\"STK %s\\n\", stk_version());\n", + " printf(\"STK %s from %s\\n\", stk_version(), stk_path);\n", "else\n", - " s = lkb_check(s, \"STK available\", false, \"absent\", \"STK_PATH\", \"STK not found: comparison skipped\", \"warn\");\n", + " s = lkb_check(s, \"STK available\", false, \"absent\", stk_path, \"STK could not be installed: comparison skipped\", \"warn\");\n", "end\n", "q2 = @(y, m) 1 - sum((y - m).^2) / sum((y - mean(y)).^2);\n", "datasets = {\"branin\", \"hartmann3\", \"borehole\"};" @@ -45,7 +75,7 @@ }, { "cell_type": "markdown", - "id": "c002", + "id": "c004", "metadata": {}, "source": [ "## 1. Same hyperparameters, same numbers" @@ -54,7 +84,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c003", + "id": "c005", "metadata": {}, "outputs": [], "source": [ @@ -79,7 +109,7 @@ }, { "cell_type": "markdown", - "id": "c004", + "id": "c006", "metadata": {}, "source": [ "## 2. Default fits" @@ -88,7 +118,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c005", + "id": "c007", "metadata": {}, "outputs": [], "source": [ @@ -122,7 +152,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c006", + "id": "c008", "metadata": {}, "outputs": [], "source": [ diff --git a/octave_all-classes_api-tour.ipynb b/octave_all-classes_api-tour.ipynb new file mode 100644 index 0000000..b0e0ccb --- /dev/null +++ b/octave_all-classes_api-tour.ipynb @@ -0,0 +1,328 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "c000", + "metadata": {}, + "source": [ + "# Octave · all classes — every libKriging model class and fit method through mLibKriging\n", + "\n", + "Same tour as `R_all-classes_api-tour.ipynb` (see there for the checks): Branin\n", + "on [0,1]², every class and fit method of the Octave binding (positional\n", + "arguments), each with the checks that hold for it whatever the data. A class\n", + "missing from the installed mLibKriging is recorded as SKIP. mLibKriging must\n", + "be on the path (`OCTAVE_PATH` or `addpath`), see `tools/install/octave.sh`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c001", + "metadata": {}, + "outputs": [], + "source": [ + "addpath(fullfile(pwd, \"tools\", \"octave\"));\n", + "s = lkb_suite(\"octave_all-classes_api-tour\");\n", + "q2 = @(y, m) 1 - sum((y - m).^2) / sum((y - mean(y)).^2);\n", + "branin = @(U) ((15 * U(:, 2)) - 5.1 / (4 * pi^2) * (15 * U(:, 1) - 5).^2 + 5 / pi * (15 * U(:, 1) - 5) - 6).^2 ...\n", + " + 10 * (1 - 1 / (8 * pi)) * cos(15 * U(:, 1) - 5) + 10;\n", + "rand(\"seed\", 1); randn(\"seed\", 1);\n", + "function U = lhs(n, d)\n", + " U = zeros(n, d);\n", + " for j = 1:d, U(:, j) = (randperm(n)' - rand(n, 1)) / n; end\n", + "end\n", + "X = lhs(30, 2); y = branin(X);\n", + "Xt = rand(500, 2); yt = branin(Xt);\n", + "tmp = tempname(); mkdir(tmp);\n", + "has_class = @(name) exist(name) == 2; % a class file on the path\n", + "\n", + "function s = roundtrip(s, label, k, Xn, loader)\n", + " f = fullfile(tempdir(), [regexprep(label, \"[^A-Za-z0-9]\", \"_\") \".json\"]);\n", + " try\n", + " k.save(f);\n", + " k2 = loader(f);\n", + " m1 = k.predict(Xn, false, false, false);\n", + " m2 = k2.predict(Xn, false, false, false);\n", + " s = lkb_close(s, [label \": load(save(k)) predicts the same mean\"], m2, m1, 1e-10);\n", + " catch err\n", + " s = lkb_check(s, [label \": save + load\"], false, \"error\", \"no error\", err.message);\n", + " end\n", + "end\n", + "\n", + "function s = skip(s, check, reason)\n", + " s.checks(end + 1, :) = {s.name, \"octave\", check, \"info\", \"SKIP\", \"\", \"\", reason};\n", + " printf(\"[SKIP] %s: %s\\n\", check, reason);\n", + "end" + ] + }, + { + "cell_type": "markdown", + "id": "c002", + "metadata": {}, + "source": [ + "## `Kriging`, noise-free" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c003", + "metadata": {}, + "outputs": [], + "source": [ + "k = Kriging(y, X, \"matern5_2\");\n", + "[m, sd] = k.predict(Xt, true, false, false);\n", + "s = lkb_metric(s, \"branin\", \"Kriging\", \"q2\", q2(yt, m));\n", + "s = lkb_check(s, \"Kriging: Q2\", q2(yt, m) > 0.95, q2(yt, m), \"> 0.95\");\n", + "mX = k.predict(X, false, false, false);\n", + "e = max(abs(mX - y)) / std(y);\n", + "s = lkb_check(s, \"Kriging: interpolates (max |m(X) - y| / sd(y))\", e < 1e-6, e, \"< 1e-6\");\n", + "S = k.simulate(int32(5), int32(2), X, false);\n", + "e = max(max(abs(S - y))) / sqrt(k.sigma2());\n", + "s = lkb_check(s, \"Kriging: conditional simulations go through the data (max |sim(X) - y| / sigma)\", e < 1e-4, e, \"< 1e-4\");\n", + "s = lkb_check(s, \"Kriging: stdev > 0 off the design\", min(sd) > 0, min(sd), \"> 0\");\n", + "Xu = lhs(5, 2); yu = branin(Xu);\n", + "ku = k.copy();\n", + "ku.update(yu, Xu, false);\n", + "th = k.theta();\n", + "k35 = Kriging([y; yu], [X; Xu], \"matern5_2\", \"constant\", false, \"none\", \"LL\", ...\n", + " Params(\"theta\", th(:)', \"is_theta_estim\", false, \"sigma2\", k.sigma2(), \"is_sigma2_estim\", false));\n", + "s = lkb_close(s, \"Kriging: update(refit = false) == fit on all points (same theta, sigma2)\", ...\n", + " ku.predict(Xt, false, false, false), k35.predict(Xt, false, false, false), 1e-6, 1e-8 * std(y));\n", + "s = roundtrip(s, \"Kriging\", k, Xt, @load_kriging);" + ] + }, + { + "cell_type": "markdown", + "id": "c004", + "metadata": {}, + "source": [ + "## `Kriging` with noise: estimated nugget, known variances\n", + "\n", + "Noise of standard deviation 5% of sd(y). The nugget model is fitted with 10\n", + "restarts (`\"BFGS10\"`); the default single start is compared with it\n", + "(warn-level: a lower likelihood means a local optimum). As DiceKriging's\n", + "nugget, the predictor returns y exactly *at* the design points (the nugget is\n", + "a micro-scale variation): smoothing is checked next to them." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c005", + "metadata": {}, + "outputs": [], + "source": [ + "sd_noise = 0.05 * std(y);\n", + "yn = y + sd_noise * randn(size(y));\n", + "kn = Kriging(yn, X, \"matern5_2\", \"constant\", false, \"BFGS10\", \"LL\", [], \"nugget\");\n", + "mn = kn.predict(Xt, false, false, false);\n", + "nug = kn.nugget(); ll10 = kn.logLikelihood();\n", + "s = lkb_metric(s, \"branin noisy\", \"Kriging nugget BFGS10\", \"q2\", q2(yt, mn));\n", + "s = lkb_check(s, \"Kriging(nugget): estimated nugget > 0\", nug > 0, nug, \"> 0\");\n", + "s = lkb_check(s, \"Kriging(nugget): Q2\", q2(yt, mn) > 0.95, q2(yt, mn), \"> 0.95\");\n", + "r = sqrt(mean((kn.predict(min(X + 1e-7, 1), false, false, false) - yn).^2)) / sd_noise;\n", + "s = lkb_check(s, \"Kriging(nugget): smooths the noisy data next to the design points (rms residual / noise sd)\", r > 0.3, r, \"> 0.3\");\n", + "kn1 = Kriging(yn, X, \"matern5_2\", \"constant\", false, \"BFGS\", \"LL\", [], \"nugget\");\n", + "ll1 = kn1.logLikelihood();\n", + "s = lkb_metric(s, \"branin noisy\", \"Kriging nugget BFGS\", \"q2\", q2(yt, kn1.predict(Xt, false, false, false)));\n", + "s = lkb_check(s, \"Kriging(nugget): LL(single start) - LL(10 restarts)\", ll1 - ll10 > -1, ll1 - ll10, \"> -1\", \"\", \"warn\");\n", + "s = roundtrip(s, \"Kriging(nugget)\", kn, Xt, @load_kriging);\n", + "v = sd_noise^2 * ones(size(yn));\n", + "kh = Kriging(yn, X, \"matern5_2\", \"constant\", false, \"BFGS\", \"LL\", [], \"heterogeneous\", v);\n", + "s = lkb_close(s, \"Kriging(noise = variances): noise() returns them\", kh.noise(), v, 1e-12);\n", + "[mh, sh] = kh.predict(Xt, true, false, false);\n", + "s = lkb_check(s, \"Kriging(noise = variances): finite predictions\", all(isfinite([mh; sh])), sum(~isfinite([mh; sh])), \"0 non-finite\");\n", + "s = lkb_metric(s, \"branin noisy\", \"Kriging known noise\", \"q2\", q2(yt, mh));" + ] + }, + { + "cell_type": "markdown", + "id": "c006", + "metadata": {}, + "source": [ + "## `Kriging` with an approximate fit: Vecchia, Nyström (n = 400)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c007", + "metadata": {}, + "outputs": [], + "source": [ + "Xl = lhs(400, 2); yl = branin(Xl);\n", + "for mth = {\"vecchia(20)\", \"nystrom(60)\"}\n", + " mth = mth{1}; short = regexprep(mth, \"\\\\(.*\", \"\");\n", + " label = [\"Kriging(method = '\" mth \"')\"];\n", + " try\n", + " ka = Kriging(yl, Xl, \"matern5_2\", \"constant\", false, \"BFGS\", \"LL\", [], \"none\", [], mth);\n", + " got = ka.method();\n", + " s = lkb_check(s, [label \": method()\"], strcmp(got, mth), got, mth);\n", + " [ma, sa] = ka.predict(Xt, true, false, false, short);\n", + " s = lkb_check(s, [label \": finite predictions\"], all(isfinite([ma; sa])), sum(~isfinite([ma; sa])), \"0 non-finite\");\n", + " s = lkb_metric(s, \"branin n=400\", [\"Kriging \" mth], \"q2\", q2(yt, ma));\n", + " s = lkb_check(s, [label \": Q2\"], q2(yt, ma) > 0.95, q2(yt, ma), \"> 0.95\", \"\", \"warn\");\n", + " catch err\n", + " s = lkb_check(s, [label \": fit + predict\"], false, \"error\", \"no error\", err.message);\n", + " end\n", + "end" + ] + }, + { + "cell_type": "markdown", + "id": "c008", + "metadata": {}, + "source": [ + "## `WarpKriging`: continuous warpings, then a categorical input\n", + "\n", + "The categorical case: x₂ restricted to 5 levels (coded 0, …, 4), embedded in ℝ²." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c009", + "metadata": {}, + "outputs": [], + "source": [ + "if has_class(\"WarpKriging\")\n", + " try\n", + " kw = WarpKriging(y, X, {\"kumaraswamy\", \"kumaraswamy\"}, \"matern5_2\");\n", + " [mw, sw] = kw.predict(Xt, true, false, false);\n", + " s = lkb_check(s, \"WarpKriging(kumaraswamy): finite predictions\", all(isfinite([mw; sw])), sum(~isfinite([mw; sw])), \"0 non-finite\");\n", + " s = lkb_metric(s, \"branin\", \"WarpKriging kumaraswamy\", \"q2\", q2(yt, mw));\n", + " s = lkb_check(s, \"WarpKriging(kumaraswamy): Q2\", q2(yt, mw) > 0.9, q2(yt, mw), \"> 0.9\", \"\", \"warn\");\n", + " s = roundtrip(s, \"WarpKriging(kumaraswamy)\", kw, Xt, @load_kriging);\n", + " catch err\n", + " s = lkb_check(s, \"WarpKriging(kumaraswamy): fit + predict\", false, \"error\", \"no error\", err.message);\n", + " end\n", + " lev = @(U) [U(:, 1), round(U(:, 2) * 4)];\n", + " unlev = @(L) [L(:, 1), L(:, 2) / 4];\n", + " Xc = lev(lhs(40, 2)); yc = branin(unlev(Xc));\n", + " Xtc = lev(Xt); ytc = branin(unlev(Xtc));\n", + " try\n", + " kc = WarpKriging(yc, Xc, {\"none\", \"categorical(5,2)\"}, \"matern5_2\");\n", + " [mc, sc] = kc.predict(Xtc, true, false, false);\n", + " s = lkb_check(s, \"WarpKriging(categorical): finite predictions\", all(isfinite([mc; sc])), sum(~isfinite([mc; sc])), \"0 non-finite\");\n", + " s = lkb_metric(s, \"branin, x2 categorical\", \"WarpKriging categorical\", \"q2\", q2(ytc, mc));\n", + " s = lkb_check(s, \"WarpKriging(categorical): Q2\", q2(ytc, mc) > 0.8, q2(ytc, mc), \"> 0.8\", \"\", \"warn\");\n", + " s = roundtrip(s, \"WarpKriging(categorical)\", kc, Xtc, @load_kriging);\n", + " catch err\n", + " s = lkb_check(s, \"WarpKriging(categorical): fit + predict\", false, \"error\", \"no error\", err.message);\n", + " end\n", + "else\n", + " s = skip(s, \"WarpKriging\", \"not in this mLibKriging\");\n", + "end" + ] + }, + { + "cell_type": "markdown", + "id": "c010", + "metadata": {}, + "source": [ + "## `MLPKriging`: a joint feature map ℝ² → ℝ² before the kernel" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c011", + "metadata": {}, + "outputs": [], + "source": [ + "if has_class(\"MLPKriging\")\n", + " try\n", + " km = MLPKriging(y, X, [8], 2, \"tanh\", \"matern5_2\", \"constant\", false, \"BFGS10\");\n", + " [mm, sm] = km.predict(Xt, true, false, false);\n", + " s = lkb_check(s, \"MLPKriging: finite predictions\", all(isfinite([mm; sm])), sum(~isfinite([mm; sm])), \"0 non-finite\");\n", + " s = lkb_metric(s, \"branin\", \"MLPKriging tanh [8] -> 2, BFGS10\", \"q2\", q2(yt, mm));\n", + " s = lkb_check(s, \"MLPKriging: Q2\", q2(yt, mm) > 0.9, q2(yt, mm), \"> 0.9\", \"\", \"warn\");\n", + " s = roundtrip(s, \"MLPKriging\", km, Xt, @load_kriging);\n", + " kd = MLPKriging(y, X, [8], 2, \"tanh\", \"matern5_2\"); % default optimizer\n", + " qd = q2(yt, kd.predict(Xt, false, false, false));\n", + " s = lkb_metric(s, \"branin\", \"MLPKriging tanh [8] -> 2, default optim\", \"q2\", qd);\n", + " s = lkb_check(s, \"MLPKriging: Q2(default optim) - Q2(BFGS10)\", qd - q2(yt, mm) > -0.05, qd - q2(yt, mm), \"> -0.05\", \"\", \"warn\");\n", + " catch err\n", + " s = lkb_check(s, \"MLPKriging: fit + predict\", false, \"error\", \"no error\", err.message);\n", + " end\n", + "else\n", + " s = skip(s, \"MLPKriging\", \"not in this mLibKriging\");\n", + "end" + ] + }, + { + "cell_type": "markdown", + "id": "c012", + "metadata": {}, + "source": [ + "## `NestedKriging`: 4 groups of 100 points, `NK` and `PoE` aggregations" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c013", + "metadata": {}, + "outputs": [], + "source": [ + "if has_class(\"NestedKriging\")\n", + " for agg = {\"NK\", \"PoE\"}\n", + " agg = agg{1}; label = [\"NestedKriging(\" agg \")\"];\n", + " try\n", + " kk = NestedKriging(yl, Xl, \"matern5_2\", 4, agg, \"kmeans\", 3);\n", + " [mk, sk] = kk.predict(Xt); % stdev returned when asked for (2 outputs)\n", + " s = lkb_check(s, [label \": stdev >= 0\"], all(sk >= 0), min(sk), \">= 0\");\n", + " s = lkb_metric(s, \"branin n=400\", [\"NestedKriging \" agg], \"q2\", q2(yt, mk));\n", + " s = lkb_check(s, [label \": Q2\"], q2(yt, mk) > 0.95, q2(yt, mk), \"> 0.95\", \"\", \"warn\");\n", + " if strcmp(agg, \"NK\")\n", + " e = max(abs(kk.predict(Xl) - yl)) / std(yl);\n", + " s = lkb_check(s, \"NestedKriging(NK): interpolates (max |m(X) - y| / sd(y))\", e < 1e-3, e, \"< 1e-3\");\n", + " end\n", + " f = fullfile(tmp, [\"nested_\" agg \".json\"]);\n", + " kk.save(f); k2 = load_kriging(f);\n", + " s = lkb_close(s, [label \": load(save(k)) predicts the same mean\"], k2.predict(Xt), kk.predict(Xt), 1e-10);\n", + " catch err\n", + " s = lkb_check(s, [label \": fit + predict\"], false, \"error\", \"no error\", err.message);\n", + " end\n", + " end\n", + "else\n", + " s = skip(s, \"NestedKriging\", \"not in this mLibKriging\");\n", + "end" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c014", + "metadata": {}, + "outputs": [], + "source": [ + "q = strcmp(s.metrics(:, 5), \"q2\");\n", + "disp(s.metrics(q, [3, 4, 6]))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c015", + "metadata": {}, + "outputs": [], + "source": [ + "lkb_finalize(s);" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Octave", + "language": "octave", + "name": "octave" + }, + "language_info": { + "name": "octave" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/python/03_noise.ipynb b/python_Kriging-nugget-noise_noisy-data.ipynb similarity index 98% rename from notebooks/python/03_noise.ipynb rename to python_Kriging-nugget-noise_noisy-data.ipynb index c737eb7..1d0c11b 100644 --- a/notebooks/python/03_noise.ipynb +++ b/python_Kriging-nugget-noise_noisy-data.ipynb @@ -5,7 +5,7 @@ "id": "c000", "metadata": {}, "source": [ - "# 03 — Noisy observations: `noise=\"nugget\"` and known noise variances\n", + "# Python · Kriging with noise (`noise=\"nugget\"`, known variances) — noisy observations\n", "\n", "`Kriging` handles noisy responses through its `noise` argument: `\"nugget\"`\n", "estimates one homogeneous noise variance, a vector gives a known variance per\n", @@ -48,7 +48,7 @@ "import lkbench\n", "from lkbench import TESTS, design, metrics, compat\n", "\n", - "suite = lkbench.Suite(\"py03_noise\")\n", + "suite = lkbench.Suite(\"python_Kriging-nugget-noise_noisy-data\")\n", "print(\"pylibkriging\", compat.version())\n", "\n", "\n", diff --git a/notebooks/python/07_scalability.ipynb b/python_Kriging-vecchia-nystrom_large-n.ipynb similarity index 74% rename from notebooks/python/07_scalability.ipynb rename to python_Kriging-vecchia-nystrom_large-n.ipynb index 822d335..8f3f051 100644 --- a/notebooks/python/07_scalability.ipynb +++ b/python_Kriging-vecchia-nystrom_large-n.ipynb @@ -5,18 +5,19 @@ "id": "c000", "metadata": {}, "source": [ - "# 07 — Large n: Vecchia, Nyström, NestedKriging, subset of data\n", + "# Python · Kriging with `method=\"vecchia\"` / `\"nystrom\"` — large n\n", "\n", - "The exact fit costs O(n³). libKriging offers:\n", + "The exact fit costs O(n³). `Kriging` offers:\n", "\n", "* `method=\"vecchia(m)\"`: Vecchia likelihood with m conditioning neighbours\n", " (local, recommended in low dimension);\n", "* `method=\"nystrom(k)\"`: low-rank Nyström approximation with k landmarks\n", " (global);\n", - "* `NestedKriging`: a partition into groups, one sub-model per group,\n", - " aggregated predictions (`NK`, `PoE`, `gPoE`, `BCM`, `rBCM`);\n", "* `Kriging.subsetOfData`: a representative subset before an exact fit.\n", "\n", + "(`NestedKriging`, the partition-based alternative, on the same problems:\n", + "`python_NestedKriging_aggregations-large-n.ipynb`.)\n", + "\n", "Problems: Hartmann 3 (3D, local structure: Vecchia's ground) and the OTL\n", "circuit (6D, smooth: Nyström's ground), n = 1500 (exact fit still affordable,\n", "for reference) and n = 5000 (approximations only).\n", @@ -26,9 +27,7 @@ "* the method is explicit at prediction time: `predict(method=\"exact\")` on a\n", " Nyström/Vecchia fit raises, `method=` gives finite predictions;\n", "* accuracy of each approximation close to the exact fit (warn-level gaps,\n", - " error-level floors) and faster than the exact fit at n = 1500 (warn-level);\n", - "* `NestedKriging` `NK` interpolates the data (it is a proper kriging\n", - " predictor); the PoE family does not need to." + " error-level floors) and faster than the exact fit at n = 1500 (warn-level)." ] }, { @@ -47,7 +46,7 @@ "import lkbench\n", "from lkbench import TESTS, design, metrics, compat\n", "\n", - "suite = lkbench.Suite(\"py07_scalability\")\n", + "suite = lkbench.Suite(\"python_Kriging-vecchia-nystrom_large-n\")\n", "print(\"pylibkriging\", compat.version())\n", "HAS_METHOD = compat.has(\"method\")\n", "if not HAS_METHOD:\n", @@ -131,11 +130,6 @@ " \"vecchia(30)\": lambda Xa, ya: (lk.Kriging(ya, Xa, \"matern5_2\", method=\"vecchia(30)\"), \"vecchia\"),\n", " \"nystrom(100)\": lambda Xa, ya: (lk.Kriging(ya, Xa, \"matern5_2\", method=\"nystrom(100)\"), \"nystrom\"),\n", " })\n", - " if compat.has(\"NestedKriging\"):\n", - " variants[\"nested NK (20 groups)\"] = lambda Xa, ya: (\n", - " lk.NestedKriging(ya, Xa, \"matern5_2\", nb_groups=20, aggregation=\"NK\", seed=1), None)\n", - " variants[\"nested rBCM (20 groups)\"] = lambda Xa, ya: (\n", - " lk.NestedKriging(ya, Xa, \"matern5_2\", nb_groups=20, aggregation=\"rBCM\", seed=1), None)\n", " if compat.has(\"subsetOfData\"):\n", " def sod(Xa, ya):\n", " idx = np.ravel(lk.Kriging.subsetOfData(Xa, n_max=500, method=\"kmeans\", seed=1)).astype(int)\n", @@ -159,44 +153,10 @@ "suite.table().round(4)" ] }, - { - "cell_type": "markdown", - "id": "c006", - "metadata": {}, - "source": [ - "## `NestedKriging` aggregations: interpolation and variance" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c007", - "metadata": {}, - "outputs": [], - "source": [ - "if compat.has(\"NestedKriging\"):\n", - " tf = TESTS[\"branin\"]\n", - " X, y, Xt, yt = design.dataset(tf, 200, n_test=500, seed=53)\n", - " for agg in [\"NK\", \"PoE\", \"gPoE\", \"BCM\", \"rBCM\"]:\n", - " with suite.runs(f\"nested {agg}: fit\"):\n", - " k = lk.NestedKriging(y, X, \"matern5_2\", nb_groups=8, aggregation=agg, seed=2)\n", - " m, s = (np.ravel(a) for a in k.predict(Xt, return_stdev=True)[:2])\n", - " suite.true(f\"nested {agg}: stdev >= 0\", np.all(s >= 0))\n", - " suite.metric(\"branin/nested\", agg, **metrics.all_metrics(yt, m, s))\n", - " if agg == \"NK\":\n", - " mX = np.ravel(k.predict(X, return_stdev=True)[0])\n", - " suite.less(\"nested NK: interpolates (max |m(X)-y| / sd(y))\", np.max(np.abs(mX - y)) / np.std(y),\n", - " 1e-4)\n", - " display(suite.table().loc[\"branin/nested\"].round(4))\n", - " suite.raises(\"nested NK with a linear trend is rejected\",\n", - " lambda: lk.NestedKriging(y, X, \"matern5_2\", nb_groups=4, aggregation=\"NK\", regmodel=\"linear\"),\n", - " severity=\"warn\")" - ] - }, { "cell_type": "code", "execution_count": null, - "id": "c008", + "id": "c006", "metadata": {}, "outputs": [], "source": [ diff --git a/notebooks/python/11_cross_binding.ipynb b/python_Kriging_cross-binding-reference.ipynb similarity index 94% rename from notebooks/python/11_cross_binding.ipynb rename to python_Kriging_cross-binding-reference.ipynb index 945bb71..3f8ae7b 100644 --- a/notebooks/python/11_cross_binding.ipynb +++ b/python_Kriging_cross-binding-reference.ipynb @@ -5,7 +5,7 @@ "id": "c000", "metadata": {}, "source": [ - "# 11 — Cross-binding consistency: the Python side\n", + "# Python · Kriging — cross-binding consistency: the Python reference\n", "\n", "All bindings (Python, R, Octave, Julia) wrap the same C++ library: on the\n", "same data with the same options they must return the same numbers. Each\n", @@ -34,18 +34,16 @@ "outputs": [], "source": [ "import csv\n", - "from pathlib import Path\n", "\n", "import numpy as np\n", "import pylibkriging as lk\n", "\n", "import lkbench\n", "from lkbench import compat\n", - "from lkbench.checks import RESULTS_DIR\n", + "from lkbench.checks import DATA_DIR as DATA, RESULTS_DIR\n", "\n", - "suite = lkbench.Suite(\"py11_cross_binding\")\n", + "suite = lkbench.Suite(\"python_Kriging_cross-binding-reference\")\n", "print(\"pylibkriging\", compat.version())\n", - "DATA = Path(\"../../data\")\n", "DATASETS = [\"branin\", \"hartmann3\", \"borehole\"]\n", "rows = []\n", "\n", diff --git a/notebooks/python/06_high_dimension_mlp.ipynb b/python_Kriging_high-dimension-normalize-screening.ipynb similarity index 61% rename from notebooks/python/06_high_dimension_mlp.ipynb rename to python_Kriging_high-dimension-normalize-screening.ipynb index 0142185..85c77ee 100644 --- a/notebooks/python/06_high_dimension_mlp.ipynb +++ b/python_Kriging_high-dimension-normalize-screening.ipynb @@ -5,7 +5,7 @@ "id": "c000", "metadata": {}, "source": [ - "# 06 — Moderate and high dimension: engineering functions, `normalize`, `MLPKriging`\n", + "# Python · Kriging — moderate and high dimension: engineering functions, `normalize`, screening\n", "\n", "| function | d | character |\n", "|---|---|---|\n", @@ -23,11 +23,10 @@ " inputs in native units (Borehole spans 0.05 to 50 000) must not break a fit;\n", "* Q² floors per function (regression detection);\n", "* the fitted θ of Welch's **inactive** inputs are larger than those of the\n", - " active ones (screening ability, warn-level);\n", - "* `MLPKriging` (deep kernel) on a function with a low-dimensional structure:\n", - " finite predictions; with a multistart optimizer, a Q² close to plain\n", - " `Kriging`'s (error-level floor); the default single-start fit is compared\n", - " with the multistart one (warn-level)." + " active ones (screening ability, warn-level).\n", + "\n", + "`MLPKriging` on a high-dimensional function with a low-dimensional structure:\n", + "`python_MLPKriging_deep-kernel-projected-branin.ipynb`." ] }, { @@ -46,7 +45,7 @@ "import lkbench\n", "from lkbench import TESTS, design, metrics, compat\n", "\n", - "suite = lkbench.Suite(\"py06_high_dimension_mlp\")\n", + "suite = lkbench.Suite(\"python_Kriging_high-dimension-normalize-screening\")\n", "print(\"pylibkriging\", compat.version())\n", "\n", "\n", @@ -129,67 +128,10 @@ "pd.Series(theta, index=[f\"x{i + 1}\" for i in range(20)]).round(3).to_frame(\"theta\").T" ] }, - { - "cell_type": "markdown", - "id": "c006", - "metadata": {}, - "source": [ - "## `MLPKriging`: deep kernel on a low-dimensional structure\n", - "\n", - "f(x) = Branin(Wx) for x ∈ [0,1]¹⁰ and a fixed 2 × 10 projection W: the\n", - "function only varies in a 2D subspace, which a joint feature map can find\n", - "whereas a per-axis kernel cannot." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c007", - "metadata": {}, - "outputs": [], - "source": [ - "if compat.has(\"MLPKriging\"):\n", - " rng = np.random.default_rng(43)\n", - " W = rng.normal(size=(2, 10))\n", - " W /= np.abs(W).sum(axis=1, keepdims=True)\n", - " tfb = TESTS[\"branin\"]\n", - "\n", - " def projected(X):\n", - " Z = X @ W.T\n", - " Z = (Z - (W.clip(max=0).sum(axis=1))) / np.abs(W).sum(axis=1) # back to [0,1]^2\n", - " return tfb.unit(Z)\n", - "\n", - " X = design.lhs(120, 10, seed=44); Xt = design.uniform(2000, 10, seed=45)\n", - " y, yt = projected(X), projected(Xt)\n", - " kp = lk.Kriging(y, X, \"matern5_2\")\n", - " res_k = metrics.all_metrics(yt, *pred(kp, Xt))\n", - " suite.metric(\"projected_branin_10d\", \"Kriging\", **res_k)\n", - " for act in [\"tanh\", \"selu\"]:\n", - " for optim in [\"BFGS\", \"BFGS10\"]:\n", - " case = f\"MLPKriging({act}, {optim})\"\n", - " with suite.runs(f\"{case}: fit\"):\n", - " t0 = time.time()\n", - " km = lk.MLPKriging(y, X, hidden_dims=[16, 8], d_out=2, activation=act, kernel=\"matern5_2\",\n", - " optim=optim)\n", - " fit_s = time.time() - t0\n", - " m, s = pred(km, Xt)\n", - " suite.finite(f\"{case}: predictions finite\", m)\n", - " res = metrics.all_metrics(yt, m, s)\n", - " suite.metric(\"projected_branin_10d\", f\"MLPKriging {act} {optim}\", fit_s=fit_s,\n", - " ll=float(km.logLikelihood()), **res)\n", - " if optim == \"BFGS10\":\n", - " suite.greater(f\"{case}: Q2\", res[\"q2\"], 0.8)\n", - " else:\n", - " suite.greater(f\"{case}: Q2 - Q2(Kriging)\", res[\"q2\"] - res_k[\"q2\"], -0.1, severity=\"warn\")\n", - " display(suite.table().loc[\"projected_branin_10d\"].round(4))\n", - "else:\n", - " suite.skip(\"MLPKriging\", \"not available in this pylibkriging\")" - ] - }, { "cell_type": "code", "execution_count": null, - "id": "c008", + "id": "c006", "metadata": {}, "outputs": [], "source": [ diff --git a/notebooks/python/01_kriging_kernels_trends.ipynb b/python_Kriging_kernels-trends-interpolation.ipynb similarity index 97% rename from notebooks/python/01_kriging_kernels_trends.ipynb rename to python_Kriging_kernels-trends-interpolation.ipynb index 5517780..27f9a85 100644 --- a/notebooks/python/01_kriging_kernels_trends.ipynb +++ b/python_Kriging_kernels-trends-interpolation.ipynb @@ -5,7 +5,7 @@ "id": "c000", "metadata": {}, "source": [ - "# 01 — Kriging: kernels × trends on 1D/2D/3D functions\n", + "# Python · Kriging — kernels × trends, interpolation, predict (covariance, derivatives)\n", "\n", "Plain noise-free `Kriging`, every kernel (`gauss`, `exp`, `matern3_2`,\n", "`matern5_2`) crossed with every trend (`constant`, `linear`, `interactive`,\n", @@ -32,8 +32,8 @@ " `matern5_2` / `constant` with a multistart optimizer (`BFGS10`): a\n", " regression shows up here first. The same floor for the default single-start\n", " `BFGS` is warn-level: a single start can end in a poor local optimum of the\n", - " likelihood (see notebook 02), which is worth seeing in the report but is not\n", - " a defect of the model;\n", + " likelihood (see `python_Kriging_objectives-LL-LOO-LMP-gradients.ipynb`),\n", + " which is worth seeing in the report but is not a defect of the model;\n", "* **sanity of the fitted hyperparameters**: finite, positive θ and σ²;\n", "* **consistency**: `predict` on a batch equals `predict` point by point, the\n", " returned covariance has the predicted variances on its diagonal, and is\n", @@ -57,7 +57,7 @@ "import lkbench\n", "from lkbench import TESTS, design, metrics, compat\n", "\n", - "suite = lkbench.Suite(\"py01_kernels_trends\")\n", + "suite = lkbench.Suite(\"python_Kriging_kernels-trends-interpolation\")\n", "print(\"pylibkriging\", compat.version())\n", "\n", "KERNELS = [\"gauss\", \"exp\", \"matern3_2\", \"matern5_2\"]\n", diff --git a/notebooks/python/02_objectives_gradients.ipynb b/python_Kriging_objectives-LL-LOO-LMP-gradients.ipynb similarity index 98% rename from notebooks/python/02_objectives_gradients.ipynb rename to python_Kriging_objectives-LL-LOO-LMP-gradients.ipynb index c2e9912..9ba2c39 100644 --- a/notebooks/python/02_objectives_gradients.ipynb +++ b/python_Kriging_objectives-LL-LOO-LMP-gradients.ipynb @@ -5,7 +5,7 @@ "id": "c000", "metadata": {}, "source": [ - "# 02 — Objectives (LL, LOO, LMP), their gradients and optima\n", + "# Python · Kriging — objectives (LL, LOO, LMP), their gradients and optima\n", "\n", "The hyperparameters θ of `Kriging` are estimated by maximizing the\n", "log-likelihood (`LL`), maximizing the log marginal posterior (`LMP`), or\n", @@ -45,7 +45,7 @@ "import lkbench\n", "from lkbench import TESTS, design, metrics, compat\n", "\n", - "suite = lkbench.Suite(\"py02_objectives_gradients\")\n", + "suite = lkbench.Suite(\"python_Kriging_objectives-LL-LOO-LMP-gradients\")\n", "print(\"pylibkriging\", compat.version())\n", "CASES = {\"branin\": 25, \"hartmann3\": 40, \"ishigami\": 60, \"friedman\": 50}\n", "KERNELS = [\"gauss\", \"exp\", \"matern3_2\", \"matern5_2\"]\n", diff --git a/notebooks/python/09_robustness.ipynb b/python_Kriging_robustness-invalid-inputs.ipynb similarity index 98% rename from notebooks/python/09_robustness.ipynb rename to python_Kriging_robustness-invalid-inputs.ipynb index 337faf6..2867784 100644 --- a/notebooks/python/09_robustness.ipynb +++ b/python_Kriging_robustness-invalid-inputs.ipynb @@ -5,7 +5,7 @@ "id": "c000", "metadata": {}, "source": [ - "# 09 — Robustness: degenerate and adversarial inputs\n", + "# Python · Kriging — robustness: degenerate and adversarial inputs\n", "\n", "What a user eventually feeds a surrogate model, and what must happen:\n", "\n", @@ -40,7 +40,7 @@ "import lkbench\n", "from lkbench import TESTS, design, compat\n", "\n", - "suite = lkbench.Suite(\"py09_robustness\")\n", + "suite = lkbench.Suite(\"python_Kriging_robustness-invalid-inputs\")\n", "print(\"pylibkriging\", compat.version())\n", "tf = TESTS[\"branin\"]\n", "X, y, Xt, yt = design.dataset(tf, 20, n_test=200, seed=71)\n", diff --git a/notebooks/python/04_update_simulate.ipynb b/python_Kriging_update-simulate.ipynb similarity index 98% rename from notebooks/python/04_update_simulate.ipynb rename to python_Kriging_update-simulate.ipynb index 44b13ff..2e0c8a4 100644 --- a/notebooks/python/04_update_simulate.ipynb +++ b/python_Kriging_update-simulate.ipynb @@ -5,7 +5,7 @@ "id": "c000", "metadata": {}, "source": [ - "# 04 — `update`, `simulate`, `update_simulate`\n", + "# Python · Kriging — `update`, `simulate`, `update_simulate`\n", "\n", "Sequential use of a fitted model (sequential design, Bayesian optimization):\n", "\n", @@ -37,7 +37,7 @@ "import lkbench\n", "from lkbench import TESTS, design, compat\n", "\n", - "suite = lkbench.Suite(\"py04_update_simulate\")\n", + "suite = lkbench.Suite(\"python_Kriging_update-simulate\")\n", "print(\"pylibkriging\", compat.version())\n", "\n", "\n", diff --git a/notebooks/python/10_compare_python_packages.ipynb b/python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy.ipynb similarity index 90% rename from notebooks/python/10_compare_python_packages.ipynb rename to python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy.ipynb index 2181f55..abd91e1 100644 --- a/notebooks/python/10_compare_python_packages.ipynb +++ b/python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy.ipynb @@ -5,7 +5,7 @@ "id": "c000", "metadata": {}, "source": [ - "# 10 — Comparison with Python GP packages\n", + "# Python · Kriging vs scikit-learn, SMT, OpenTURNS, GPy\n", "\n", "Two kinds of comparisons, which should not be confused:\n", "\n", @@ -21,14 +21,47 @@ " benchmark, and the checks are warn-level: libKriging should not be\n", " markedly less accurate than the best package.\n", "\n", - "Packages (each optional: skipped if not installed): scikit-learn, SMT,\n", - "OpenTURNS, GPy." + "Packages (each installed below, and skipped if that fails): scikit-learn,\n", + "SMT, OpenTURNS, GPy." + ] + }, + { + "cell_type": "markdown", + "id": "c001", + "metadata": {}, + "source": [ + "## Installing scikit-learn, SMT, OpenTURNS, GPy\n", + "\n", + "With pip, into the kernel's environment, numpy pinned to its installed version\n", + "(a package that would downgrade it is not installed: the libKriging extension\n", + "is built against it). A package that cannot be installed is skipped, and\n", + "reported." ] }, { "cell_type": "code", "execution_count": null, - "id": "c001", + "id": "c002", + "metadata": {}, + "outputs": [], + "source": [ + "import importlib\n", + "import subprocess\n", + "import sys\n", + "\n", + "import numpy\n", + "\n", + "for package in [\"scikit-learn\", \"smt\", \"openturns\", \"GPy\"]:\n", + " r = subprocess.run([sys.executable, \"-m\", \"pip\", \"install\", \"--quiet\", package, f\"numpy=={numpy.__version__}\"],\n", + " capture_output=True, text=True)\n", + " print(f\"{package}: \" + (\"installed\" if r.returncode == 0 else \"NOT installed\\n\" + r.stderr.strip()[-800:]))\n", + "importlib.invalidate_caches()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c003", "metadata": {}, "outputs": [], "source": [ @@ -43,7 +76,7 @@ "from lkbench import TESTS, design, metrics, compat\n", "\n", "warnings.filterwarnings(\"ignore\")\n", - "suite = lkbench.Suite(\"py10_compare_python_packages\")\n", + "suite = lkbench.Suite(\"python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy\")\n", "sk = compat.optional(\"sklearn.gaussian_process\")\n", "smt = compat.optional(\"smt.surrogate_models\")\n", "ot = compat.optional(\"openturns\")\n", @@ -61,7 +94,7 @@ }, { "cell_type": "markdown", - "id": "c002", + "id": "c004", "metadata": {}, "source": [ "## 1. Same hyperparameters, same numbers" @@ -70,7 +103,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c003", + "id": "c005", "metadata": {}, "outputs": [], "source": [ @@ -112,7 +145,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c004", + "id": "c006", "metadata": {}, "outputs": [], "source": [ @@ -144,7 +177,7 @@ }, { "cell_type": "markdown", - "id": "c005", + "id": "c007", "metadata": {}, "source": [ "## 2. Default fits on a set of test functions\n", @@ -156,7 +189,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c006", + "id": "c008", "metadata": {}, "outputs": [], "source": [ @@ -245,7 +278,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c007", + "id": "c009", "metadata": {}, "outputs": [], "source": [ @@ -255,7 +288,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c008", + "id": "c010", "metadata": {}, "outputs": [], "source": [ @@ -273,7 +306,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c009", + "id": "c011", "metadata": {}, "outputs": [], "source": [ diff --git a/python_MLPKriging_deep-kernel-projected-branin.ipynb b/python_MLPKriging_deep-kernel-projected-branin.ipynb new file mode 100644 index 0000000..6705b30 --- /dev/null +++ b/python_MLPKriging_deep-kernel-projected-branin.ipynb @@ -0,0 +1,113 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "c000", + "metadata": {}, + "source": [ + "# Python · MLPKriging — deep kernel on a 10D function with a 2D structure\n", + "\n", + "f(x) = Branin(Wx) for x ∈ [0,1]¹⁰ and a fixed 2 × 10 projection W: the\n", + "function only varies in a 2D subspace, which a joint feature map (an MLP from\n", + "ℝ¹⁰ to ℝ², then a stationary kernel) can find, whereas a per-axis kernel\n", + "cannot.\n", + "\n", + "Checks: finite predictions; with a multistart optimizer, a Q² floor\n", + "(error-level); the single-start fits compared with plain `Kriging`\n", + "(warn-level), for two activations (`tanh`, smooth; `selu`, the default, with\n", + "a kink at 0)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c001", + "metadata": {}, + "outputs": [], + "source": [ + "import time\n", + "\n", + "import numpy as np\n", + "import pylibkriging as lk\n", + "\n", + "import lkbench\n", + "from lkbench import TESTS, design, metrics, compat\n", + "\n", + "suite = lkbench.Suite(\"python_MLPKriging_deep-kernel-projected-branin\")\n", + "print(\"pylibkriging\", compat.version())\n", + "\n", + "\n", + "def pred(k, X):\n", + " m, s, *_ = k.predict(X, return_stdev=True)\n", + " return np.ravel(m), np.ravel(s)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c002", + "metadata": {}, + "outputs": [], + "source": [ + "if compat.has(\"MLPKriging\"):\n", + " rng = np.random.default_rng(43)\n", + " W = rng.normal(size=(2, 10))\n", + " W /= np.abs(W).sum(axis=1, keepdims=True)\n", + " tfb = TESTS[\"branin\"]\n", + "\n", + " def projected(X):\n", + " Z = X @ W.T\n", + " Z = (Z - (W.clip(max=0).sum(axis=1))) / np.abs(W).sum(axis=1) # back to [0,1]^2\n", + " return tfb.unit(Z)\n", + "\n", + " X = design.lhs(120, 10, seed=44); Xt = design.uniform(2000, 10, seed=45)\n", + " y, yt = projected(X), projected(Xt)\n", + " kp = lk.Kriging(y, X, \"matern5_2\")\n", + " res_k = metrics.all_metrics(yt, *pred(kp, Xt))\n", + " suite.metric(\"projected_branin_10d\", \"Kriging\", **res_k)\n", + " for act in [\"tanh\", \"selu\"]:\n", + " for optim in [\"BFGS\", \"BFGS10\"]:\n", + " case = f\"MLPKriging({act}, {optim})\"\n", + " with suite.runs(f\"{case}: fit\"):\n", + " t0 = time.time()\n", + " km = lk.MLPKriging(y, X, hidden_dims=[16, 8], d_out=2, activation=act, kernel=\"matern5_2\",\n", + " optim=optim)\n", + " fit_s = time.time() - t0\n", + " m, s = pred(km, Xt)\n", + " suite.finite(f\"{case}: predictions finite\", m)\n", + " res = metrics.all_metrics(yt, m, s)\n", + " suite.metric(\"projected_branin_10d\", f\"MLPKriging {act} {optim}\", fit_s=fit_s,\n", + " ll=float(km.logLikelihood()), **res)\n", + " if optim == \"BFGS10\":\n", + " suite.greater(f\"{case}: Q2\", res[\"q2\"], 0.8)\n", + " else:\n", + " suite.greater(f\"{case}: Q2 - Q2(Kriging)\", res[\"q2\"] - res_k[\"q2\"], -0.1, severity=\"warn\")\n", + " display(suite.table().loc[\"projected_branin_10d\"].round(4))\n", + "else:\n", + " suite.skip(\"MLPKriging\", \"not available in this pylibkriging\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c003", + "metadata": {}, + "outputs": [], + "source": [ + "suite.finalize()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/python_NestedKriging_aggregations-large-n.ipynb b/python_NestedKriging_aggregations-large-n.ipynb new file mode 100644 index 0000000..9d70d4b --- /dev/null +++ b/python_NestedKriging_aggregations-large-n.ipynb @@ -0,0 +1,161 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "c000", + "metadata": {}, + "source": [ + "# Python · NestedKriging — aggregations, large n\n", + "\n", + "`NestedKriging` partitions the design into groups, fits one `Kriging`\n", + "sub-model per group (common hyperparameters), and aggregates the sub-model\n", + "predictions: `NK` (nested kriging: kriging of the sub-predictors, a proper\n", + "kriging predictor), or the products of experts `PoE`, `gPoE`, `BCM`, `rBCM`\n", + "(cheaper, variances not fully consistent).\n", + "\n", + "Problems: as in `python_Kriging-vecchia-nystrom_large-n.ipynb`, Hartmann 3\n", + "and the OTL circuit with n = 1500 (exact fit as reference) and n = 5000; then\n", + "every aggregation on Branin.\n", + "\n", + "Checks:\n", + "* accuracy close to the exact fit (warn-level gap, error-level floor), and\n", + " faster than the exact fit at n = 1500 (warn-level);\n", + "* `NK` interpolates the data (it is a proper kriging predictor); the PoE family\n", + " does not need to; all stdev ≥ 0;\n", + "* `NK` with a non-constant trend is rejected (simple-kriging theory)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c001", + "metadata": {}, + "outputs": [], + "source": [ + "import time\n", + "\n", + "import numpy as np\n", + "import pandas as pd\n", + "import pylibkriging as lk\n", + "\n", + "import lkbench\n", + "from lkbench import TESTS, design, metrics, compat\n", + "\n", + "suite = lkbench.Suite(\"python_NestedKriging_aggregations-large-n\")\n", + "print(\"pylibkriging\", compat.version())\n", + "if not compat.has(\"NestedKriging\"):\n", + " suite.skip(\"NestedKriging\", \"not available in this pylibkriging\")\n", + "\n", + "\n", + "def pred(k, X, **kw):\n", + " m, s, *_ = k.predict(X, return_stdev=True, **kw)\n", + " return np.ravel(m), np.ravel(s)\n", + "\n", + "\n", + "def timed(f):\n", + " t0 = time.time()\n", + " out = f()\n", + " return out, time.time() - t0" + ] + }, + { + "cell_type": "markdown", + "id": "c002", + "metadata": {}, + "source": [ + "## Accuracy and cost against the exact fit" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c003", + "metadata": {}, + "outputs": [], + "source": [ + "CASES = {\"hartmann3\": (1500, 5000), \"otl_circuit\": (1500, 5000)}\n", + "FLOORS = {\"hartmann3\": 0.98, \"otl_circuit\": 0.99}\n", + "if compat.has(\"NestedKriging\"):\n", + " for fname, (n_mid, n_big) in CASES.items():\n", + " tf = TESTS[fname]\n", + " X, y, Xt, yt = design.dataset(tf, n_big, n_test=2000, seed=52)\n", + " ke, t_exact = timed(lambda: lk.Kriging(y[:n_mid], X[:n_mid], \"matern5_2\"))\n", + " res_e = metrics.all_metrics(yt, *pred(ke, Xt))\n", + " suite.metric(f\"{fname}/n={n_mid}\", \"exact\", fit_s=t_exact, **res_e)\n", + " for agg in [\"NK\", \"rBCM\"]:\n", + " label = f\"nested {agg} (20 groups)\"\n", + " for n in (n_mid, n_big):\n", + " case = f\"{fname}/n={n}\"\n", + " with suite.runs(f\"{case}/{label}: fit + predict\"):\n", + " k, t = timed(lambda: lk.NestedKriging(y[:n], X[:n], \"matern5_2\", nb_groups=20, aggregation=agg,\n", + " seed=1))\n", + " m, s = pred(k, Xt)\n", + " suite.finite(f\"{case}/{label}: finite predictions\", m)\n", + " res = metrics.all_metrics(yt, m, s)\n", + " suite.metric(case, label, fit_s=t, **res)\n", + " suite.greater(f\"{case}/{label}: Q2\", res[\"q2\"], FLOORS[fname] - 0.05)\n", + " if n == n_mid:\n", + " suite.greater(f\"{case}/{label}: Q2 - Q2(exact)\", res[\"q2\"] - res_e[\"q2\"], -0.02,\n", + " severity=\"warn\")\n", + " suite.less(f\"{case}/{label}: fit time / exact fit time\", t / t_exact, 1.0, severity=\"warn\")\n", + " display(suite.table().round(4))" + ] + }, + { + "cell_type": "markdown", + "id": "c004", + "metadata": {}, + "source": [ + "## `NestedKriging` aggregations: interpolation and variance" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c005", + "metadata": {}, + "outputs": [], + "source": [ + "if compat.has(\"NestedKriging\"):\n", + " tf = TESTS[\"branin\"]\n", + " X, y, Xt, yt = design.dataset(tf, 200, n_test=500, seed=53)\n", + " for agg in [\"NK\", \"PoE\", \"gPoE\", \"BCM\", \"rBCM\"]:\n", + " with suite.runs(f\"nested {agg}: fit\"):\n", + " k = lk.NestedKriging(y, X, \"matern5_2\", nb_groups=8, aggregation=agg, seed=2)\n", + " m, s = (np.ravel(a) for a in k.predict(Xt, return_stdev=True)[:2])\n", + " suite.true(f\"nested {agg}: stdev >= 0\", np.all(s >= 0))\n", + " suite.metric(\"branin/nested\", agg, **metrics.all_metrics(yt, m, s))\n", + " if agg == \"NK\":\n", + " mX = np.ravel(k.predict(X, return_stdev=True)[0])\n", + " suite.less(\"nested NK: interpolates (max |m(X)-y| / sd(y))\", np.max(np.abs(mX - y)) / np.std(y),\n", + " 1e-4)\n", + " display(suite.table().loc[\"branin/nested\"].round(4))\n", + " suite.raises(\"nested NK with a linear trend is rejected\",\n", + " lambda: lk.NestedKriging(y, X, \"matern5_2\", nb_groups=4, aggregation=\"NK\", regmodel=\"linear\"),\n", + " severity=\"warn\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c006", + "metadata": {}, + "outputs": [], + "source": [ + "suite.finalize()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/python/05_warpkriging.ipynb b/python_WarpKriging_warpings-categorical-ordinal.ipynb similarity index 98% rename from notebooks/python/05_warpkriging.ipynb rename to python_WarpKriging_warpings-categorical-ordinal.ipynb index 89c0ecf..11e4741 100644 --- a/notebooks/python/05_warpkriging.ipynb +++ b/python_WarpKriging_warpings-categorical-ordinal.ipynb @@ -5,7 +5,7 @@ "id": "c000", "metadata": {}, "source": [ - "# 05 — `WarpKriging`: non-stationarity, categorical and ordinal inputs\n", + "# Python · WarpKriging — non-stationarity, categorical and ordinal inputs\n", "\n", "`WarpKriging` learns a transform per input variable jointly with the GP\n", "hyperparameters. Problems where a plain stationary GP is the wrong model:\n", @@ -37,7 +37,7 @@ "import lkbench\n", "from lkbench import TESTS, design, metrics, compat\n", "\n", - "suite = lkbench.Suite(\"py05_warpkriging\")\n", + "suite = lkbench.Suite(\"python_WarpKriging_warpings-categorical-ordinal\")\n", "print(\"pylibkriging\", compat.version())\n", "if not compat.has(\"WarpKriging\"):\n", " suite.skip(\"WarpKriging\", \"not available in this pylibkriging\")\n", diff --git a/notebooks/python/08_api_persistence.ipynb b/python_all-classes_save-load-pickle-sklearn.ipynb similarity index 89% rename from notebooks/python/08_api_persistence.ipynb rename to python_all-classes_save-load-pickle-sklearn.ipynb index f7ae830..ebdcf22 100644 --- a/notebooks/python/08_api_persistence.ipynb +++ b/python_all-classes_save-load-pickle-sklearn.ipynb @@ -5,7 +5,7 @@ "id": "c000", "metadata": {}, "source": [ - "# 08 — API: save/load, pickle, copy, accessors, scikit-learn estimators\n", + "# Python · all classes — save/load, pickle, copy, accessors, scikit-learn estimators\n", "\n", "Persistence and introspection, for every model class:\n", "\n", @@ -38,7 +38,7 @@ "import lkbench\n", "from lkbench import TESTS, design, compat\n", "\n", - "suite = lkbench.Suite(\"py08_api_persistence\")\n", + "suite = lkbench.Suite(\"python_all-classes_save-load-pickle-sklearn\")\n", "print(\"pylibkriging\", compat.version())\n", "tmp = Path(tempfile.mkdtemp())\n", "\n", @@ -190,7 +190,13 @@ "id": "c008", "metadata": {}, "source": [ - "## scikit-learn estimators" + "## Installing scikit-learn\n", + "\n", + "Needed by the estimators of `pylibkriging.sklearn` below. With pip, into the\n", + "kernel's environment, numpy pinned to its installed version (a package that\n", + "would downgrade it is not installed: the libKriging extension is built\n", + "against it). A package that cannot be installed is skipped, and\n", + "reported." ] }, { @@ -199,6 +205,34 @@ "id": "c009", "metadata": {}, "outputs": [], + "source": [ + "import importlib\n", + "import subprocess\n", + "import sys\n", + "\n", + "import numpy\n", + "\n", + "for package in [\"scikit-learn\"]:\n", + " r = subprocess.run([sys.executable, \"-m\", \"pip\", \"install\", \"--quiet\", package, f\"numpy=={numpy.__version__}\"],\n", + " capture_output=True, text=True)\n", + " print(f\"{package}: \" + (\"installed\" if r.returncode == 0 else \"NOT installed\\n\" + r.stderr.strip()[-800:]))\n", + "importlib.invalidate_caches()" + ] + }, + { + "cell_type": "markdown", + "id": "c010", + "metadata": {}, + "source": [ + "## scikit-learn estimators" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c011", + "metadata": {}, + "outputs": [], "source": [ "if compat.has(\"sklearn\"):\n", " from sklearn.base import clone\n", @@ -239,7 +273,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c010", + "id": "c012", "metadata": {}, "outputs": [], "source": [ diff --git a/notebooks/python/00_test_functions.ipynb b/python_no-model_test-functions-reference-values.ipynb similarity index 96% rename from notebooks/python/00_test_functions.ipynb rename to python_no-model_test-functions-reference-values.ipynb index 1ee5720..ed2f7e9 100644 --- a/notebooks/python/00_test_functions.ipynb +++ b/python_no-model_test-functions-reference-values.ipynb @@ -5,7 +5,7 @@ "id": "c000", "metadata": {}, "source": [ - "# 00 — Test functions\n", + "# Python · no model — the test functions against their published reference values\n", "\n", "Every benchmark of this repository relies on the test functions of\n", "`lkbench.functions` (after the [Virtual Library of Simulation Experiments](https://www.sfu.ca/~ssurjano/)).\n", @@ -28,7 +28,7 @@ "import lkbench\n", "from lkbench import TESTS, design\n", "\n", - "suite = lkbench.Suite(\"py00_test_functions\")" + "suite = lkbench.Suite(\"python_no-model_test-functions-reference-values\")" ] }, { diff --git a/tools/baseline/metrics-master.csv b/tools/baseline/metrics-master.csv new file mode 100644 index 0000000..88c120c --- /dev/null +++ b/tools/baseline/metrics-master.csv @@ -0,0 +1,956 @@ +suite,lang,case,package,metric,value +python_Kriging_kernels-trends-interpolation,python,forrester/gauss/constant,pylibkriging,fit_s,0.00161386 +python_Kriging_kernels-trends-interpolation,python,forrester/gauss/constant,pylibkriging,rmse,0.0848271 +python_Kriging_kernels-trends-interpolation,python,forrester/gauss/constant,pylibkriging,q2,0.999634 +python_Kriging_kernels-trends-interpolation,python,forrester/gauss/constant,pylibkriging,nlpd,-3.06486 +python_Kriging_kernels-trends-interpolation,python,forrester/gauss/linear,pylibkriging,fit_s,0.00126195 +python_Kriging_kernels-trends-interpolation,python,forrester/gauss/linear,pylibkriging,rmse,0.0855468 +python_Kriging_kernels-trends-interpolation,python,forrester/gauss/linear,pylibkriging,q2,0.999628 +python_Kriging_kernels-trends-interpolation,python,forrester/gauss/linear,pylibkriging,nlpd,-3.11029 +python_Kriging_kernels-trends-interpolation,python,forrester/gauss/interactive,pylibkriging,fit_s,0.00143647 +python_Kriging_kernels-trends-interpolation,python,forrester/gauss/interactive,pylibkriging,rmse,0.0855468 +python_Kriging_kernels-trends-interpolation,python,forrester/gauss/interactive,pylibkriging,q2,0.999628 +python_Kriging_kernels-trends-interpolation,python,forrester/gauss/interactive,pylibkriging,nlpd,-3.11029 +python_Kriging_kernels-trends-interpolation,python,forrester/gauss/quadratic,pylibkriging,fit_s,0.000566483 +python_Kriging_kernels-trends-interpolation,python,forrester/gauss/quadratic,pylibkriging,rmse,1.57249 +python_Kriging_kernels-trends-interpolation,python,forrester/gauss/quadratic,pylibkriging,q2,0.874394 +python_Kriging_kernels-trends-interpolation,python,forrester/gauss/quadratic,pylibkriging,nlpd,5.33879 +python_Kriging_kernels-trends-interpolation,python,forrester/exp/constant,pylibkriging,fit_s,0.00041604 +python_Kriging_kernels-trends-interpolation,python,forrester/exp/constant,pylibkriging,rmse,0.721582 +python_Kriging_kernels-trends-interpolation,python,forrester/exp/constant,pylibkriging,q2,0.973551 +python_Kriging_kernels-trends-interpolation,python,forrester/exp/constant,pylibkriging,nlpd,1.54551 +python_Kriging_kernels-trends-interpolation,python,forrester/exp/linear,pylibkriging,fit_s,0.000438213 +python_Kriging_kernels-trends-interpolation,python,forrester/exp/linear,pylibkriging,rmse,0.793413 +python_Kriging_kernels-trends-interpolation,python,forrester/exp/linear,pylibkriging,q2,0.968024 +python_Kriging_kernels-trends-interpolation,python,forrester/exp/linear,pylibkriging,nlpd,1.50639 +python_Kriging_kernels-trends-interpolation,python,forrester/exp/interactive,pylibkriging,fit_s,0.000480413 +python_Kriging_kernels-trends-interpolation,python,forrester/exp/interactive,pylibkriging,rmse,0.793413 +python_Kriging_kernels-trends-interpolation,python,forrester/exp/interactive,pylibkriging,q2,0.968024 +python_Kriging_kernels-trends-interpolation,python,forrester/exp/interactive,pylibkriging,nlpd,1.50639 +python_Kriging_kernels-trends-interpolation,python,forrester/exp/quadratic,pylibkriging,fit_s,0.000730038 +python_Kriging_kernels-trends-interpolation,python,forrester/exp/quadratic,pylibkriging,rmse,0.723267 +python_Kriging_kernels-trends-interpolation,python,forrester/exp/quadratic,pylibkriging,q2,0.973428 +python_Kriging_kernels-trends-interpolation,python,forrester/exp/quadratic,pylibkriging,nlpd,1.28782 +python_Kriging_kernels-trends-interpolation,python,forrester/matern3_2/constant,pylibkriging,fit_s,0.00060606 +python_Kriging_kernels-trends-interpolation,python,forrester/matern3_2/constant,pylibkriging,rmse,0.334007 +python_Kriging_kernels-trends-interpolation,python,forrester/matern3_2/constant,pylibkriging,q2,0.994333 +python_Kriging_kernels-trends-interpolation,python,forrester/matern3_2/constant,pylibkriging,nlpd,-0.227715 +python_Kriging_kernels-trends-interpolation,python,forrester/matern3_2/linear,pylibkriging,fit_s,0.000588894 +python_Kriging_kernels-trends-interpolation,python,forrester/matern3_2/linear,pylibkriging,rmse,0.364477 +python_Kriging_kernels-trends-interpolation,python,forrester/matern3_2/linear,pylibkriging,q2,0.993252 +python_Kriging_kernels-trends-interpolation,python,forrester/matern3_2/linear,pylibkriging,nlpd,-0.187262 +python_Kriging_kernels-trends-interpolation,python,forrester/matern3_2/interactive,pylibkriging,fit_s,0.000905514 +python_Kriging_kernels-trends-interpolation,python,forrester/matern3_2/interactive,pylibkriging,rmse,0.364477 +python_Kriging_kernels-trends-interpolation,python,forrester/matern3_2/interactive,pylibkriging,q2,0.993252 +python_Kriging_kernels-trends-interpolation,python,forrester/matern3_2/interactive,pylibkriging,nlpd,-0.187262 +python_Kriging_kernels-trends-interpolation,python,forrester/matern3_2/quadratic,pylibkriging,fit_s,0.000975609 +python_Kriging_kernels-trends-interpolation,python,forrester/matern3_2/quadratic,pylibkriging,rmse,0.435609 +python_Kriging_kernels-trends-interpolation,python,forrester/matern3_2/quadratic,pylibkriging,q2,0.990361 +python_Kriging_kernels-trends-interpolation,python,forrester/matern3_2/quadratic,pylibkriging,nlpd,-0.183076 +python_Kriging_kernels-trends-interpolation,python,forrester/matern5_2/constant,pylibkriging,fit_s,0.000726223 +python_Kriging_kernels-trends-interpolation,python,forrester/matern5_2/constant,pylibkriging,rmse,0.281945 +python_Kriging_kernels-trends-interpolation,python,forrester/matern5_2/constant,pylibkriging,q2,0.995962 +python_Kriging_kernels-trends-interpolation,python,forrester/matern5_2/constant,pylibkriging,nlpd,-1.01945 +python_Kriging_kernels-trends-interpolation,python,forrester/matern5_2/constant,pylibkriging BFGS10,rmse,0.281945 +python_Kriging_kernels-trends-interpolation,python,forrester/matern5_2/constant,pylibkriging BFGS10,q2,0.995962 +python_Kriging_kernels-trends-interpolation,python,forrester/matern5_2/constant,pylibkriging BFGS10,nlpd,-1.01945 +python_Kriging_kernels-trends-interpolation,python,forrester/matern5_2/linear,pylibkriging,fit_s,0.000932932 +python_Kriging_kernels-trends-interpolation,python,forrester/matern5_2/linear,pylibkriging,rmse,0.298959 +python_Kriging_kernels-trends-interpolation,python,forrester/matern5_2/linear,pylibkriging,q2,0.99546 +python_Kriging_kernels-trends-interpolation,python,forrester/matern5_2/linear,pylibkriging,nlpd,-0.95551 +python_Kriging_kernels-trends-interpolation,python,forrester/matern5_2/interactive,pylibkriging,fit_s,0.000725269 +python_Kriging_kernels-trends-interpolation,python,forrester/matern5_2/interactive,pylibkriging,rmse,0.298959 +python_Kriging_kernels-trends-interpolation,python,forrester/matern5_2/interactive,pylibkriging,q2,0.99546 +python_Kriging_kernels-trends-interpolation,python,forrester/matern5_2/interactive,pylibkriging,nlpd,-0.95551 +python_Kriging_kernels-trends-interpolation,python,forrester/matern5_2/quadratic,pylibkriging,fit_s,0.00131941 +python_Kriging_kernels-trends-interpolation,python,forrester/matern5_2/quadratic,pylibkriging,rmse,0.365039 +python_Kriging_kernels-trends-interpolation,python,forrester/matern5_2/quadratic,pylibkriging,q2,0.993231 +python_Kriging_kernels-trends-interpolation,python,forrester/matern5_2/quadratic,pylibkriging,nlpd,-0.790032 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/gauss/constant,pylibkriging,fit_s,0.0016458 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/gauss/constant,pylibkriging,rmse,0.191274 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/gauss/constant,pylibkriging,q2,0.978742 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/gauss/constant,pylibkriging,nlpd,-0.704948 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/gauss/linear,pylibkriging,fit_s,0.00319505 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/gauss/linear,pylibkriging,rmse,0.236028 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/gauss/linear,pylibkriging,q2,0.96763 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/gauss/linear,pylibkriging,nlpd,-0.580993 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/gauss/interactive,pylibkriging,fit_s,0.00250673 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/gauss/interactive,pylibkriging,rmse,0.236028 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/gauss/interactive,pylibkriging,q2,0.96763 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/gauss/interactive,pylibkriging,nlpd,-0.580993 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/gauss/quadratic,pylibkriging,fit_s,0.00243235 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/gauss/quadratic,pylibkriging,rmse,0.138869 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/gauss/quadratic,pylibkriging,q2,0.988795 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/gauss/quadratic,pylibkriging,nlpd,-0.530437 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/exp/constant,pylibkriging,fit_s,0.000752449 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/exp/constant,pylibkriging,rmse,0.196586 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/exp/constant,pylibkriging,q2,0.977545 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/exp/constant,pylibkriging,nlpd,-0.0621192 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/exp/linear,pylibkriging,fit_s,0.0010047 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/exp/linear,pylibkriging,rmse,0.20979 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/exp/linear,pylibkriging,q2,0.974427 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/exp/linear,pylibkriging,nlpd,-0.0466568 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/exp/interactive,pylibkriging,fit_s,0.000931501 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/exp/interactive,pylibkriging,rmse,0.20979 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/exp/interactive,pylibkriging,q2,0.974427 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/exp/interactive,pylibkriging,nlpd,-0.0466568 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/exp/quadratic,pylibkriging,fit_s,0.0025785 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/exp/quadratic,pylibkriging,rmse,0.220579 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/exp/quadratic,pylibkriging,q2,0.971729 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/exp/quadratic,pylibkriging,nlpd,0.136521 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern3_2/constant,pylibkriging,fit_s,0.000761509 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern3_2/constant,pylibkriging,rmse,0.177253 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern3_2/constant,pylibkriging,q2,0.981744 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern3_2/constant,pylibkriging,nlpd,-0.482212 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern3_2/linear,pylibkriging,fit_s,0.00132823 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern3_2/linear,pylibkriging,rmse,0.211499 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern3_2/linear,pylibkriging,q2,0.974009 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern3_2/linear,pylibkriging,nlpd,-0.375844 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern3_2/interactive,pylibkriging,fit_s,0.000978708 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern3_2/interactive,pylibkriging,rmse,0.211499 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern3_2/interactive,pylibkriging,q2,0.974009 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern3_2/interactive,pylibkriging,nlpd,-0.375844 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern3_2/quadratic,pylibkriging,fit_s,0.00228167 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern3_2/quadratic,pylibkriging,rmse,0.173012 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern3_2/quadratic,pylibkriging,q2,0.982607 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern3_2/quadratic,pylibkriging,nlpd,-0.223308 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern5_2/constant,pylibkriging,fit_s,0.00131416 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern5_2/constant,pylibkriging,rmse,0.177638 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern5_2/constant,pylibkriging,q2,0.981665 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern5_2/constant,pylibkriging,nlpd,-0.575421 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern5_2/constant,pylibkriging BFGS10,rmse,0.177638 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern5_2/constant,pylibkriging BFGS10,q2,0.981665 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern5_2/constant,pylibkriging BFGS10,nlpd,-0.575422 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern5_2/linear,pylibkriging,fit_s,0.00130415 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern5_2/linear,pylibkriging,rmse,0.219759 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern5_2/linear,pylibkriging,q2,0.971939 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern5_2/linear,pylibkriging,nlpd,-0.458933 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern5_2/interactive,pylibkriging,fit_s,0.00119758 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern5_2/interactive,pylibkriging,rmse,0.219759 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern5_2/interactive,pylibkriging,q2,0.971939 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern5_2/interactive,pylibkriging,nlpd,-0.458933 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern5_2/quadratic,pylibkriging,fit_s,0.00169826 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern5_2/quadratic,pylibkriging,rmse,0.160506 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern5_2/quadratic,pylibkriging,q2,0.985031 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern5_2/quadratic,pylibkriging,nlpd,-0.337276 +python_Kriging_kernels-trends-interpolation,python,branin/gauss/constant,pylibkriging,fit_s,0.00104284 +python_Kriging_kernels-trends-interpolation,python,branin/gauss/constant,pylibkriging,rmse,7.2629 +python_Kriging_kernels-trends-interpolation,python,branin/gauss/constant,pylibkriging,q2,0.980655 +python_Kriging_kernels-trends-interpolation,python,branin/gauss/constant,pylibkriging,nlpd,4.3152 +python_Kriging_kernels-trends-interpolation,python,branin/gauss/linear,pylibkriging,fit_s,0.00129008 +python_Kriging_kernels-trends-interpolation,python,branin/gauss/linear,pylibkriging,rmse,7.25733 +python_Kriging_kernels-trends-interpolation,python,branin/gauss/linear,pylibkriging,q2,0.980685 +python_Kriging_kernels-trends-interpolation,python,branin/gauss/linear,pylibkriging,nlpd,4.30813 +python_Kriging_kernels-trends-interpolation,python,branin/gauss/interactive,pylibkriging,fit_s,0.00280094 +python_Kriging_kernels-trends-interpolation,python,branin/gauss/interactive,pylibkriging,rmse,7.2576 +python_Kriging_kernels-trends-interpolation,python,branin/gauss/interactive,pylibkriging,q2,0.980683 +python_Kriging_kernels-trends-interpolation,python,branin/gauss/interactive,pylibkriging,nlpd,4.30176 +python_Kriging_kernels-trends-interpolation,python,branin/gauss/quadratic,pylibkriging,fit_s,0.00112319 +python_Kriging_kernels-trends-interpolation,python,branin/gauss/quadratic,pylibkriging,rmse,7.27808 +python_Kriging_kernels-trends-interpolation,python,branin/gauss/quadratic,pylibkriging,q2,0.980574 +python_Kriging_kernels-trends-interpolation,python,branin/gauss/quadratic,pylibkriging,nlpd,4.29399 +python_Kriging_kernels-trends-interpolation,python,branin/exp/constant,pylibkriging,fit_s,0.00222707 +python_Kriging_kernels-trends-interpolation,python,branin/exp/constant,pylibkriging,rmse,14.5887 +python_Kriging_kernels-trends-interpolation,python,branin/exp/constant,pylibkriging,q2,0.921947 +python_Kriging_kernels-trends-interpolation,python,branin/exp/constant,pylibkriging,nlpd,4.09457 +python_Kriging_kernels-trends-interpolation,python,branin/exp/linear,pylibkriging,fit_s,0.00213099 +python_Kriging_kernels-trends-interpolation,python,branin/exp/linear,pylibkriging,rmse,14.9622 +python_Kriging_kernels-trends-interpolation,python,branin/exp/linear,pylibkriging,q2,0.9179 +python_Kriging_kernels-trends-interpolation,python,branin/exp/linear,pylibkriging,nlpd,4.0909 +python_Kriging_kernels-trends-interpolation,python,branin/exp/interactive,pylibkriging,fit_s,0.00196409 +python_Kriging_kernels-trends-interpolation,python,branin/exp/interactive,pylibkriging,rmse,11.6963 +python_Kriging_kernels-trends-interpolation,python,branin/exp/interactive,pylibkriging,q2,0.94983 +python_Kriging_kernels-trends-interpolation,python,branin/exp/interactive,pylibkriging,nlpd,3.79671 +python_Kriging_kernels-trends-interpolation,python,branin/exp/quadratic,pylibkriging,fit_s,0.00227737 +python_Kriging_kernels-trends-interpolation,python,branin/exp/quadratic,pylibkriging,rmse,9.33909 +python_Kriging_kernels-trends-interpolation,python,branin/exp/quadratic,pylibkriging,q2,0.968014 +python_Kriging_kernels-trends-interpolation,python,branin/exp/quadratic,pylibkriging,nlpd,3.62362 +python_Kriging_kernels-trends-interpolation,python,branin/matern3_2/constant,pylibkriging,fit_s,0.00244212 +python_Kriging_kernels-trends-interpolation,python,branin/matern3_2/constant,pylibkriging,rmse,3.24796 +python_Kriging_kernels-trends-interpolation,python,branin/matern3_2/constant,pylibkriging,q2,0.996131 +python_Kriging_kernels-trends-interpolation,python,branin/matern3_2/constant,pylibkriging,nlpd,2.30444 +python_Kriging_kernels-trends-interpolation,python,branin/matern3_2/linear,pylibkriging,fit_s,0.00244641 +python_Kriging_kernels-trends-interpolation,python,branin/matern3_2/linear,pylibkriging,rmse,3.14402 +python_Kriging_kernels-trends-interpolation,python,branin/matern3_2/linear,pylibkriging,q2,0.996375 +python_Kriging_kernels-trends-interpolation,python,branin/matern3_2/linear,pylibkriging,nlpd,2.28067 +python_Kriging_kernels-trends-interpolation,python,branin/matern3_2/interactive,pylibkriging,fit_s,0.00703263 +python_Kriging_kernels-trends-interpolation,python,branin/matern3_2/interactive,pylibkriging,rmse,3.19226 +python_Kriging_kernels-trends-interpolation,python,branin/matern3_2/interactive,pylibkriging,q2,0.996263 +python_Kriging_kernels-trends-interpolation,python,branin/matern3_2/interactive,pylibkriging,nlpd,2.20359 +python_Kriging_kernels-trends-interpolation,python,branin/matern3_2/quadratic,pylibkriging,fit_s,0.00404716 +python_Kriging_kernels-trends-interpolation,python,branin/matern3_2/quadratic,pylibkriging,rmse,3.01232 +python_Kriging_kernels-trends-interpolation,python,branin/matern3_2/quadratic,pylibkriging,q2,0.996672 +python_Kriging_kernels-trends-interpolation,python,branin/matern3_2/quadratic,pylibkriging,nlpd,2.11627 +python_Kriging_kernels-trends-interpolation,python,branin/matern5_2/constant,pylibkriging,fit_s,0.0029068 +python_Kriging_kernels-trends-interpolation,python,branin/matern5_2/constant,pylibkriging,rmse,0.52012 +python_Kriging_kernels-trends-interpolation,python,branin/matern5_2/constant,pylibkriging,q2,0.999901 +python_Kriging_kernels-trends-interpolation,python,branin/matern5_2/constant,pylibkriging,nlpd,0.283399 +python_Kriging_kernels-trends-interpolation,python,branin/matern5_2/constant,pylibkriging BFGS10,rmse,0.52012 +python_Kriging_kernels-trends-interpolation,python,branin/matern5_2/constant,pylibkriging BFGS10,q2,0.999901 +python_Kriging_kernels-trends-interpolation,python,branin/matern5_2/constant,pylibkriging BFGS10,nlpd,0.283395 +python_Kriging_kernels-trends-interpolation,python,branin/matern5_2/linear,pylibkriging,fit_s,0.00212407 +python_Kriging_kernels-trends-interpolation,python,branin/matern5_2/linear,pylibkriging,rmse,0.460868 +python_Kriging_kernels-trends-interpolation,python,branin/matern5_2/linear,pylibkriging,q2,0.999922 +python_Kriging_kernels-trends-interpolation,python,branin/matern5_2/linear,pylibkriging,nlpd,0.209741 +python_Kriging_kernels-trends-interpolation,python,branin/matern5_2/interactive,pylibkriging,fit_s,0.0018971 +python_Kriging_kernels-trends-interpolation,python,branin/matern5_2/interactive,pylibkriging,rmse,0.498484 +python_Kriging_kernels-trends-interpolation,python,branin/matern5_2/interactive,pylibkriging,q2,0.999909 +python_Kriging_kernels-trends-interpolation,python,branin/matern5_2/interactive,pylibkriging,nlpd,0.239079 +python_Kriging_kernels-trends-interpolation,python,branin/matern5_2/quadratic,pylibkriging,fit_s,0.00570393 +python_Kriging_kernels-trends-interpolation,python,branin/matern5_2/quadratic,pylibkriging,rmse,0.67472 +python_Kriging_kernels-trends-interpolation,python,branin/matern5_2/quadratic,pylibkriging,q2,0.999833 +python_Kriging_kernels-trends-interpolation,python,branin/matern5_2/quadratic,pylibkriging,nlpd,0.419266 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/gauss/constant,pylibkriging,fit_s,0.00197244 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/gauss/constant,pylibkriging,rmse,18.9501 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/gauss/constant,pylibkriging,q2,0.536354 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/gauss/constant,pylibkriging,nlpd,84.151 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/gauss/linear,pylibkriging,fit_s,0.00170588 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/gauss/linear,pylibkriging,rmse,18.9579 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/gauss/linear,pylibkriging,q2,0.535976 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/gauss/linear,pylibkriging,nlpd,44.8696 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/gauss/interactive,pylibkriging,fit_s,0.0020082 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/gauss/interactive,pylibkriging,rmse,18.9819 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/gauss/interactive,pylibkriging,q2,0.5348 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/gauss/interactive,pylibkriging,nlpd,35.2765 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/gauss/quadratic,pylibkriging,fit_s,0.00248289 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/gauss/quadratic,pylibkriging,rmse,10.9058 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/gauss/quadratic,pylibkriging,q2,0.846441 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/gauss/quadratic,pylibkriging,nlpd,4.92017 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/exp/constant,pylibkriging,fit_s,0.00100708 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/exp/constant,pylibkriging,rmse,16.9653 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/exp/constant,pylibkriging,q2,0.628393 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/exp/constant,pylibkriging,nlpd,4.48595 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/exp/linear,pylibkriging,fit_s,0.00118208 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/exp/linear,pylibkriging,rmse,17.1781 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/exp/linear,pylibkriging,q2,0.619012 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/exp/linear,pylibkriging,nlpd,4.47061 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/exp/interactive,pylibkriging,fit_s,0.00123763 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/exp/interactive,pylibkriging,rmse,17.1856 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/exp/interactive,pylibkriging,q2,0.618679 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/exp/interactive,pylibkriging,nlpd,4.43133 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/exp/quadratic,pylibkriging,fit_s,0.0013876 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/exp/quadratic,pylibkriging,rmse,7.99484 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/exp/quadratic,pylibkriging,q2,0.917476 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/exp/quadratic,pylibkriging,nlpd,3.35612 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern3_2/constant,pylibkriging,fit_s,0.00175476 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern3_2/constant,pylibkriging,rmse,18.4958 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern3_2/constant,pylibkriging,q2,0.558321 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern3_2/constant,pylibkriging,nlpd,18.8349 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern3_2/linear,pylibkriging,fit_s,0.00206542 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern3_2/linear,pylibkriging,rmse,18.5562 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern3_2/linear,pylibkriging,q2,0.555433 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern3_2/linear,pylibkriging,nlpd,14.9494 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern3_2/interactive,pylibkriging,fit_s,0.00178266 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern3_2/interactive,pylibkriging,rmse,18.561 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern3_2/interactive,pylibkriging,q2,0.555202 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern3_2/interactive,pylibkriging,nlpd,12.9923 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern3_2/quadratic,pylibkriging,fit_s,0.00338221 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern3_2/quadratic,pylibkriging,rmse,8.91645 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern3_2/quadratic,pylibkriging,q2,0.897353 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern3_2/quadratic,pylibkriging,nlpd,3.94939 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern5_2/constant,pylibkriging,fit_s,0.00228882 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern5_2/constant,pylibkriging,rmse,18.3236 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern5_2/constant,pylibkriging,q2,0.566507 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern5_2/constant,pylibkriging,nlpd,18.4322 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern5_2/constant,pylibkriging BFGS10,rmse,1.36208 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern5_2/constant,pylibkriging BFGS10,q2,0.997605 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern5_2/constant,pylibkriging BFGS10,nlpd,1.03215 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern5_2/linear,pylibkriging,fit_s,0.00242186 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern5_2/linear,pylibkriging,rmse,18.7127 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern5_2/linear,pylibkriging,q2,0.547901 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern5_2/linear,pylibkriging,nlpd,24.5771 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern5_2/interactive,pylibkriging,fit_s,0.00234485 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern5_2/interactive,pylibkriging,rmse,18.7194 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern5_2/interactive,pylibkriging,q2,0.547577 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern5_2/interactive,pylibkriging,nlpd,20.4255 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern5_2/quadratic,pylibkriging,fit_s,0.00251341 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern5_2/quadratic,pylibkriging,rmse,10.2878 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern5_2/quadratic,pylibkriging,q2,0.863351 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern5_2/quadratic,pylibkriging,nlpd,4.46344 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/gauss/constant,pylibkriging,fit_s,0.00588727 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/gauss/constant,pylibkriging,rmse,0.295934 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/gauss/constant,pylibkriging,q2,0.920537 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/gauss/constant,pylibkriging,nlpd,1.58666 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/gauss/linear,pylibkriging,fit_s,0.00581551 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/gauss/linear,pylibkriging,rmse,0.296355 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/gauss/linear,pylibkriging,q2,0.92031 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/gauss/linear,pylibkriging,nlpd,1.2962 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/gauss/interactive,pylibkriging,fit_s,0.00644922 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/gauss/interactive,pylibkriging,rmse,0.285136 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/gauss/interactive,pylibkriging,q2,0.926229 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/gauss/interactive,pylibkriging,nlpd,1.04122 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/gauss/quadratic,pylibkriging,fit_s,0.00578475 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/gauss/quadratic,pylibkriging,rmse,0.237308 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/gauss/quadratic,pylibkriging,q2,0.948902 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/gauss/quadratic,pylibkriging,nlpd,0.146484 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/exp/constant,pylibkriging,fit_s,0.00258231 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/exp/constant,pylibkriging,rmse,0.334876 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/exp/constant,pylibkriging,q2,0.898247 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/exp/constant,pylibkriging,nlpd,0.339235 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/exp/linear,pylibkriging,fit_s,0.00211 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/exp/linear,pylibkriging,rmse,0.333447 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/exp/linear,pylibkriging,q2,0.899114 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/exp/linear,pylibkriging,nlpd,0.336515 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/exp/interactive,pylibkriging,fit_s,0.0025239 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/exp/interactive,pylibkriging,rmse,0.336819 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/exp/interactive,pylibkriging,q2,0.897063 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/exp/interactive,pylibkriging,nlpd,0.329566 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/exp/quadratic,pylibkriging,fit_s,0.00309515 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/exp/quadratic,pylibkriging,rmse,0.327816 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/exp/quadratic,pylibkriging,q2,0.902492 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/exp/quadratic,pylibkriging,nlpd,0.322056 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern3_2/constant,pylibkriging,fit_s,0.00425816 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern3_2/constant,pylibkriging,rmse,0.684362 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern3_2/constant,pylibkriging,q2,0.575038 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern3_2/constant,pylibkriging,nlpd,0.972803 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern3_2/linear,pylibkriging,fit_s,0.00413442 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern3_2/linear,pylibkriging,rmse,0.650201 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern3_2/linear,pylibkriging,q2,0.616404 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern3_2/linear,pylibkriging,nlpd,0.921455 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern3_2/interactive,pylibkriging,fit_s,0.00704145 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern3_2/interactive,pylibkriging,rmse,0.274326 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern3_2/interactive,pylibkriging,q2,0.931717 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern3_2/interactive,pylibkriging,nlpd,0.0549679 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern3_2/quadratic,pylibkriging,fit_s,0.00996184 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern3_2/quadratic,pylibkriging,rmse,0.263745 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern3_2/quadratic,pylibkriging,q2,0.936883 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern3_2/quadratic,pylibkriging,nlpd,0.0446267 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern5_2/constant,pylibkriging,fit_s,0.00410438 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern5_2/constant,pylibkriging,rmse,0.674391 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern5_2/constant,pylibkriging,q2,0.58733 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern5_2/constant,pylibkriging,nlpd,0.944765 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern5_2/constant,pylibkriging BFGS10,rmse,0.268216 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern5_2/constant,pylibkriging BFGS10,q2,0.934725 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern5_2/constant,pylibkriging BFGS10,nlpd,0.169585 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern5_2/linear,pylibkriging,fit_s,0.00728607 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern5_2/linear,pylibkriging,rmse,0.268808 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern5_2/linear,pylibkriging,q2,0.934436 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern5_2/linear,pylibkriging,nlpd,0.142382 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern5_2/interactive,pylibkriging,fit_s,0.00524259 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern5_2/interactive,pylibkriging,rmse,0.271088 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern5_2/interactive,pylibkriging,q2,0.933319 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern5_2/interactive,pylibkriging,nlpd,0.128788 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern5_2/quadratic,pylibkriging,fit_s,0.011641 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern5_2/quadratic,pylibkriging,rmse,0.253093 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern5_2/quadratic,pylibkriging,q2,0.941878 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern5_2/quadratic,pylibkriging,nlpd,0.025469 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/gauss/constant,pylibkriging,fit_s,0.00113583 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/gauss/constant,pylibkriging,rmse,0.026127 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/gauss/constant,pylibkriging,q2,0.999737 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/gauss/constant,pylibkriging,nlpd,-2.69817 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/gauss/linear,pylibkriging,fit_s,0.00152946 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/gauss/linear,pylibkriging,rmse,1.06243 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/gauss/linear,pylibkriging,q2,0.564691 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/gauss/linear,pylibkriging,nlpd,2.7394 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/gauss/interactive,pylibkriging,fit_s,0.00153446 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/gauss/interactive,pylibkriging,rmse,1.06504 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/gauss/interactive,pylibkriging,q2,0.562551 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/gauss/interactive,pylibkriging,nlpd,2.33575 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/gauss/quadratic,pylibkriging,fit_s,0.00196862 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/gauss/quadratic,pylibkriging,rmse,0.281816 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/gauss/quadratic,pylibkriging,q2,0.969371 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/gauss/quadratic,pylibkriging,nlpd,0.71682 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/exp/constant,pylibkriging,fit_s,0.000828028 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/exp/constant,pylibkriging,rmse,0.249023 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/exp/constant,pylibkriging,q2,0.976085 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/exp/constant,pylibkriging,nlpd,0.394159 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/exp/linear,pylibkriging,fit_s,0.000962973 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/exp/linear,pylibkriging,rmse,0.245558 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/exp/linear,pylibkriging,q2,0.976746 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/exp/linear,pylibkriging,nlpd,0.224593 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/exp/interactive,pylibkriging,fit_s,0.00143671 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/exp/interactive,pylibkriging,rmse,0.239032 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/exp/interactive,pylibkriging,q2,0.977965 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/exp/interactive,pylibkriging,nlpd,0.226133 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/exp/quadratic,pylibkriging,fit_s,0.00274634 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/exp/quadratic,pylibkriging,rmse,0.237713 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/exp/quadratic,pylibkriging,q2,0.978208 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/exp/quadratic,pylibkriging,nlpd,-0.139582 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern3_2/constant,pylibkriging,fit_s,0.0011251 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern3_2/constant,pylibkriging,rmse,0.0600911 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern3_2/constant,pylibkriging,q2,0.998607 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern3_2/constant,pylibkriging,nlpd,-1.34658 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern3_2/linear,pylibkriging,fit_s,0.000877857 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern3_2/linear,pylibkriging,rmse,0.0584376 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern3_2/linear,pylibkriging,q2,0.998683 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern3_2/linear,pylibkriging,nlpd,-1.34912 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern3_2/interactive,pylibkriging,fit_s,0.00140738 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern3_2/interactive,pylibkriging,rmse,0.0576096 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern3_2/interactive,pylibkriging,q2,0.99872 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern3_2/interactive,pylibkriging,nlpd,-1.35376 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern3_2/quadratic,pylibkriging,fit_s,0.00383472 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern3_2/quadratic,pylibkriging,rmse,0.215336 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern3_2/quadratic,pylibkriging,q2,0.982117 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern3_2/quadratic,pylibkriging,nlpd,-0.350837 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern5_2/constant,pylibkriging,fit_s,0.000718832 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern5_2/constant,pylibkriging,rmse,0.0246348 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern5_2/constant,pylibkriging,q2,0.999766 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern5_2/constant,pylibkriging,nlpd,-2.59598 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern5_2/constant,pylibkriging BFGS10,rmse,0.0246338 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern5_2/constant,pylibkriging BFGS10,q2,0.999766 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern5_2/constant,pylibkriging BFGS10,nlpd,-2.59598 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern5_2/linear,pylibkriging,fit_s,0.0013113 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern5_2/linear,pylibkriging,rmse,0.0232483 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern5_2/linear,pylibkriging,q2,0.999792 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern5_2/linear,pylibkriging,nlpd,-2.58843 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern5_2/interactive,pylibkriging,fit_s,0.00150442 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern5_2/interactive,pylibkriging,rmse,0.0230248 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern5_2/interactive,pylibkriging,q2,0.999796 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern5_2/interactive,pylibkriging,nlpd,-2.58839 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern5_2/quadratic,pylibkriging,fit_s,0.00300908 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern5_2/quadratic,pylibkriging,rmse,0.209802 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern5_2/quadratic,pylibkriging,q2,0.983025 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern5_2/quadratic,pylibkriging,nlpd,-0.403998 +python_Kriging_kernels-trends-interpolation,python,ishigami/gauss/constant,pylibkriging,fit_s,0.0101857 +python_Kriging_kernels-trends-interpolation,python,ishigami/gauss/constant,pylibkriging,rmse,2.07486 +python_Kriging_kernels-trends-interpolation,python,ishigami/gauss/constant,pylibkriging,q2,0.668191 +python_Kriging_kernels-trends-interpolation,python,ishigami/gauss/constant,pylibkriging,nlpd,2.80326 +python_Kriging_kernels-trends-interpolation,python,ishigami/gauss/linear,pylibkriging,fit_s,0.0193923 +python_Kriging_kernels-trends-interpolation,python,ishigami/gauss/linear,pylibkriging,rmse,2.8459 +python_Kriging_kernels-trends-interpolation,python,ishigami/gauss/linear,pylibkriging,q2,0.375762 +python_Kriging_kernels-trends-interpolation,python,ishigami/gauss/linear,pylibkriging,nlpd,3.41919 +python_Kriging_kernels-trends-interpolation,python,ishigami/gauss/interactive,pylibkriging,fit_s,0.0174267 +python_Kriging_kernels-trends-interpolation,python,ishigami/gauss/interactive,pylibkriging,rmse,2.85366 +python_Kriging_kernels-trends-interpolation,python,ishigami/gauss/interactive,pylibkriging,q2,0.372353 +python_Kriging_kernels-trends-interpolation,python,ishigami/gauss/interactive,pylibkriging,nlpd,3.28643 +python_Kriging_kernels-trends-interpolation,python,ishigami/gauss/quadratic,pylibkriging,fit_s,0.00865173 +python_Kriging_kernels-trends-interpolation,python,ishigami/gauss/quadratic,pylibkriging,rmse,2.08981 +python_Kriging_kernels-trends-interpolation,python,ishigami/gauss/quadratic,pylibkriging,q2,0.663392 +python_Kriging_kernels-trends-interpolation,python,ishigami/gauss/quadratic,pylibkriging,nlpd,2.80097 +python_Kriging_kernels-trends-interpolation,python,ishigami/exp/constant,pylibkriging,fit_s,0.00899458 +python_Kriging_kernels-trends-interpolation,python,ishigami/exp/constant,pylibkriging,rmse,1.41632 +python_Kriging_kernels-trends-interpolation,python,ishigami/exp/constant,pylibkriging,q2,0.845391 +python_Kriging_kernels-trends-interpolation,python,ishigami/exp/constant,pylibkriging,nlpd,1.75239 +python_Kriging_kernels-trends-interpolation,python,ishigami/exp/linear,pylibkriging,fit_s,0.0115979 +python_Kriging_kernels-trends-interpolation,python,ishigami/exp/linear,pylibkriging,rmse,1.4095 +python_Kriging_kernels-trends-interpolation,python,ishigami/exp/linear,pylibkriging,q2,0.846877 +python_Kriging_kernels-trends-interpolation,python,ishigami/exp/linear,pylibkriging,nlpd,1.74779 +python_Kriging_kernels-trends-interpolation,python,ishigami/exp/interactive,pylibkriging,fit_s,0.00745058 +python_Kriging_kernels-trends-interpolation,python,ishigami/exp/interactive,pylibkriging,rmse,1.41511 +python_Kriging_kernels-trends-interpolation,python,ishigami/exp/interactive,pylibkriging,q2,0.845657 +python_Kriging_kernels-trends-interpolation,python,ishigami/exp/interactive,pylibkriging,nlpd,1.75024 +python_Kriging_kernels-trends-interpolation,python,ishigami/exp/quadratic,pylibkriging,fit_s,0.00843048 +python_Kriging_kernels-trends-interpolation,python,ishigami/exp/quadratic,pylibkriging,rmse,1.40329 +python_Kriging_kernels-trends-interpolation,python,ishigami/exp/quadratic,pylibkriging,q2,0.848223 +python_Kriging_kernels-trends-interpolation,python,ishigami/exp/quadratic,pylibkriging,nlpd,1.73254 +python_Kriging_kernels-trends-interpolation,python,ishigami/matern3_2/constant,pylibkriging,fit_s,0.0480943 +python_Kriging_kernels-trends-interpolation,python,ishigami/matern3_2/constant,pylibkriging,rmse,1.32971 +python_Kriging_kernels-trends-interpolation,python,ishigami/matern3_2/constant,pylibkriging,q2,0.863723 +python_Kriging_kernels-trends-interpolation,python,ishigami/matern3_2/constant,pylibkriging,nlpd,1.50265 +python_Kriging_kernels-trends-interpolation,python,ishigami/matern3_2/linear,pylibkriging,fit_s,0.047653 +python_Kriging_kernels-trends-interpolation,python,ishigami/matern3_2/linear,pylibkriging,rmse,1.30244 +python_Kriging_kernels-trends-interpolation,python,ishigami/matern3_2/linear,pylibkriging,q2,0.869255 +python_Kriging_kernels-trends-interpolation,python,ishigami/matern3_2/linear,pylibkriging,nlpd,1.47906 +python_Kriging_kernels-trends-interpolation,python,ishigami/matern3_2/interactive,pylibkriging,fit_s,0.060349 +python_Kriging_kernels-trends-interpolation,python,ishigami/matern3_2/interactive,pylibkriging,rmse,1.30591 +python_Kriging_kernels-trends-interpolation,python,ishigami/matern3_2/interactive,pylibkriging,q2,0.868558 +python_Kriging_kernels-trends-interpolation,python,ishigami/matern3_2/interactive,pylibkriging,nlpd,1.47384 +python_Kriging_kernels-trends-interpolation,python,ishigami/matern3_2/quadratic,pylibkriging,fit_s,0.0471144 +python_Kriging_kernels-trends-interpolation,python,ishigami/matern3_2/quadratic,pylibkriging,rmse,1.32197 +python_Kriging_kernels-trends-interpolation,python,ishigami/matern3_2/quadratic,pylibkriging,q2,0.865305 +python_Kriging_kernels-trends-interpolation,python,ishigami/matern3_2/quadratic,pylibkriging,nlpd,1.46827 +python_Kriging_kernels-trends-interpolation,python,ishigami/matern5_2/constant,pylibkriging,fit_s,0.0145204 +python_Kriging_kernels-trends-interpolation,python,ishigami/matern5_2/constant,pylibkriging,rmse,3.00566 +python_Kriging_kernels-trends-interpolation,python,ishigami/matern5_2/constant,pylibkriging,q2,0.303709 +python_Kriging_kernels-trends-interpolation,python,ishigami/matern5_2/constant,pylibkriging,nlpd,3.42085 +python_Kriging_kernels-trends-interpolation,python,ishigami/matern5_2/constant,pylibkriging BFGS10,rmse,1.33903 +python_Kriging_kernels-trends-interpolation,python,ishigami/matern5_2/constant,pylibkriging BFGS10,q2,0.861806 +python_Kriging_kernels-trends-interpolation,python,ishigami/matern5_2/constant,pylibkriging BFGS10,nlpd,1.66132 +python_Kriging_kernels-trends-interpolation,python,ishigami/matern5_2/linear,pylibkriging,fit_s,0.0518982 +python_Kriging_kernels-trends-interpolation,python,ishigami/matern5_2/linear,pylibkriging,rmse,1.3434 +python_Kriging_kernels-trends-interpolation,python,ishigami/matern5_2/linear,pylibkriging,q2,0.860901 +python_Kriging_kernels-trends-interpolation,python,ishigami/matern5_2/linear,pylibkriging,nlpd,1.65103 +python_Kriging_kernels-trends-interpolation,python,ishigami/matern5_2/interactive,pylibkriging,fit_s,0.0539804 +python_Kriging_kernels-trends-interpolation,python,ishigami/matern5_2/interactive,pylibkriging,rmse,1.34479 +python_Kriging_kernels-trends-interpolation,python,ishigami/matern5_2/interactive,pylibkriging,q2,0.860615 +python_Kriging_kernels-trends-interpolation,python,ishigami/matern5_2/interactive,pylibkriging,nlpd,1.64153 +python_Kriging_kernels-trends-interpolation,python,ishigami/matern5_2/quadratic,pylibkriging,fit_s,0.052948 +python_Kriging_kernels-trends-interpolation,python,ishigami/matern5_2/quadratic,pylibkriging,rmse,1.37082 +python_Kriging_kernels-trends-interpolation,python,ishigami/matern5_2/quadratic,pylibkriging,q2,0.855166 +python_Kriging_kernels-trends-interpolation,python,ishigami/matern5_2/quadratic,pylibkriging,nlpd,1.62228 +python_Kriging_kernels-trends-interpolation,python,hartmann3/gauss/constant,pylibkriging,fit_s,0.00805306 +python_Kriging_kernels-trends-interpolation,python,hartmann3/gauss/constant,pylibkriging,rmse,0.156226 +python_Kriging_kernels-trends-interpolation,python,hartmann3/gauss/constant,pylibkriging,q2,0.971116 +python_Kriging_kernels-trends-interpolation,python,hartmann3/gauss/constant,pylibkriging,nlpd,-0.555623 +python_Kriging_kernels-trends-interpolation,python,hartmann3/gauss/linear,pylibkriging,fit_s,0.0129044 +python_Kriging_kernels-trends-interpolation,python,hartmann3/gauss/linear,pylibkriging,rmse,0.153297 +python_Kriging_kernels-trends-interpolation,python,hartmann3/gauss/linear,pylibkriging,q2,0.972189 +python_Kriging_kernels-trends-interpolation,python,hartmann3/gauss/linear,pylibkriging,nlpd,-0.568404 +python_Kriging_kernels-trends-interpolation,python,hartmann3/gauss/interactive,pylibkriging,fit_s,0.00773382 +python_Kriging_kernels-trends-interpolation,python,hartmann3/gauss/interactive,pylibkriging,rmse,0.712948 +python_Kriging_kernels-trends-interpolation,python,hartmann3/gauss/interactive,pylibkriging,q2,0.398455 +python_Kriging_kernels-trends-interpolation,python,hartmann3/gauss/interactive,pylibkriging,nlpd,1.74554 +python_Kriging_kernels-trends-interpolation,python,hartmann3/gauss/quadratic,pylibkriging,fit_s,0.00593877 +python_Kriging_kernels-trends-interpolation,python,hartmann3/gauss/quadratic,pylibkriging,rmse,0.73238 +python_Kriging_kernels-trends-interpolation,python,hartmann3/gauss/quadratic,pylibkriging,q2,0.365217 +python_Kriging_kernels-trends-interpolation,python,hartmann3/gauss/quadratic,pylibkriging,nlpd,2.14587 +python_Kriging_kernels-trends-interpolation,python,hartmann3/exp/constant,pylibkriging,fit_s,0.00374722 +python_Kriging_kernels-trends-interpolation,python,hartmann3/exp/constant,pylibkriging,rmse,0.347348 +python_Kriging_kernels-trends-interpolation,python,hartmann3/exp/constant,pylibkriging,q2,0.857215 +python_Kriging_kernels-trends-interpolation,python,hartmann3/exp/constant,pylibkriging,nlpd,0.352683 +python_Kriging_kernels-trends-interpolation,python,hartmann3/exp/linear,pylibkriging,fit_s,0.00216174 +python_Kriging_kernels-trends-interpolation,python,hartmann3/exp/linear,pylibkriging,rmse,0.347273 +python_Kriging_kernels-trends-interpolation,python,hartmann3/exp/linear,pylibkriging,q2,0.857276 +python_Kriging_kernels-trends-interpolation,python,hartmann3/exp/linear,pylibkriging,nlpd,0.343631 +python_Kriging_kernels-trends-interpolation,python,hartmann3/exp/interactive,pylibkriging,fit_s,0.0024302 +python_Kriging_kernels-trends-interpolation,python,hartmann3/exp/interactive,pylibkriging,rmse,0.378349 +python_Kriging_kernels-trends-interpolation,python,hartmann3/exp/interactive,pylibkriging,q2,0.83059 +python_Kriging_kernels-trends-interpolation,python,hartmann3/exp/interactive,pylibkriging,nlpd,0.385842 +python_Kriging_kernels-trends-interpolation,python,hartmann3/exp/quadratic,pylibkriging,fit_s,0.0039556 +python_Kriging_kernels-trends-interpolation,python,hartmann3/exp/quadratic,pylibkriging,rmse,0.355119 +python_Kriging_kernels-trends-interpolation,python,hartmann3/exp/quadratic,pylibkriging,q2,0.850755 +python_Kriging_kernels-trends-interpolation,python,hartmann3/exp/quadratic,pylibkriging,nlpd,0.334371 +python_Kriging_kernels-trends-interpolation,python,hartmann3/matern3_2/constant,pylibkriging,fit_s,0.0110524 +python_Kriging_kernels-trends-interpolation,python,hartmann3/matern3_2/constant,pylibkriging,rmse,0.227521 +python_Kriging_kernels-trends-interpolation,python,hartmann3/matern3_2/constant,pylibkriging,q2,0.938737 +python_Kriging_kernels-trends-interpolation,python,hartmann3/matern3_2/constant,pylibkriging,nlpd,-0.181207 +python_Kriging_kernels-trends-interpolation,python,hartmann3/matern3_2/linear,pylibkriging,fit_s,0.00763154 +python_Kriging_kernels-trends-interpolation,python,hartmann3/matern3_2/linear,pylibkriging,rmse,0.23038 +python_Kriging_kernels-trends-interpolation,python,hartmann3/matern3_2/linear,pylibkriging,q2,0.937188 +python_Kriging_kernels-trends-interpolation,python,hartmann3/matern3_2/linear,pylibkriging,nlpd,-0.162203 +python_Kriging_kernels-trends-interpolation,python,hartmann3/matern3_2/interactive,pylibkriging,fit_s,0.00412679 +python_Kriging_kernels-trends-interpolation,python,hartmann3/matern3_2/interactive,pylibkriging,rmse,0.315752 +python_Kriging_kernels-trends-interpolation,python,hartmann3/matern3_2/interactive,pylibkriging,q2,0.88201 +python_Kriging_kernels-trends-interpolation,python,hartmann3/matern3_2/interactive,pylibkriging,nlpd,0.898253 +python_Kriging_kernels-trends-interpolation,python,hartmann3/matern3_2/quadratic,pylibkriging,fit_s,0.00620675 +python_Kriging_kernels-trends-interpolation,python,hartmann3/matern3_2/quadratic,pylibkriging,rmse,0.291393 +python_Kriging_kernels-trends-interpolation,python,hartmann3/matern3_2/quadratic,pylibkriging,q2,0.899513 +python_Kriging_kernels-trends-interpolation,python,hartmann3/matern3_2/quadratic,pylibkriging,nlpd,0.53743 +python_Kriging_kernels-trends-interpolation,python,hartmann3/matern5_2/constant,pylibkriging,fit_s,0.00997186 +python_Kriging_kernels-trends-interpolation,python,hartmann3/matern5_2/constant,pylibkriging,rmse,0.190287 +python_Kriging_kernels-trends-interpolation,python,hartmann3/matern5_2/constant,pylibkriging,q2,0.957148 +python_Kriging_kernels-trends-interpolation,python,hartmann3/matern5_2/constant,pylibkriging,nlpd,-0.403054 +python_Kriging_kernels-trends-interpolation,python,hartmann3/matern5_2/constant,pylibkriging BFGS10,rmse,0.190286 +python_Kriging_kernels-trends-interpolation,python,hartmann3/matern5_2/constant,pylibkriging BFGS10,q2,0.957148 +python_Kriging_kernels-trends-interpolation,python,hartmann3/matern5_2/constant,pylibkriging BFGS10,nlpd,-0.40306 +python_Kriging_kernels-trends-interpolation,python,hartmann3/matern5_2/linear,pylibkriging,fit_s,0.0127335 +python_Kriging_kernels-trends-interpolation,python,hartmann3/matern5_2/linear,pylibkriging,rmse,0.189633 +python_Kriging_kernels-trends-interpolation,python,hartmann3/matern5_2/linear,pylibkriging,q2,0.957442 +python_Kriging_kernels-trends-interpolation,python,hartmann3/matern5_2/linear,pylibkriging,nlpd,-0.410075 +python_Kriging_kernels-trends-interpolation,python,hartmann3/matern5_2/interactive,pylibkriging,fit_s,0.0101018 +python_Kriging_kernels-trends-interpolation,python,hartmann3/matern5_2/interactive,pylibkriging,rmse,0.307114 +python_Kriging_kernels-trends-interpolation,python,hartmann3/matern5_2/interactive,pylibkriging,q2,0.888377 +python_Kriging_kernels-trends-interpolation,python,hartmann3/matern5_2/interactive,pylibkriging,nlpd,1.50063 +python_Kriging_kernels-trends-interpolation,python,hartmann3/matern5_2/quadratic,pylibkriging,fit_s,0.0101326 +python_Kriging_kernels-trends-interpolation,python,hartmann3/matern5_2/quadratic,pylibkriging,rmse,0.285355 +python_Kriging_kernels-trends-interpolation,python,hartmann3/matern5_2/quadratic,pylibkriging,q2,0.903634 +python_Kriging_kernels-trends-interpolation,python,hartmann3/matern5_2/quadratic,pylibkriging,nlpd,0.898128 +python_Kriging_objectives-LL-LOO-LMP-gradients,python,branin/LL,pylibkriging,rmse,0.833715 +python_Kriging_objectives-LL-LOO-LMP-gradients,python,branin/LL,pylibkriging,q2,0.999729 +python_Kriging_objectives-LL-LOO-LMP-gradients,python,branin/LL,pylibkriging,nlpd,0.697034 +python_Kriging_objectives-LL-LOO-LMP-gradients,python,branin/LOO,pylibkriging,rmse,0.550995 +python_Kriging_objectives-LL-LOO-LMP-gradients,python,branin/LOO,pylibkriging,q2,0.999882 +python_Kriging_objectives-LL-LOO-LMP-gradients,python,branin/LOO,pylibkriging,nlpd,0.471613 +python_Kriging_objectives-LL-LOO-LMP-gradients,python,branin/LMP,pylibkriging,rmse,0.66967 +python_Kriging_objectives-LL-LOO-LMP-gradients,python,branin/LMP,pylibkriging,q2,0.999825 +python_Kriging_objectives-LL-LOO-LMP-gradients,python,branin/LMP,pylibkriging,nlpd,0.617527 +python_Kriging_objectives-LL-LOO-LMP-gradients,python,hartmann3/LL,pylibkriging,rmse,0.157276 +python_Kriging_objectives-LL-LOO-LMP-gradients,python,hartmann3/LL,pylibkriging,q2,0.970945 +python_Kriging_objectives-LL-LOO-LMP-gradients,python,hartmann3/LL,pylibkriging,nlpd,-0.497548 +python_Kriging_objectives-LL-LOO-LMP-gradients,python,hartmann3/LOO,pylibkriging,rmse,0.144982 +python_Kriging_objectives-LL-LOO-LMP-gradients,python,hartmann3/LOO,pylibkriging,q2,0.97531 +python_Kriging_objectives-LL-LOO-LMP-gradients,python,hartmann3/LOO,pylibkriging,nlpd,-0.584885 +python_Kriging_objectives-LL-LOO-LMP-gradients,python,hartmann3/LMP,pylibkriging,rmse,0.150075 +python_Kriging_objectives-LL-LOO-LMP-gradients,python,hartmann3/LMP,pylibkriging,q2,0.973544 +python_Kriging_objectives-LL-LOO-LMP-gradients,python,hartmann3/LMP,pylibkriging,nlpd,-0.528379 +python_Kriging_objectives-LL-LOO-LMP-gradients,python,ishigami/LL,pylibkriging,rmse,3.41501 +python_Kriging_objectives-LL-LOO-LMP-gradients,python,ishigami/LL,pylibkriging,q2,0.203357 +python_Kriging_objectives-LL-LOO-LMP-gradients,python,ishigami/LL,pylibkriging,nlpd,4.86258 +python_Kriging_objectives-LL-LOO-LMP-gradients,python,ishigami/LOO,pylibkriging,rmse,2.45853 +python_Kriging_objectives-LL-LOO-LMP-gradients,python,ishigami/LOO,pylibkriging,q2,0.587112 +python_Kriging_objectives-LL-LOO-LMP-gradients,python,ishigami/LOO,pylibkriging,nlpd,2.72077 +python_Kriging_objectives-LL-LOO-LMP-gradients,python,ishigami/LMP,pylibkriging,rmse,3.47385 +python_Kriging_objectives-LL-LOO-LMP-gradients,python,ishigami/LMP,pylibkriging,q2,0.175667 +python_Kriging_objectives-LL-LOO-LMP-gradients,python,ishigami/LMP,pylibkriging,nlpd,5.27033 +python_Kriging_objectives-LL-LOO-LMP-gradients,python,friedman/LL,pylibkriging,rmse,0.273164 +python_Kriging_objectives-LL-LOO-LMP-gradients,python,friedman/LL,pylibkriging,q2,0.996855 +python_Kriging_objectives-LL-LOO-LMP-gradients,python,friedman/LL,pylibkriging,nlpd,-0.118408 +python_Kriging_objectives-LL-LOO-LMP-gradients,python,friedman/LOO,pylibkriging,rmse,0.233921 +python_Kriging_objectives-LL-LOO-LMP-gradients,python,friedman/LOO,pylibkriging,q2,0.997694 +python_Kriging_objectives-LL-LOO-LMP-gradients,python,friedman/LOO,pylibkriging,nlpd,-0.169369 +python_Kriging_objectives-LL-LOO-LMP-gradients,python,friedman/LMP,pylibkriging,rmse,0.269913 +python_Kriging_objectives-LL-LOO-LMP-gradients,python,friedman/LMP,pylibkriging,q2,0.996929 +python_Kriging_objectives-LL-LOO-LMP-gradients,python,friedman/LMP,pylibkriging,nlpd,-0.122709 +python_Kriging-nugget-noise_noisy-data,python,forrester,nugget,rmse,0.198091 +python_Kriging-nugget-noise_noisy-data,python,forrester,nugget,q2,0.998087 +python_Kriging-nugget-noise_noisy-data,python,forrester,nugget,nlpd,0.286631 +python_Kriging-nugget-noise_noisy-data,python,forrester,noise-free fit,rmse,0.487916 +python_Kriging-nugget-noise_noisy-data,python,forrester,noise-free fit,q2,0.988392 +python_Kriging-nugget-noise_noisy-data,python,forrester,noise-free fit,nlpd,862.67 +python_Kriging-nugget-noise_noisy-data,python,friedman,nugget,rmse,0.467335 +python_Kriging-nugget-noise_noisy-data,python,friedman,nugget,q2,0.99089 +python_Kriging-nugget-noise_noisy-data,python,friedman,nugget,nlpd,1.07073 +python_Kriging-nugget-noise_noisy-data,python,friedman,noise-free fit,rmse,1.27157 +python_Kriging-nugget-noise_noisy-data,python,friedman,noise-free fit,q2,0.932558 +python_Kriging-nugget-noise_noisy-data,python,friedman,noise-free fit,nlpd,1.64497 +python_Kriging-nugget-noise_noisy-data,python,gramacy_lee_2012/hetero,known variances,rmse,0.168741 +python_Kriging-nugget-noise_noisy-data,python,gramacy_lee_2012/hetero,known variances,q2,0.982515 +python_Kriging-nugget-noise_noisy-data,python,gramacy_lee_2012/hetero,known variances,nlpd,-0.570574 +python_Kriging-nugget-noise_noisy-data,python,gramacy_lee_2012/hetero,nugget,rmse,0.170945 +python_Kriging-nugget-noise_noisy-data,python,gramacy_lee_2012/hetero,nugget,q2,0.982055 +python_Kriging-nugget-noise_noisy-data,python,gramacy_lee_2012/hetero,nugget,nlpd,-0.221568 +python_WarpKriging_warpings-categorical-ordinal,python,gl2012,Kriging,rmse,0.167263 +python_WarpKriging_warpings-categorical-ordinal,python,gl2012,Kriging,q2,0.981176 +python_WarpKriging_warpings-categorical-ordinal,python,gl2012,Kriging,nlpd,-1.08917 +python_WarpKriging_warpings-categorical-ordinal,python,gl2012,affine,rmse,0.167263 +python_WarpKriging_warpings-categorical-ordinal,python,gl2012,affine,q2,0.981176 +python_WarpKriging_warpings-categorical-ordinal,python,gl2012,affine,nlpd,-1.08917 +python_WarpKriging_warpings-categorical-ordinal,python,gl2012,boxcox,rmse,0.16759 +python_WarpKriging_warpings-categorical-ordinal,python,gl2012,boxcox,q2,0.981103 +python_WarpKriging_warpings-categorical-ordinal,python,gl2012,boxcox,nlpd,282.422 +python_WarpKriging_warpings-categorical-ordinal,python,gl2012,kumaraswamy,rmse,0.168286 +python_WarpKriging_warpings-categorical-ordinal,python,gl2012,kumaraswamy,q2,0.980945 +python_WarpKriging_warpings-categorical-ordinal,python,gl2012,kumaraswamy,nlpd,2.04577e+06 +python_WarpKriging_warpings-categorical-ordinal,python,gl2012,knots(5),rmse,0.16791 +python_WarpKriging_warpings-categorical-ordinal,python,gl2012,knots(5),q2,0.98103 +python_WarpKriging_warpings-categorical-ordinal,python,gl2012,knots(5),nlpd,2.04376e+06 +python_WarpKriging_warpings-categorical-ordinal,python,gl2012,neural_mono(8),rmse,0.170824 +python_WarpKriging_warpings-categorical-ordinal,python,gl2012,neural_mono(8),q2,0.980366 +python_WarpKriging_warpings-categorical-ordinal,python,gl2012,neural_mono(8),nlpd,-0.265141 +python_WarpKriging_warpings-categorical-ordinal,python,gl2008,Kriging,rmse,0.0534649 +python_WarpKriging_warpings-categorical-ordinal,python,gl2008,Kriging,q2,0.585636 +python_WarpKriging_warpings-categorical-ordinal,python,gl2008,Kriging,nlpd,-1.74569 +python_WarpKriging_warpings-categorical-ordinal,python,gl2008,kumaraswamy,rmse,0.0535298 +python_WarpKriging_warpings-categorical-ordinal,python,gl2008,kumaraswamy,q2,0.58463 +python_WarpKriging_warpings-categorical-ordinal,python,gl2008,kumaraswamy,nlpd,-1.74552 +python_WarpKriging_warpings-categorical-ordinal,python,mixed,Kriging (level as number),rmse,16.7693 +python_WarpKriging_warpings-categorical-ordinal,python,mixed,Kriging (level as number),q2,0.838452 +python_WarpKriging_warpings-categorical-ordinal,python,mixed,Kriging (level as number),nlpd,3.81063 +python_WarpKriging_warpings-categorical-ordinal,python,mixed,"categorical(5,2)",rmse,16.1438 +python_WarpKriging_warpings-categorical-ordinal,python,mixed,"categorical(5,2)",q2,0.85028 +python_WarpKriging_warpings-categorical-ordinal,python,mixed,"categorical(5,2)",nlpd,3.3206 +python_WarpKriging_warpings-categorical-ordinal,python,ordinal,Kriging (rank as number),rmse,0.213845 +python_WarpKriging_warpings-categorical-ordinal,python,ordinal,Kriging (rank as number),q2,0.953177 +python_WarpKriging_warpings-categorical-ordinal,python,ordinal,Kriging (rank as number),nlpd,0.810969 +python_WarpKriging_warpings-categorical-ordinal,python,ordinal,ordinal(6),rmse,0.213043 +python_WarpKriging_warpings-categorical-ordinal,python,ordinal,ordinal(6),q2,0.953528 +python_WarpKriging_warpings-categorical-ordinal,python,ordinal,ordinal(6),nlpd,0.78888 +python_Kriging_high-dimension-normalize-screening,python,otl_circuit,unit cube,fit_s,0.00789523 +python_Kriging_high-dimension-normalize-screening,python,otl_circuit,unit cube,rmse,0.0150514 +python_Kriging_high-dimension-normalize-screening,python,otl_circuit,unit cube,q2,0.999836 +python_Kriging_high-dimension-normalize-screening,python,otl_circuit,unit cube,nlpd,-2.90452 +python_Kriging_high-dimension-normalize-screening,python,otl_circuit,native + normalize,rmse,0.0150514 +python_Kriging_high-dimension-normalize-screening,python,otl_circuit,native + normalize,q2,0.999836 +python_Kriging_high-dimension-normalize-screening,python,otl_circuit,native + normalize,nlpd,-2.90452 +python_Kriging_high-dimension-normalize-screening,python,piston,unit cube,fit_s,0.0197392 +python_Kriging_high-dimension-normalize-screening,python,piston,unit cube,rmse,0.0104258 +python_Kriging_high-dimension-normalize-screening,python,piston,unit cube,q2,0.994462 +python_Kriging_high-dimension-normalize-screening,python,piston,unit cube,nlpd,-3.49038 +python_Kriging_high-dimension-normalize-screening,python,piston,native + normalize,rmse,0.0104258 +python_Kriging_high-dimension-normalize-screening,python,piston,native + normalize,q2,0.994462 +python_Kriging_high-dimension-normalize-screening,python,piston,native + normalize,nlpd,-3.49038 +python_Kriging_high-dimension-normalize-screening,python,borehole,unit cube,fit_s,0.0140877 +python_Kriging_high-dimension-normalize-screening,python,borehole,unit cube,rmse,0.893341 +python_Kriging_high-dimension-normalize-screening,python,borehole,unit cube,q2,0.999599 +python_Kriging_high-dimension-normalize-screening,python,borehole,unit cube,nlpd,1.08433 +python_Kriging_high-dimension-normalize-screening,python,borehole,native + normalize,rmse,0.893341 +python_Kriging_high-dimension-normalize-screening,python,borehole,native + normalize,q2,0.999599 +python_Kriging_high-dimension-normalize-screening,python,borehole,native + normalize,nlpd,1.08433 +python_Kriging_high-dimension-normalize-screening,python,robot_arm,unit cube,fit_s,0.167482 +python_Kriging_high-dimension-normalize-screening,python,robot_arm,unit cube,rmse,0.358507 +python_Kriging_high-dimension-normalize-screening,python,robot_arm,unit cube,q2,0.571154 +python_Kriging_high-dimension-normalize-screening,python,robot_arm,unit cube,nlpd,0.355106 +python_Kriging_high-dimension-normalize-screening,python,robot_arm,native + normalize,rmse,0.358507 +python_Kriging_high-dimension-normalize-screening,python,robot_arm,native + normalize,q2,0.571154 +python_Kriging_high-dimension-normalize-screening,python,robot_arm,native + normalize,nlpd,0.355106 +python_Kriging_high-dimension-normalize-screening,python,dette_pepelyshev,unit cube,fit_s,0.0246282 +python_Kriging_high-dimension-normalize-screening,python,dette_pepelyshev,unit cube,rmse,0.424908 +python_Kriging_high-dimension-normalize-screening,python,dette_pepelyshev,unit cube,q2,0.997757 +python_Kriging_high-dimension-normalize-screening,python,dette_pepelyshev,unit cube,nlpd,0.510276 +python_Kriging_high-dimension-normalize-screening,python,dette_pepelyshev,native + normalize,rmse,0.424908 +python_Kriging_high-dimension-normalize-screening,python,dette_pepelyshev,native + normalize,q2,0.997757 +python_Kriging_high-dimension-normalize-screening,python,dette_pepelyshev,native + normalize,nlpd,0.510276 +python_Kriging_high-dimension-normalize-screening,python,wing_weight,unit cube,fit_s,0.0245481 +python_Kriging_high-dimension-normalize-screening,python,wing_weight,unit cube,rmse,0.809743 +python_Kriging_high-dimension-normalize-screening,python,wing_weight,unit cube,q2,0.999716 +python_Kriging_high-dimension-normalize-screening,python,wing_weight,unit cube,nlpd,1.07161 +python_Kriging_high-dimension-normalize-screening,python,wing_weight,native + normalize,rmse,0.809743 +python_Kriging_high-dimension-normalize-screening,python,wing_weight,native + normalize,q2,0.999716 +python_Kriging_high-dimension-normalize-screening,python,wing_weight,native + normalize,nlpd,1.07161 +python_Kriging_high-dimension-normalize-screening,python,welch,unit cube,fit_s,0.564758 +python_Kriging_high-dimension-normalize-screening,python,welch,unit cube,rmse,0.167729 +python_Kriging_high-dimension-normalize-screening,python,welch,unit cube,q2,0.993452 +python_Kriging_high-dimension-normalize-screening,python,welch,unit cube,nlpd,-0.429138 +python_Kriging_high-dimension-normalize-screening,python,welch,native + normalize,rmse,0.167692 +python_Kriging_high-dimension-normalize-screening,python,welch,native + normalize,q2,0.993454 +python_Kriging_high-dimension-normalize-screening,python,welch,native + normalize,nlpd,-0.429291 +python_MLPKriging_deep-kernel-projected-branin,python,projected_branin_10d,Kriging,rmse,4.42284 +python_MLPKriging_deep-kernel-projected-branin,python,projected_branin_10d,Kriging,q2,0.959122 +python_MLPKriging_deep-kernel-projected-branin,python,projected_branin_10d,Kriging,nlpd,2.89997 +python_MLPKriging_deep-kernel-projected-branin,python,projected_branin_10d,MLPKriging tanh BFGS,fit_s,0.130867 +python_MLPKriging_deep-kernel-projected-branin,python,projected_branin_10d,MLPKriging tanh BFGS,rmse,57.402 +python_MLPKriging_deep-kernel-projected-branin,python,projected_branin_10d,MLPKriging tanh BFGS,q2,-5.88553 +python_MLPKriging_deep-kernel-projected-branin,python,projected_branin_10d,MLPKriging tanh BFGS,nlpd,136.771 +python_MLPKriging_deep-kernel-projected-branin,python,projected_branin_10d,MLPKriging tanh BFGS10,fit_s,0.442791 +python_MLPKriging_deep-kernel-projected-branin,python,projected_branin_10d,MLPKriging tanh BFGS10,rmse,4.28606 +python_MLPKriging_deep-kernel-projected-branin,python,projected_branin_10d,MLPKriging tanh BFGS10,q2,0.961612 +python_MLPKriging_deep-kernel-projected-branin,python,projected_branin_10d,MLPKriging tanh BFGS10,nlpd,29.0854 +python_MLPKriging_deep-kernel-projected-branin,python,projected_branin_10d,MLPKriging selu BFGS,fit_s,0.139126 +python_MLPKriging_deep-kernel-projected-branin,python,projected_branin_10d,MLPKriging selu BFGS,rmse,58.9913 +python_MLPKriging_deep-kernel-projected-branin,python,projected_branin_10d,MLPKriging selu BFGS,q2,-6.2721 +python_MLPKriging_deep-kernel-projected-branin,python,projected_branin_10d,MLPKriging selu BFGS,nlpd,46.9788 +python_MLPKriging_deep-kernel-projected-branin,python,projected_branin_10d,MLPKriging selu BFGS10,fit_s,0.440179 +python_MLPKriging_deep-kernel-projected-branin,python,projected_branin_10d,MLPKriging selu BFGS10,rmse,4.39195 +python_MLPKriging_deep-kernel-projected-branin,python,projected_branin_10d,MLPKriging selu BFGS10,q2,0.959691 +python_MLPKriging_deep-kernel-projected-branin,python,projected_branin_10d,MLPKriging selu BFGS10,nlpd,10.6053 +python_Kriging-vecchia-nystrom_large-n,python,hartmann3/n=1500,exact,fit_s,11.1133 +python_NestedKriging_aggregations-large-n,python,hartmann3/n=1500,exact,fit_s,11.1133 +python_Kriging-vecchia-nystrom_large-n,python,hartmann3/n=1500,exact,rmse,0.000259233 +python_NestedKriging_aggregations-large-n,python,hartmann3/n=1500,exact,rmse,0.000259233 +python_Kriging-vecchia-nystrom_large-n,python,hartmann3/n=1500,exact,q2,1 +python_NestedKriging_aggregations-large-n,python,hartmann3/n=1500,exact,q2,1 +python_Kriging-vecchia-nystrom_large-n,python,hartmann3/n=1500,exact,nlpd,-6.92184 +python_NestedKriging_aggregations-large-n,python,hartmann3/n=1500,exact,nlpd,-6.92184 +python_Kriging-vecchia-nystrom_large-n,python,hartmann3/n=1500,vecchia(30),fit_s,5.43768 +python_Kriging-vecchia-nystrom_large-n,python,hartmann3/n=1500,vecchia(30),rmse,0.00353225 +python_Kriging-vecchia-nystrom_large-n,python,hartmann3/n=1500,vecchia(30),q2,0.999987 +python_Kriging-vecchia-nystrom_large-n,python,hartmann3/n=1500,vecchia(30),nlpd,-4.6266 +python_Kriging-vecchia-nystrom_large-n,python,hartmann3/n=5000,vecchia(30),fit_s,16.2077 +python_Kriging-vecchia-nystrom_large-n,python,hartmann3/n=5000,vecchia(30),rmse,0.000851738 +python_Kriging-vecchia-nystrom_large-n,python,hartmann3/n=5000,vecchia(30),q2,0.999999 +python_Kriging-vecchia-nystrom_large-n,python,hartmann3/n=5000,vecchia(30),nlpd,-5.95078 +python_Kriging-vecchia-nystrom_large-n,python,hartmann3/n=1500,nystrom(100),fit_s,2.04779 +python_Kriging-vecchia-nystrom_large-n,python,hartmann3/n=1500,nystrom(100),rmse,1.51127 +python_Kriging-vecchia-nystrom_large-n,python,hartmann3/n=1500,nystrom(100),q2,-1.39237 +python_Kriging-vecchia-nystrom_large-n,python,hartmann3/n=1500,nystrom(100),nlpd,1.14197e+24 +python_Kriging-vecchia-nystrom_large-n,python,hartmann3/n=5000,nystrom(100),fit_s,3.25622 +python_Kriging-vecchia-nystrom_large-n,python,hartmann3/n=5000,nystrom(100),rmse,9.95398 +python_Kriging-vecchia-nystrom_large-n,python,hartmann3/n=5000,nystrom(100),q2,-102.786 +python_Kriging-vecchia-nystrom_large-n,python,hartmann3/n=5000,nystrom(100),nlpd,4.95409e+25 +python_NestedKriging_aggregations-large-n,python,hartmann3/n=1500,nested NK (20 groups),fit_s,0.269932 +python_NestedKriging_aggregations-large-n,python,hartmann3/n=1500,nested NK (20 groups),rmse,0.0029908 +python_NestedKriging_aggregations-large-n,python,hartmann3/n=1500,nested NK (20 groups),q2,0.999991 +python_NestedKriging_aggregations-large-n,python,hartmann3/n=1500,nested NK (20 groups),nlpd,-4.85946 +python_NestedKriging_aggregations-large-n,python,hartmann3/n=5000,nested NK (20 groups),fit_s,2.87469 +python_NestedKriging_aggregations-large-n,python,hartmann3/n=5000,nested NK (20 groups),rmse,0.000222655 +python_NestedKriging_aggregations-large-n,python,hartmann3/n=5000,nested NK (20 groups),q2,1 +python_NestedKriging_aggregations-large-n,python,hartmann3/n=5000,nested NK (20 groups),nlpd,-7.44453 +python_NestedKriging_aggregations-large-n,python,hartmann3/n=1500,nested rBCM (20 groups),fit_s,0.25767 +python_NestedKriging_aggregations-large-n,python,hartmann3/n=1500,nested rBCM (20 groups),rmse,0.00304431 +python_NestedKriging_aggregations-large-n,python,hartmann3/n=1500,nested rBCM (20 groups),q2,0.99999 +python_NestedKriging_aggregations-large-n,python,hartmann3/n=1500,nested rBCM (20 groups),nlpd,-4.50201 +python_NestedKriging_aggregations-large-n,python,hartmann3/n=5000,nested rBCM (20 groups),fit_s,2.5369 +python_NestedKriging_aggregations-large-n,python,hartmann3/n=5000,nested rBCM (20 groups),rmse,0.000224517 +python_NestedKriging_aggregations-large-n,python,hartmann3/n=5000,nested rBCM (20 groups),q2,1 +python_NestedKriging_aggregations-large-n,python,hartmann3/n=5000,nested rBCM (20 groups),nlpd,-7.18182 +python_Kriging-vecchia-nystrom_large-n,python,hartmann3/n=1500,subset of data (500),fit_s,0.712767 +python_Kriging-vecchia-nystrom_large-n,python,hartmann3/n=1500,subset of data (500),rmse,0.00269153 +python_Kriging-vecchia-nystrom_large-n,python,hartmann3/n=1500,subset of data (500),q2,0.999992 +python_Kriging-vecchia-nystrom_large-n,python,hartmann3/n=1500,subset of data (500),nlpd,-4.68641 +python_Kriging-vecchia-nystrom_large-n,python,hartmann3/n=5000,subset of data (500),fit_s,0.950298 +python_Kriging-vecchia-nystrom_large-n,python,hartmann3/n=5000,subset of data (500),rmse,0.00247552 +python_Kriging-vecchia-nystrom_large-n,python,hartmann3/n=5000,subset of data (500),q2,0.999994 +python_Kriging-vecchia-nystrom_large-n,python,hartmann3/n=5000,subset of data (500),nlpd,-4.71359 +python_Kriging-vecchia-nystrom_large-n,python,otl_circuit/n=1500,exact,fit_s,10.6103 +python_NestedKriging_aggregations-large-n,python,otl_circuit/n=1500,exact,fit_s,10.6103 +python_Kriging-vecchia-nystrom_large-n,python,otl_circuit/n=1500,exact,rmse,9.81598e-05 +python_NestedKriging_aggregations-large-n,python,otl_circuit/n=1500,exact,rmse,9.81598e-05 +python_Kriging-vecchia-nystrom_large-n,python,otl_circuit/n=1500,exact,q2,1 +python_NestedKriging_aggregations-large-n,python,otl_circuit/n=1500,exact,q2,1 +python_Kriging-vecchia-nystrom_large-n,python,otl_circuit/n=1500,exact,nlpd,-8.21962 +python_NestedKriging_aggregations-large-n,python,otl_circuit/n=1500,exact,nlpd,-8.21962 +python_Kriging-vecchia-nystrom_large-n,python,otl_circuit/n=1500,vecchia(30),fit_s,4.73217 +python_Kriging-vecchia-nystrom_large-n,python,otl_circuit/n=1500,vecchia(30),rmse,0.00629979 +python_Kriging-vecchia-nystrom_large-n,python,otl_circuit/n=1500,vecchia(30),q2,0.999969 +python_Kriging-vecchia-nystrom_large-n,python,otl_circuit/n=1500,vecchia(30),nlpd,-3.95617 +python_Kriging-vecchia-nystrom_large-n,python,otl_circuit/n=5000,vecchia(30),fit_s,14.5549 +python_Kriging-vecchia-nystrom_large-n,python,otl_circuit/n=5000,vecchia(30),rmse,0.00312888 +python_Kriging-vecchia-nystrom_large-n,python,otl_circuit/n=5000,vecchia(30),q2,0.999992 +python_Kriging-vecchia-nystrom_large-n,python,otl_circuit/n=5000,vecchia(30),nlpd,-4.57612 +python_Kriging-vecchia-nystrom_large-n,python,otl_circuit/n=1500,nystrom(100),fit_s,1.47345 +python_Kriging-vecchia-nystrom_large-n,python,otl_circuit/n=1500,nystrom(100),rmse,0.392811 +python_Kriging-vecchia-nystrom_large-n,python,otl_circuit/n=1500,nystrom(100),q2,0.880908 +python_Kriging-vecchia-nystrom_large-n,python,otl_circuit/n=1500,nystrom(100),nlpd,7.06156e+22 +python_Kriging-vecchia-nystrom_large-n,python,otl_circuit/n=5000,nystrom(100),fit_s,4.69579 +python_Kriging-vecchia-nystrom_large-n,python,otl_circuit/n=5000,nystrom(100),rmse,1.11526 +python_Kriging-vecchia-nystrom_large-n,python,otl_circuit/n=5000,nystrom(100),q2,0.0400057 +python_Kriging-vecchia-nystrom_large-n,python,otl_circuit/n=5000,nystrom(100),nlpd,5.31511e+23 +python_NestedKriging_aggregations-large-n,python,otl_circuit/n=1500,nested NK (20 groups),fit_s,0.278129 +python_NestedKriging_aggregations-large-n,python,otl_circuit/n=1500,nested NK (20 groups),rmse,0.00226873 +python_NestedKriging_aggregations-large-n,python,otl_circuit/n=1500,nested NK (20 groups),q2,0.999996 +python_NestedKriging_aggregations-large-n,python,otl_circuit/n=1500,nested NK (20 groups),nlpd,-4.90542 +python_NestedKriging_aggregations-large-n,python,otl_circuit/n=5000,nested NK (20 groups),fit_s,3.62576 +python_NestedKriging_aggregations-large-n,python,otl_circuit/n=5000,nested NK (20 groups),rmse,0.000261686 +python_NestedKriging_aggregations-large-n,python,otl_circuit/n=5000,nested NK (20 groups),q2,1 +python_NestedKriging_aggregations-large-n,python,otl_circuit/n=5000,nested NK (20 groups),nlpd,-7.20623 +python_NestedKriging_aggregations-large-n,python,otl_circuit/n=1500,nested rBCM (20 groups),fit_s,0.315619 +python_NestedKriging_aggregations-large-n,python,otl_circuit/n=1500,nested rBCM (20 groups),rmse,0.00291869 +python_NestedKriging_aggregations-large-n,python,otl_circuit/n=1500,nested rBCM (20 groups),q2,0.999993 +python_NestedKriging_aggregations-large-n,python,otl_circuit/n=1500,nested rBCM (20 groups),nlpd,-2.55512 +python_NestedKriging_aggregations-large-n,python,otl_circuit/n=5000,nested rBCM (20 groups),fit_s,3.39037 +python_NestedKriging_aggregations-large-n,python,otl_circuit/n=5000,nested rBCM (20 groups),rmse,0.000278149 +python_NestedKriging_aggregations-large-n,python,otl_circuit/n=5000,nested rBCM (20 groups),q2,1 +python_NestedKriging_aggregations-large-n,python,otl_circuit/n=5000,nested rBCM (20 groups),nlpd,-4.34927 +python_Kriging-vecchia-nystrom_large-n,python,otl_circuit/n=1500,subset of data (500),fit_s,0.672431 +python_Kriging-vecchia-nystrom_large-n,python,otl_circuit/n=1500,subset of data (500),rmse,0.0004286 +python_Kriging-vecchia-nystrom_large-n,python,otl_circuit/n=1500,subset of data (500),q2,1 +python_Kriging-vecchia-nystrom_large-n,python,otl_circuit/n=1500,subset of data (500),nlpd,-6.72678 +python_Kriging-vecchia-nystrom_large-n,python,otl_circuit/n=5000,subset of data (500),fit_s,0.692672 +python_Kriging-vecchia-nystrom_large-n,python,otl_circuit/n=5000,subset of data (500),rmse,0.000446879 +python_Kriging-vecchia-nystrom_large-n,python,otl_circuit/n=5000,subset of data (500),q2,1 +python_Kriging-vecchia-nystrom_large-n,python,otl_circuit/n=5000,subset of data (500),nlpd,-6.78844 +python_NestedKriging_aggregations-large-n,python,branin/nested,NK,rmse,0.0343984 +python_NestedKriging_aggregations-large-n,python,branin/nested,NK,q2,1 +python_NestedKriging_aggregations-large-n,python,branin/nested,NK,nlpd,2.00082e+14 +python_NestedKriging_aggregations-large-n,python,branin/nested,PoE,rmse,0.0354265 +python_NestedKriging_aggregations-large-n,python,branin/nested,PoE,q2,1 +python_NestedKriging_aggregations-large-n,python,branin/nested,PoE,nlpd,-1.64533 +python_NestedKriging_aggregations-large-n,python,branin/nested,gPoE,rmse,0.0354265 +python_NestedKriging_aggregations-large-n,python,branin/nested,gPoE,q2,1 +python_NestedKriging_aggregations-large-n,python,branin/nested,gPoE,nlpd,-0.621836 +python_NestedKriging_aggregations-large-n,python,branin/nested,BCM,rmse,0.0354196 +python_NestedKriging_aggregations-large-n,python,branin/nested,BCM,q2,1 +python_NestedKriging_aggregations-large-n,python,branin/nested,BCM,nlpd,-1.64533 +python_NestedKriging_aggregations-large-n,python,branin/nested,rBCM,rmse,0.035287 +python_NestedKriging_aggregations-large-n,python,branin/nested,rBCM,q2,1 +python_NestedKriging_aggregations-large-n,python,branin/nested,rBCM,nlpd,-2.61369 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,branin,pylibkriging,q2,0.999944 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,branin,pylibkriging,nlpd,0.137225 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,branin,pylibkriging,fit_s,0.00261593 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,branin,pylibkriging BFGS10,q2,0.999945 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,branin,pylibkriging BFGS10,nlpd,0.137254 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,branin,pylibkriging BFGS10,fit_s,0.02301 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,branin,scikit-learn,q2,0.999597 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,branin,scikit-learn,nlpd,0.928508 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,branin,scikit-learn,fit_s,0.169693 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,branin,SMT,q2,0.999944 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,branin,SMT,nlpd,0.137259 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,branin,SMT,fit_s,0.833295 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,branin,OpenTURNS,q2,0.998221 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,branin,OpenTURNS,nlpd,1.55599 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,branin,OpenTURNS,fit_s,0.00490117 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,six_hump_camel,pylibkriging,q2,0.999418 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,six_hump_camel,pylibkriging,nlpd,0.0429342 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,six_hump_camel,pylibkriging,fit_s,0.00324678 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,six_hump_camel,pylibkriging BFGS10,q2,0.999418 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,six_hump_camel,pylibkriging BFGS10,nlpd,0.0429386 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,six_hump_camel,pylibkriging BFGS10,fit_s,0.060586 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,six_hump_camel,scikit-learn,q2,0.0163311 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,six_hump_camel,scikit-learn,nlpd,4.63763 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,six_hump_camel,scikit-learn,fit_s,0.206083 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,six_hump_camel,SMT,q2,0.999594 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,six_hump_camel,SMT,nlpd,-0.121007 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,six_hump_camel,SMT,fit_s,1.03758 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,six_hump_camel,OpenTURNS,q2,0.956713 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,six_hump_camel,OpenTURNS,nlpd,2.55446 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,six_hump_camel,OpenTURNS,fit_s,0.00741315 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann3,pylibkriging,q2,0.986148 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann3,pylibkriging,nlpd,-0.847247 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann3,pylibkriging,fit_s,0.0169713 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann3,pylibkriging BFGS10,q2,0.986148 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann3,pylibkriging BFGS10,nlpd,-0.847249 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann3,pylibkriging BFGS10,fit_s,0.139588 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann3,scikit-learn,q2,0.983889 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann3,scikit-learn,nlpd,-0.782697 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann3,scikit-learn,fit_s,0.188796 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann3,SMT,q2,0.986148 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann3,SMT,nlpd,-0.847249 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann3,SMT,fit_s,1.02907 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann3,OpenTURNS,q2,0.984712 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann3,OpenTURNS,nlpd,-0.809851 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann3,OpenTURNS,fit_s,0.0153861 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,ishigami,pylibkriging,q2,0.259556 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,ishigami,pylibkriging,nlpd,4.38991 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,ishigami,pylibkriging,fit_s,0.0256422 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,ishigami,pylibkriging BFGS10,q2,0.935722 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,ishigami,pylibkriging BFGS10,nlpd,0.857443 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,ishigami,pylibkriging BFGS10,fit_s,0.308839 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,ishigami,scikit-learn,q2,0.831142 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,ishigami,scikit-learn,nlpd,1.5455 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,ishigami,scikit-learn,fit_s,0.51733 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,ishigami,SMT,q2,0.51196 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,ishigami,SMT,nlpd,3.31349 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,ishigami,SMT,fit_s,2.77225 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,ishigami,OpenTURNS,q2,0.887251 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,ishigami,OpenTURNS,nlpd,1.39348 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,ishigami,OpenTURNS,fit_s,0.0499659 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,friedman,pylibkriging,q2,0.999771 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,friedman,pylibkriging,nlpd,-1.29726 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,friedman,pylibkriging,fit_s,0.0181558 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,friedman,pylibkriging BFGS10,q2,0.999771 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,friedman,pylibkriging BFGS10,nlpd,-1.29727 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,friedman,pylibkriging BFGS10,fit_s,0.215106 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,friedman,scikit-learn,q2,0.999754 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,friedman,scikit-learn,nlpd,-1.37702 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,friedman,scikit-learn,fit_s,0.283978 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,friedman,SMT,q2,0.999899 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,friedman,SMT,nlpd,-1.871 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,friedman,SMT,fit_s,4.3249 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,friedman,OpenTURNS,q2,0.990862 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,friedman,OpenTURNS,nlpd,0.469828 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,friedman,OpenTURNS,fit_s,0.0293689 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,otl_circuit,pylibkriging,q2,0.999922 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,otl_circuit,pylibkriging,nlpd,-3.46228 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,otl_circuit,pylibkriging,fit_s,0.013788 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,otl_circuit,pylibkriging BFGS10,q2,0.999922 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,otl_circuit,pylibkriging BFGS10,nlpd,-3.46227 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,otl_circuit,pylibkriging BFGS10,fit_s,0.141071 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,otl_circuit,scikit-learn,q2,0.999906 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,otl_circuit,scikit-learn,nlpd,-3.43062 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,otl_circuit,scikit-learn,fit_s,0.51218 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,otl_circuit,SMT,q2,0.999954 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,otl_circuit,SMT,nlpd,-3.35016 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,otl_circuit,SMT,fit_s,7.1224 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,otl_circuit,OpenTURNS,q2,0.99777 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,otl_circuit,OpenTURNS,nlpd,-1.61723 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,otl_circuit,OpenTURNS,fit_s,0.0603108 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,piston,pylibkriging,q2,0.997498 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,piston,pylibkriging,nlpd,-3.80667 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,piston,pylibkriging,fit_s,0.0395648 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,piston,pylibkriging BFGS10,q2,0.997498 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,piston,pylibkriging BFGS10,nlpd,-3.8067 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,piston,pylibkriging BFGS10,fit_s,0.327989 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,piston,scikit-learn,q2,0.998684 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,piston,scikit-learn,nlpd,-4.10497 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,piston,scikit-learn,fit_s,1.11386 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,piston,SMT,q2,0.997511 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,piston,SMT,nlpd,-3.86618 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,piston,SMT,fit_s,12.649 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,piston,OpenTURNS,q2,0.990631 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,piston,OpenTURNS,nlpd,-2.9906 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,piston,OpenTURNS,fit_s,0.11631 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,borehole,pylibkriging,q2,0.999694 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,borehole,pylibkriging,nlpd,1.00118 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,borehole,pylibkriging,fit_s,0.0149446 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,borehole,pylibkriging BFGS10,q2,0.999694 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,borehole,pylibkriging BFGS10,nlpd,1.00121 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,borehole,pylibkriging BFGS10,fit_s,0.164608 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,borehole,scikit-learn,q2,0.999929 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,borehole,scikit-learn,nlpd,0.194455 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,borehole,scikit-learn,fit_s,1.04563 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,borehole,SMT,q2,0.999925 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,borehole,SMT,nlpd,0.279122 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,borehole,SMT,fit_s,11.8531 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,borehole,OpenTURNS,q2,0.996206 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,borehole,OpenTURNS,nlpd,2.55431 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,borehole,OpenTURNS,fit_s,0.12395 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,wing_weight,pylibkriging,q2,0.99972 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,wing_weight,pylibkriging,nlpd,1.11517 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,wing_weight,pylibkriging,fit_s,0.0290489 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,wing_weight,pylibkriging BFGS10,q2,0.99972 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,wing_weight,pylibkriging BFGS10,nlpd,1.11512 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,wing_weight,pylibkriging BFGS10,fit_s,0.314707 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,wing_weight,scikit-learn,q2,0.998048 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,wing_weight,scikit-learn,nlpd,2.57304 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,wing_weight,scikit-learn,fit_s,0.729278 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,wing_weight,SMT,q2,0.999898 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,wing_weight,SMT,nlpd,0.604023 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,wing_weight,SMT,fit_s,25.8262 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,wing_weight,OpenTURNS,q2,0.993145 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,wing_weight,OpenTURNS,nlpd,2.83627 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,wing_weight,OpenTURNS,fit_s,0.280434 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann6,pylibkriging,q2,0.422938 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann6,pylibkriging,nlpd,0.0965167 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann6,pylibkriging,fit_s,0.0562823 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann6,pylibkriging BFGS10,q2,0.72352 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann6,pylibkriging BFGS10,nlpd,-0.156898 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann6,pylibkriging BFGS10,fit_s,0.822745 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann6,scikit-learn,q2,0.69049 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann6,scikit-learn,nlpd,-0.137845 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann6,scikit-learn,fit_s,1.3155 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann6,SMT,q2,0.708463 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann6,SMT,nlpd,-0.0177081 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann6,SMT,fit_s,12.0578 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann6,OpenTURNS,q2,0.696177 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann6,OpenTURNS,nlpd,-0.132838 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann6,OpenTURNS,fit_s,0.133332 diff --git a/tools/baseline/metrics-release.csv b/tools/baseline/metrics-release.csv new file mode 100644 index 0000000..5c4790d --- /dev/null +++ b/tools/baseline/metrics-release.csv @@ -0,0 +1,924 @@ +suite,lang,case,package,metric,value +python_Kriging_kernels-trends-interpolation,python,forrester/gauss/constant,pylibkriging,fit_s,0.0039537 +python_Kriging_kernels-trends-interpolation,python,forrester/gauss/constant,pylibkriging,rmse,0.0848271 +python_Kriging_kernels-trends-interpolation,python,forrester/gauss/constant,pylibkriging,q2,0.999634 +python_Kriging_kernels-trends-interpolation,python,forrester/gauss/constant,pylibkriging,nlpd,-3.06486 +python_Kriging_kernels-trends-interpolation,python,forrester/gauss/linear,pylibkriging,fit_s,0.0008564 +python_Kriging_kernels-trends-interpolation,python,forrester/gauss/linear,pylibkriging,rmse,0.0855468 +python_Kriging_kernels-trends-interpolation,python,forrester/gauss/linear,pylibkriging,q2,0.999628 +python_Kriging_kernels-trends-interpolation,python,forrester/gauss/linear,pylibkriging,nlpd,-3.11029 +python_Kriging_kernels-trends-interpolation,python,forrester/gauss/interactive,pylibkriging,fit_s,0.000818729 +python_Kriging_kernels-trends-interpolation,python,forrester/gauss/interactive,pylibkriging,rmse,0.0855468 +python_Kriging_kernels-trends-interpolation,python,forrester/gauss/interactive,pylibkriging,q2,0.999628 +python_Kriging_kernels-trends-interpolation,python,forrester/gauss/interactive,pylibkriging,nlpd,-3.11029 +python_Kriging_kernels-trends-interpolation,python,forrester/gauss/quadratic,pylibkriging,fit_s,0.000473261 +python_Kriging_kernels-trends-interpolation,python,forrester/gauss/quadratic,pylibkriging,rmse,1.57249 +python_Kriging_kernels-trends-interpolation,python,forrester/gauss/quadratic,pylibkriging,q2,0.874394 +python_Kriging_kernels-trends-interpolation,python,forrester/gauss/quadratic,pylibkriging,nlpd,5.33879 +python_Kriging_kernels-trends-interpolation,python,forrester/exp/constant,pylibkriging,fit_s,0.00042963 +python_Kriging_kernels-trends-interpolation,python,forrester/exp/constant,pylibkriging,rmse,0.721582 +python_Kriging_kernels-trends-interpolation,python,forrester/exp/constant,pylibkriging,q2,0.973551 +python_Kriging_kernels-trends-interpolation,python,forrester/exp/constant,pylibkriging,nlpd,1.54551 +python_Kriging_kernels-trends-interpolation,python,forrester/exp/linear,pylibkriging,fit_s,0.000402451 +python_Kriging_kernels-trends-interpolation,python,forrester/exp/linear,pylibkriging,rmse,0.793413 +python_Kriging_kernels-trends-interpolation,python,forrester/exp/linear,pylibkriging,q2,0.968024 +python_Kriging_kernels-trends-interpolation,python,forrester/exp/linear,pylibkriging,nlpd,1.50639 +python_Kriging_kernels-trends-interpolation,python,forrester/exp/interactive,pylibkriging,fit_s,0.000428438 +python_Kriging_kernels-trends-interpolation,python,forrester/exp/interactive,pylibkriging,rmse,0.793413 +python_Kriging_kernels-trends-interpolation,python,forrester/exp/interactive,pylibkriging,q2,0.968024 +python_Kriging_kernels-trends-interpolation,python,forrester/exp/interactive,pylibkriging,nlpd,1.50639 +python_Kriging_kernels-trends-interpolation,python,forrester/exp/quadratic,pylibkriging,fit_s,0.000418425 +python_Kriging_kernels-trends-interpolation,python,forrester/exp/quadratic,pylibkriging,rmse,0.723267 +python_Kriging_kernels-trends-interpolation,python,forrester/exp/quadratic,pylibkriging,q2,0.973428 +python_Kriging_kernels-trends-interpolation,python,forrester/exp/quadratic,pylibkriging,nlpd,1.28782 +python_Kriging_kernels-trends-interpolation,python,forrester/matern3_2/constant,pylibkriging,fit_s,0.000519753 +python_Kriging_kernels-trends-interpolation,python,forrester/matern3_2/constant,pylibkriging,rmse,0.334007 +python_Kriging_kernels-trends-interpolation,python,forrester/matern3_2/constant,pylibkriging,q2,0.994333 +python_Kriging_kernels-trends-interpolation,python,forrester/matern3_2/constant,pylibkriging,nlpd,-0.227715 +python_Kriging_kernels-trends-interpolation,python,forrester/matern3_2/linear,pylibkriging,fit_s,0.000559092 +python_Kriging_kernels-trends-interpolation,python,forrester/matern3_2/linear,pylibkriging,rmse,0.364477 +python_Kriging_kernels-trends-interpolation,python,forrester/matern3_2/linear,pylibkriging,q2,0.993252 +python_Kriging_kernels-trends-interpolation,python,forrester/matern3_2/linear,pylibkriging,nlpd,-0.187262 +python_Kriging_kernels-trends-interpolation,python,forrester/matern3_2/interactive,pylibkriging,fit_s,0.00060463 +python_Kriging_kernels-trends-interpolation,python,forrester/matern3_2/interactive,pylibkriging,rmse,0.364477 +python_Kriging_kernels-trends-interpolation,python,forrester/matern3_2/interactive,pylibkriging,q2,0.993252 +python_Kriging_kernels-trends-interpolation,python,forrester/matern3_2/interactive,pylibkriging,nlpd,-0.187262 +python_Kriging_kernels-trends-interpolation,python,forrester/matern3_2/quadratic,pylibkriging,fit_s,0.000493765 +python_Kriging_kernels-trends-interpolation,python,forrester/matern3_2/quadratic,pylibkriging,rmse,0.435609 +python_Kriging_kernels-trends-interpolation,python,forrester/matern3_2/quadratic,pylibkriging,q2,0.990361 +python_Kriging_kernels-trends-interpolation,python,forrester/matern3_2/quadratic,pylibkriging,nlpd,-0.183076 +python_Kriging_kernels-trends-interpolation,python,forrester/matern5_2/constant,pylibkriging,fit_s,0.000758886 +python_Kriging_kernels-trends-interpolation,python,forrester/matern5_2/constant,pylibkriging,rmse,0.281945 +python_Kriging_kernels-trends-interpolation,python,forrester/matern5_2/constant,pylibkriging,q2,0.995962 +python_Kriging_kernels-trends-interpolation,python,forrester/matern5_2/constant,pylibkriging,nlpd,-1.01945 +python_Kriging_kernels-trends-interpolation,python,forrester/matern5_2/constant,pylibkriging BFGS10,rmse,0.281945 +python_Kriging_kernels-trends-interpolation,python,forrester/matern5_2/constant,pylibkriging BFGS10,q2,0.995962 +python_Kriging_kernels-trends-interpolation,python,forrester/matern5_2/constant,pylibkriging BFGS10,nlpd,-1.01945 +python_Kriging_kernels-trends-interpolation,python,forrester/matern5_2/linear,pylibkriging,fit_s,0.000576973 +python_Kriging_kernels-trends-interpolation,python,forrester/matern5_2/linear,pylibkriging,rmse,0.298959 +python_Kriging_kernels-trends-interpolation,python,forrester/matern5_2/linear,pylibkriging,q2,0.99546 +python_Kriging_kernels-trends-interpolation,python,forrester/matern5_2/linear,pylibkriging,nlpd,-0.95551 +python_Kriging_kernels-trends-interpolation,python,forrester/matern5_2/interactive,pylibkriging,fit_s,0.00054884 +python_Kriging_kernels-trends-interpolation,python,forrester/matern5_2/interactive,pylibkriging,rmse,0.298959 +python_Kriging_kernels-trends-interpolation,python,forrester/matern5_2/interactive,pylibkriging,q2,0.99546 +python_Kriging_kernels-trends-interpolation,python,forrester/matern5_2/interactive,pylibkriging,nlpd,-0.95551 +python_Kriging_kernels-trends-interpolation,python,forrester/matern5_2/quadratic,pylibkriging,fit_s,0.000477076 +python_Kriging_kernels-trends-interpolation,python,forrester/matern5_2/quadratic,pylibkriging,rmse,0.365039 +python_Kriging_kernels-trends-interpolation,python,forrester/matern5_2/quadratic,pylibkriging,q2,0.993231 +python_Kriging_kernels-trends-interpolation,python,forrester/matern5_2/quadratic,pylibkriging,nlpd,-0.790032 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/gauss/constant,pylibkriging,fit_s,0.0016222 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/gauss/constant,pylibkriging,rmse,0.191274 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/gauss/constant,pylibkriging,q2,0.978742 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/gauss/constant,pylibkriging,nlpd,-0.704948 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/gauss/linear,pylibkriging,fit_s,0.00240469 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/gauss/linear,pylibkriging,rmse,0.236028 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/gauss/linear,pylibkriging,q2,0.96763 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/gauss/linear,pylibkriging,nlpd,-0.580993 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/gauss/interactive,pylibkriging,fit_s,0.00205922 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/gauss/interactive,pylibkriging,rmse,0.236028 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/gauss/interactive,pylibkriging,q2,0.96763 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/gauss/interactive,pylibkriging,nlpd,-0.580993 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/gauss/quadratic,pylibkriging,fit_s,0.00200701 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/gauss/quadratic,pylibkriging,rmse,0.150914 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/gauss/quadratic,pylibkriging,q2,0.986767 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/gauss/quadratic,pylibkriging,nlpd,-0.458431 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/exp/constant,pylibkriging,fit_s,0.00123 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/exp/constant,pylibkriging,rmse,0.196586 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/exp/constant,pylibkriging,q2,0.977545 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/exp/constant,pylibkriging,nlpd,-0.0621192 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/exp/linear,pylibkriging,fit_s,0.00172782 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/exp/linear,pylibkriging,rmse,0.20979 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/exp/linear,pylibkriging,q2,0.974427 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/exp/linear,pylibkriging,nlpd,-0.0466568 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/exp/interactive,pylibkriging,fit_s,0.0017736 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/exp/interactive,pylibkriging,rmse,0.20979 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/exp/interactive,pylibkriging,q2,0.974427 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/exp/interactive,pylibkriging,nlpd,-0.0466568 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/exp/quadratic,pylibkriging,fit_s,0.00394821 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/exp/quadratic,pylibkriging,rmse,0.252565 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/exp/quadratic,pylibkriging,q2,0.962935 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/exp/quadratic,pylibkriging,nlpd,0.220609 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern3_2/constant,pylibkriging,fit_s,0.000825167 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern3_2/constant,pylibkriging,rmse,0.177253 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern3_2/constant,pylibkriging,q2,0.981744 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern3_2/constant,pylibkriging,nlpd,-0.482212 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern3_2/linear,pylibkriging,fit_s,0.000857115 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern3_2/linear,pylibkriging,rmse,0.211499 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern3_2/linear,pylibkriging,q2,0.974009 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern3_2/linear,pylibkriging,nlpd,-0.375844 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern3_2/interactive,pylibkriging,fit_s,0.00114012 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern3_2/interactive,pylibkriging,rmse,0.211499 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern3_2/interactive,pylibkriging,q2,0.974009 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern3_2/interactive,pylibkriging,nlpd,-0.375844 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern3_2/quadratic,pylibkriging,fit_s,0.00237894 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern3_2/quadratic,pylibkriging,rmse,0.184528 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern3_2/quadratic,pylibkriging,q2,0.980215 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern3_2/quadratic,pylibkriging,nlpd,-0.171795 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern5_2/constant,pylibkriging,fit_s,0.000836849 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern5_2/constant,pylibkriging,rmse,0.177638 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern5_2/constant,pylibkriging,q2,0.981665 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern5_2/constant,pylibkriging,nlpd,-0.575421 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern5_2/constant,pylibkriging BFGS10,rmse,0.177638 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern5_2/constant,pylibkriging BFGS10,q2,0.981665 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern5_2/constant,pylibkriging BFGS10,nlpd,-0.575422 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern5_2/linear,pylibkriging,fit_s,0.000859499 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern5_2/linear,pylibkriging,rmse,0.219759 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern5_2/linear,pylibkriging,q2,0.971939 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern5_2/linear,pylibkriging,nlpd,-0.458933 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern5_2/interactive,pylibkriging,fit_s,0.00080061 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern5_2/interactive,pylibkriging,rmse,0.219759 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern5_2/interactive,pylibkriging,q2,0.971939 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern5_2/interactive,pylibkriging,nlpd,-0.458933 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern5_2/quadratic,pylibkriging,fit_s,0.00249147 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern5_2/quadratic,pylibkriging,rmse,0.169941 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern5_2/quadratic,pylibkriging,q2,0.983219 +python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern5_2/quadratic,pylibkriging,nlpd,-0.28938 +python_Kriging_kernels-trends-interpolation,python,branin/gauss/constant,pylibkriging,fit_s,0.00172019 +python_Kriging_kernels-trends-interpolation,python,branin/gauss/constant,pylibkriging,rmse,7.2629 +python_Kriging_kernels-trends-interpolation,python,branin/gauss/constant,pylibkriging,q2,0.980655 +python_Kriging_kernels-trends-interpolation,python,branin/gauss/constant,pylibkriging,nlpd,4.3152 +python_Kriging_kernels-trends-interpolation,python,branin/gauss/linear,pylibkriging,fit_s,0.00188136 +python_Kriging_kernels-trends-interpolation,python,branin/gauss/linear,pylibkriging,rmse,7.25733 +python_Kriging_kernels-trends-interpolation,python,branin/gauss/linear,pylibkriging,q2,0.980685 +python_Kriging_kernels-trends-interpolation,python,branin/gauss/linear,pylibkriging,nlpd,4.30813 +python_Kriging_kernels-trends-interpolation,python,branin/gauss/interactive,pylibkriging,fit_s,0.00456238 +python_Kriging_kernels-trends-interpolation,python,branin/gauss/interactive,pylibkriging,rmse,7.2576 +python_Kriging_kernels-trends-interpolation,python,branin/gauss/interactive,pylibkriging,q2,0.980683 +python_Kriging_kernels-trends-interpolation,python,branin/gauss/interactive,pylibkriging,nlpd,4.30176 +python_Kriging_kernels-trends-interpolation,python,branin/gauss/quadratic,pylibkriging,fit_s,0.00182772 +python_Kriging_kernels-trends-interpolation,python,branin/gauss/quadratic,pylibkriging,rmse,7.27808 +python_Kriging_kernels-trends-interpolation,python,branin/gauss/quadratic,pylibkriging,q2,0.980574 +python_Kriging_kernels-trends-interpolation,python,branin/gauss/quadratic,pylibkriging,nlpd,4.29399 +python_Kriging_kernels-trends-interpolation,python,branin/exp/constant,pylibkriging,fit_s,0.00215006 +python_Kriging_kernels-trends-interpolation,python,branin/exp/constant,pylibkriging,rmse,14.5887 +python_Kriging_kernels-trends-interpolation,python,branin/exp/constant,pylibkriging,q2,0.921947 +python_Kriging_kernels-trends-interpolation,python,branin/exp/constant,pylibkriging,nlpd,4.09457 +python_Kriging_kernels-trends-interpolation,python,branin/exp/linear,pylibkriging,fit_s,0.00210071 +python_Kriging_kernels-trends-interpolation,python,branin/exp/linear,pylibkriging,rmse,14.9622 +python_Kriging_kernels-trends-interpolation,python,branin/exp/linear,pylibkriging,q2,0.9179 +python_Kriging_kernels-trends-interpolation,python,branin/exp/linear,pylibkriging,nlpd,4.0909 +python_Kriging_kernels-trends-interpolation,python,branin/exp/interactive,pylibkriging,fit_s,0.00220799 +python_Kriging_kernels-trends-interpolation,python,branin/exp/interactive,pylibkriging,rmse,11.6963 +python_Kriging_kernels-trends-interpolation,python,branin/exp/interactive,pylibkriging,q2,0.94983 +python_Kriging_kernels-trends-interpolation,python,branin/exp/interactive,pylibkriging,nlpd,3.79671 +python_Kriging_kernels-trends-interpolation,python,branin/exp/quadratic,pylibkriging,fit_s,0.00237751 +python_Kriging_kernels-trends-interpolation,python,branin/exp/quadratic,pylibkriging,rmse,9.33909 +python_Kriging_kernels-trends-interpolation,python,branin/exp/quadratic,pylibkriging,q2,0.968014 +python_Kriging_kernels-trends-interpolation,python,branin/exp/quadratic,pylibkriging,nlpd,3.62362 +python_Kriging_kernels-trends-interpolation,python,branin/matern3_2/constant,pylibkriging,fit_s,0.00252843 +python_Kriging_kernels-trends-interpolation,python,branin/matern3_2/constant,pylibkriging,rmse,3.24796 +python_Kriging_kernels-trends-interpolation,python,branin/matern3_2/constant,pylibkriging,q2,0.996131 +python_Kriging_kernels-trends-interpolation,python,branin/matern3_2/constant,pylibkriging,nlpd,2.30444 +python_Kriging_kernels-trends-interpolation,python,branin/matern3_2/linear,pylibkriging,fit_s,0.00200725 +python_Kriging_kernels-trends-interpolation,python,branin/matern3_2/linear,pylibkriging,rmse,3.14402 +python_Kriging_kernels-trends-interpolation,python,branin/matern3_2/linear,pylibkriging,q2,0.996375 +python_Kriging_kernels-trends-interpolation,python,branin/matern3_2/linear,pylibkriging,nlpd,2.28067 +python_Kriging_kernels-trends-interpolation,python,branin/matern3_2/interactive,pylibkriging,fit_s,0.00414944 +python_Kriging_kernels-trends-interpolation,python,branin/matern3_2/interactive,pylibkriging,rmse,3.19226 +python_Kriging_kernels-trends-interpolation,python,branin/matern3_2/interactive,pylibkriging,q2,0.996263 +python_Kriging_kernels-trends-interpolation,python,branin/matern3_2/interactive,pylibkriging,nlpd,2.20359 +python_Kriging_kernels-trends-interpolation,python,branin/matern3_2/quadratic,pylibkriging,fit_s,0.00253224 +python_Kriging_kernels-trends-interpolation,python,branin/matern3_2/quadratic,pylibkriging,rmse,3.01232 +python_Kriging_kernels-trends-interpolation,python,branin/matern3_2/quadratic,pylibkriging,q2,0.996672 +python_Kriging_kernels-trends-interpolation,python,branin/matern3_2/quadratic,pylibkriging,nlpd,2.11627 +python_Kriging_kernels-trends-interpolation,python,branin/matern5_2/constant,pylibkriging,fit_s,0.00171685 +python_Kriging_kernels-trends-interpolation,python,branin/matern5_2/constant,pylibkriging,rmse,0.52012 +python_Kriging_kernels-trends-interpolation,python,branin/matern5_2/constant,pylibkriging,q2,0.999901 +python_Kriging_kernels-trends-interpolation,python,branin/matern5_2/constant,pylibkriging,nlpd,0.283399 +python_Kriging_kernels-trends-interpolation,python,branin/matern5_2/constant,pylibkriging BFGS10,rmse,0.52012 +python_Kriging_kernels-trends-interpolation,python,branin/matern5_2/constant,pylibkriging BFGS10,q2,0.999901 +python_Kriging_kernels-trends-interpolation,python,branin/matern5_2/constant,pylibkriging BFGS10,nlpd,0.283395 +python_Kriging_kernels-trends-interpolation,python,branin/matern5_2/linear,pylibkriging,fit_s,0.0021255 +python_Kriging_kernels-trends-interpolation,python,branin/matern5_2/linear,pylibkriging,rmse,0.460868 +python_Kriging_kernels-trends-interpolation,python,branin/matern5_2/linear,pylibkriging,q2,0.999922 +python_Kriging_kernels-trends-interpolation,python,branin/matern5_2/linear,pylibkriging,nlpd,0.209741 +python_Kriging_kernels-trends-interpolation,python,branin/matern5_2/interactive,pylibkriging,fit_s,0.00307012 +python_Kriging_kernels-trends-interpolation,python,branin/matern5_2/interactive,pylibkriging,rmse,0.498484 +python_Kriging_kernels-trends-interpolation,python,branin/matern5_2/interactive,pylibkriging,q2,0.999909 +python_Kriging_kernels-trends-interpolation,python,branin/matern5_2/interactive,pylibkriging,nlpd,0.239079 +python_Kriging_kernels-trends-interpolation,python,branin/matern5_2/quadratic,pylibkriging,fit_s,0.0092895 +python_Kriging_kernels-trends-interpolation,python,branin/matern5_2/quadratic,pylibkriging,rmse,0.67472 +python_Kriging_kernels-trends-interpolation,python,branin/matern5_2/quadratic,pylibkriging,q2,0.999833 +python_Kriging_kernels-trends-interpolation,python,branin/matern5_2/quadratic,pylibkriging,nlpd,0.419266 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/gauss/constant,pylibkriging,fit_s,0.00194049 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/gauss/constant,pylibkriging,rmse,18.9501 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/gauss/constant,pylibkriging,q2,0.536354 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/gauss/constant,pylibkriging,nlpd,84.151 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/gauss/linear,pylibkriging,fit_s,0.00170755 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/gauss/linear,pylibkriging,rmse,18.9579 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/gauss/linear,pylibkriging,q2,0.535976 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/gauss/linear,pylibkriging,nlpd,44.8696 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/gauss/interactive,pylibkriging,fit_s,0.00180864 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/gauss/interactive,pylibkriging,rmse,18.9819 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/gauss/interactive,pylibkriging,q2,0.5348 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/gauss/interactive,pylibkriging,nlpd,35.2765 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/gauss/quadratic,pylibkriging,fit_s,0.00178218 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/gauss/quadratic,pylibkriging,rmse,10.9058 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/gauss/quadratic,pylibkriging,q2,0.846441 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/gauss/quadratic,pylibkriging,nlpd,4.92017 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/exp/constant,pylibkriging,fit_s,0.000977039 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/exp/constant,pylibkriging,rmse,16.9653 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/exp/constant,pylibkriging,q2,0.628393 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/exp/constant,pylibkriging,nlpd,4.48595 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/exp/linear,pylibkriging,fit_s,0.00107837 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/exp/linear,pylibkriging,rmse,17.1781 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/exp/linear,pylibkriging,q2,0.619012 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/exp/linear,pylibkriging,nlpd,4.47061 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/exp/interactive,pylibkriging,fit_s,0.00123405 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/exp/interactive,pylibkriging,rmse,17.1856 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/exp/interactive,pylibkriging,q2,0.618679 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/exp/interactive,pylibkriging,nlpd,4.43133 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/exp/quadratic,pylibkriging,fit_s,0.00120115 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/exp/quadratic,pylibkriging,rmse,7.99484 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/exp/quadratic,pylibkriging,q2,0.917476 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/exp/quadratic,pylibkriging,nlpd,3.35612 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern3_2/constant,pylibkriging,fit_s,0.00264788 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern3_2/constant,pylibkriging,rmse,18.4958 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern3_2/constant,pylibkriging,q2,0.558321 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern3_2/constant,pylibkriging,nlpd,18.8349 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern3_2/linear,pylibkriging,fit_s,0.00251079 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern3_2/linear,pylibkriging,rmse,18.5562 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern3_2/linear,pylibkriging,q2,0.555433 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern3_2/linear,pylibkriging,nlpd,14.9494 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern3_2/interactive,pylibkriging,fit_s,0.00148797 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern3_2/interactive,pylibkriging,rmse,18.561 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern3_2/interactive,pylibkriging,q2,0.555202 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern3_2/interactive,pylibkriging,nlpd,12.9923 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern3_2/quadratic,pylibkriging,fit_s,0.00289869 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern3_2/quadratic,pylibkriging,rmse,8.91645 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern3_2/quadratic,pylibkriging,q2,0.897353 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern3_2/quadratic,pylibkriging,nlpd,3.94939 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern5_2/constant,pylibkriging,fit_s,0.00238967 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern5_2/constant,pylibkriging,rmse,18.3236 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern5_2/constant,pylibkriging,q2,0.566507 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern5_2/constant,pylibkriging,nlpd,18.4322 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern5_2/constant,pylibkriging BFGS10,rmse,1.36208 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern5_2/constant,pylibkriging BFGS10,q2,0.997605 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern5_2/constant,pylibkriging BFGS10,nlpd,1.03215 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern5_2/linear,pylibkriging,fit_s,0.00207353 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern5_2/linear,pylibkriging,rmse,18.7127 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern5_2/linear,pylibkriging,q2,0.547901 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern5_2/linear,pylibkriging,nlpd,24.5771 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern5_2/interactive,pylibkriging,fit_s,0.0020268 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern5_2/interactive,pylibkriging,rmse,18.7194 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern5_2/interactive,pylibkriging,q2,0.547577 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern5_2/interactive,pylibkriging,nlpd,20.4255 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern5_2/quadratic,pylibkriging,fit_s,0.00144219 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern5_2/quadratic,pylibkriging,rmse,10.2878 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern5_2/quadratic,pylibkriging,q2,0.863351 +python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern5_2/quadratic,pylibkriging,nlpd,4.46344 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/gauss/constant,pylibkriging,fit_s,0.00526643 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/gauss/constant,pylibkriging,rmse,0.295934 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/gauss/constant,pylibkriging,q2,0.920537 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/gauss/constant,pylibkriging,nlpd,1.58666 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/gauss/linear,pylibkriging,fit_s,0.00606179 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/gauss/linear,pylibkriging,rmse,0.296355 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/gauss/linear,pylibkriging,q2,0.92031 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/gauss/linear,pylibkriging,nlpd,1.2962 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/gauss/interactive,pylibkriging,fit_s,0.00633311 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/gauss/interactive,pylibkriging,rmse,0.285136 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/gauss/interactive,pylibkriging,q2,0.926229 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/gauss/interactive,pylibkriging,nlpd,1.04122 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/gauss/quadratic,pylibkriging,fit_s,0.00615239 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/gauss/quadratic,pylibkriging,rmse,0.237308 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/gauss/quadratic,pylibkriging,q2,0.948902 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/gauss/quadratic,pylibkriging,nlpd,0.146484 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/exp/constant,pylibkriging,fit_s,0.00271893 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/exp/constant,pylibkriging,rmse,0.334876 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/exp/constant,pylibkriging,q2,0.898247 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/exp/constant,pylibkriging,nlpd,0.339235 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/exp/linear,pylibkriging,fit_s,0.00223827 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/exp/linear,pylibkriging,rmse,0.333447 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/exp/linear,pylibkriging,q2,0.899114 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/exp/linear,pylibkriging,nlpd,0.336515 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/exp/interactive,pylibkriging,fit_s,0.00192547 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/exp/interactive,pylibkriging,rmse,0.336819 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/exp/interactive,pylibkriging,q2,0.897063 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/exp/interactive,pylibkriging,nlpd,0.329566 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/exp/quadratic,pylibkriging,fit_s,0.00333118 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/exp/quadratic,pylibkriging,rmse,0.327816 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/exp/quadratic,pylibkriging,q2,0.902492 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/exp/quadratic,pylibkriging,nlpd,0.322056 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern3_2/constant,pylibkriging,fit_s,0.00606227 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern3_2/constant,pylibkriging,rmse,0.684362 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern3_2/constant,pylibkriging,q2,0.575038 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern3_2/constant,pylibkriging,nlpd,0.972803 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern3_2/linear,pylibkriging,fit_s,0.00382805 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern3_2/linear,pylibkriging,rmse,0.650201 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern3_2/linear,pylibkriging,q2,0.616404 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern3_2/linear,pylibkriging,nlpd,0.921455 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern3_2/interactive,pylibkriging,fit_s,0.00707507 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern3_2/interactive,pylibkriging,rmse,0.274326 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern3_2/interactive,pylibkriging,q2,0.931717 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern3_2/interactive,pylibkriging,nlpd,0.0549679 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern3_2/quadratic,pylibkriging,fit_s,0.00617123 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern3_2/quadratic,pylibkriging,rmse,0.263745 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern3_2/quadratic,pylibkriging,q2,0.936883 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern3_2/quadratic,pylibkriging,nlpd,0.0446267 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern5_2/constant,pylibkriging,fit_s,0.00426936 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern5_2/constant,pylibkriging,rmse,0.674391 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern5_2/constant,pylibkriging,q2,0.58733 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern5_2/constant,pylibkriging,nlpd,0.944765 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern5_2/constant,pylibkriging BFGS10,rmse,0.268216 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern5_2/constant,pylibkriging BFGS10,q2,0.934725 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern5_2/constant,pylibkriging BFGS10,nlpd,0.169585 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern5_2/linear,pylibkriging,fit_s,0.00730562 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern5_2/linear,pylibkriging,rmse,0.268808 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern5_2/linear,pylibkriging,q2,0.934436 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern5_2/linear,pylibkriging,nlpd,0.142382 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern5_2/interactive,pylibkriging,fit_s,0.00520349 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern5_2/interactive,pylibkriging,rmse,0.271088 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern5_2/interactive,pylibkriging,q2,0.933319 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern5_2/interactive,pylibkriging,nlpd,0.128788 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern5_2/quadratic,pylibkriging,fit_s,0.00757623 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern5_2/quadratic,pylibkriging,rmse,0.253093 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern5_2/quadratic,pylibkriging,q2,0.941878 +python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern5_2/quadratic,pylibkriging,nlpd,0.025469 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/gauss/constant,pylibkriging,fit_s,0.000713825 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/gauss/constant,pylibkriging,rmse,0.026127 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/gauss/constant,pylibkriging,q2,0.999737 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/gauss/constant,pylibkriging,nlpd,-2.69817 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/gauss/linear,pylibkriging,fit_s,0.000786066 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/gauss/linear,pylibkriging,rmse,1.06243 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/gauss/linear,pylibkriging,q2,0.564691 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/gauss/linear,pylibkriging,nlpd,2.7394 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/gauss/interactive,pylibkriging,fit_s,0.000851393 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/gauss/interactive,pylibkriging,rmse,1.06504 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/gauss/interactive,pylibkriging,q2,0.562551 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/gauss/interactive,pylibkriging,nlpd,2.33575 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/gauss/quadratic,pylibkriging,fit_s,0.0012238 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/gauss/quadratic,pylibkriging,rmse,0.281816 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/gauss/quadratic,pylibkriging,q2,0.969371 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/gauss/quadratic,pylibkriging,nlpd,0.71682 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/exp/constant,pylibkriging,fit_s,0.000771999 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/exp/constant,pylibkriging,rmse,0.249023 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/exp/constant,pylibkriging,q2,0.976085 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/exp/constant,pylibkriging,nlpd,0.394159 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/exp/linear,pylibkriging,fit_s,0.0006814 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/exp/linear,pylibkriging,rmse,0.245558 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/exp/linear,pylibkriging,q2,0.976746 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/exp/linear,pylibkriging,nlpd,0.224593 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/exp/interactive,pylibkriging,fit_s,0.000779867 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/exp/interactive,pylibkriging,rmse,0.239032 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/exp/interactive,pylibkriging,q2,0.977965 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/exp/interactive,pylibkriging,nlpd,0.226133 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/exp/quadratic,pylibkriging,fit_s,0.00164485 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/exp/quadratic,pylibkriging,rmse,0.237713 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/exp/quadratic,pylibkriging,q2,0.978208 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/exp/quadratic,pylibkriging,nlpd,-0.139582 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern3_2/constant,pylibkriging,fit_s,0.000581741 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern3_2/constant,pylibkriging,rmse,0.0600911 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern3_2/constant,pylibkriging,q2,0.998607 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern3_2/constant,pylibkriging,nlpd,-1.34658 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern3_2/linear,pylibkriging,fit_s,0.000698566 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern3_2/linear,pylibkriging,rmse,0.0584376 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern3_2/linear,pylibkriging,q2,0.998683 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern3_2/linear,pylibkriging,nlpd,-1.34912 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern3_2/interactive,pylibkriging,fit_s,0.000763416 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern3_2/interactive,pylibkriging,rmse,0.0576096 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern3_2/interactive,pylibkriging,q2,0.99872 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern3_2/interactive,pylibkriging,nlpd,-1.35376 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern3_2/quadratic,pylibkriging,fit_s,0.00313497 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern3_2/quadratic,pylibkriging,rmse,0.215336 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern3_2/quadratic,pylibkriging,q2,0.982117 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern3_2/quadratic,pylibkriging,nlpd,-0.350837 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern5_2/constant,pylibkriging,fit_s,0.000642776 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern5_2/constant,pylibkriging,rmse,0.0246348 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern5_2/constant,pylibkriging,q2,0.999766 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern5_2/constant,pylibkriging,nlpd,-2.59598 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern5_2/constant,pylibkriging BFGS10,rmse,0.0246338 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern5_2/constant,pylibkriging BFGS10,q2,0.999766 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern5_2/constant,pylibkriging BFGS10,nlpd,-2.59598 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern5_2/linear,pylibkriging,fit_s,0.000878334 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern5_2/linear,pylibkriging,rmse,0.0232483 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern5_2/linear,pylibkriging,q2,0.999792 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern5_2/linear,pylibkriging,nlpd,-2.58843 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern5_2/interactive,pylibkriging,fit_s,0.000845909 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern5_2/interactive,pylibkriging,rmse,0.0230248 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern5_2/interactive,pylibkriging,q2,0.999796 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern5_2/interactive,pylibkriging,nlpd,-2.58839 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern5_2/quadratic,pylibkriging,fit_s,0.00176644 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern5_2/quadratic,pylibkriging,rmse,0.209802 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern5_2/quadratic,pylibkriging,q2,0.983025 +python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern5_2/quadratic,pylibkriging,nlpd,-0.403998 +python_Kriging_kernels-trends-interpolation,python,ishigami/gauss/constant,pylibkriging,fit_s,0.0820026 +python_Kriging_kernels-trends-interpolation,python,ishigami/gauss/constant,pylibkriging,rmse,1.41975 +python_Kriging_kernels-trends-interpolation,python,ishigami/gauss/constant,pylibkriging,q2,0.844642 +python_Kriging_kernels-trends-interpolation,python,ishigami/gauss/constant,pylibkriging,nlpd,2.25894 +python_Kriging_kernels-trends-interpolation,python,ishigami/gauss/linear,pylibkriging,fit_s,0.0247335 +python_Kriging_kernels-trends-interpolation,python,ishigami/gauss/linear,pylibkriging,rmse,2.84579 +python_Kriging_kernels-trends-interpolation,python,ishigami/gauss/linear,pylibkriging,q2,0.375811 +python_Kriging_kernels-trends-interpolation,python,ishigami/gauss/linear,pylibkriging,nlpd,3.41805 +python_Kriging_kernels-trends-interpolation,python,ishigami/gauss/interactive,pylibkriging,fit_s,0.0169995 +python_Kriging_kernels-trends-interpolation,python,ishigami/gauss/interactive,pylibkriging,rmse,2.85366 +python_Kriging_kernels-trends-interpolation,python,ishigami/gauss/interactive,pylibkriging,q2,0.372353 +python_Kriging_kernels-trends-interpolation,python,ishigami/gauss/interactive,pylibkriging,nlpd,3.28643 +python_Kriging_kernels-trends-interpolation,python,ishigami/gauss/quadratic,pylibkriging,fit_s,0.00751019 +python_Kriging_kernels-trends-interpolation,python,ishigami/gauss/quadratic,pylibkriging,rmse,2.08982 +python_Kriging_kernels-trends-interpolation,python,ishigami/gauss/quadratic,pylibkriging,q2,0.66339 +python_Kriging_kernels-trends-interpolation,python,ishigami/gauss/quadratic,pylibkriging,nlpd,2.80099 +python_Kriging_kernels-trends-interpolation,python,ishigami/exp/constant,pylibkriging,fit_s,0.00825953 +python_Kriging_kernels-trends-interpolation,python,ishigami/exp/constant,pylibkriging,rmse,1.41632 +python_Kriging_kernels-trends-interpolation,python,ishigami/exp/constant,pylibkriging,q2,0.845391 +python_Kriging_kernels-trends-interpolation,python,ishigami/exp/constant,pylibkriging,nlpd,1.75239 +python_Kriging_kernels-trends-interpolation,python,ishigami/exp/linear,pylibkriging,fit_s,0.00724149 +python_Kriging_kernels-trends-interpolation,python,ishigami/exp/linear,pylibkriging,rmse,1.4095 +python_Kriging_kernels-trends-interpolation,python,ishigami/exp/linear,pylibkriging,q2,0.846877 +python_Kriging_kernels-trends-interpolation,python,ishigami/exp/linear,pylibkriging,nlpd,1.74779 +python_Kriging_kernels-trends-interpolation,python,ishigami/exp/interactive,pylibkriging,fit_s,0.00750446 +python_Kriging_kernels-trends-interpolation,python,ishigami/exp/interactive,pylibkriging,rmse,1.41511 +python_Kriging_kernels-trends-interpolation,python,ishigami/exp/interactive,pylibkriging,q2,0.845657 +python_Kriging_kernels-trends-interpolation,python,ishigami/exp/interactive,pylibkriging,nlpd,1.75024 +python_Kriging_kernels-trends-interpolation,python,ishigami/exp/quadratic,pylibkriging,fit_s,0.00750661 +python_Kriging_kernels-trends-interpolation,python,ishigami/exp/quadratic,pylibkriging,rmse,1.40329 +python_Kriging_kernels-trends-interpolation,python,ishigami/exp/quadratic,pylibkriging,q2,0.848223 +python_Kriging_kernels-trends-interpolation,python,ishigami/exp/quadratic,pylibkriging,nlpd,1.73254 +python_Kriging_kernels-trends-interpolation,python,ishigami/matern3_2/constant,pylibkriging,fit_s,0.0483243 +python_Kriging_kernels-trends-interpolation,python,ishigami/matern3_2/constant,pylibkriging,rmse,1.32974 +python_Kriging_kernels-trends-interpolation,python,ishigami/matern3_2/constant,pylibkriging,q2,0.863717 +python_Kriging_kernels-trends-interpolation,python,ishigami/matern3_2/constant,pylibkriging,nlpd,1.50266 +python_Kriging_kernels-trends-interpolation,python,ishigami/matern3_2/linear,pylibkriging,fit_s,0.0462224 +python_Kriging_kernels-trends-interpolation,python,ishigami/matern3_2/linear,pylibkriging,rmse,1.30244 +python_Kriging_kernels-trends-interpolation,python,ishigami/matern3_2/linear,pylibkriging,q2,0.869255 +python_Kriging_kernels-trends-interpolation,python,ishigami/matern3_2/linear,pylibkriging,nlpd,1.47906 +python_Kriging_kernels-trends-interpolation,python,ishigami/matern3_2/interactive,pylibkriging,fit_s,0.0462408 +python_Kriging_kernels-trends-interpolation,python,ishigami/matern3_2/interactive,pylibkriging,rmse,1.30591 +python_Kriging_kernels-trends-interpolation,python,ishigami/matern3_2/interactive,pylibkriging,q2,0.868558 +python_Kriging_kernels-trends-interpolation,python,ishigami/matern3_2/interactive,pylibkriging,nlpd,1.47384 +python_Kriging_kernels-trends-interpolation,python,ishigami/matern3_2/quadratic,pylibkriging,fit_s,0.0412879 +python_Kriging_kernels-trends-interpolation,python,ishigami/matern3_2/quadratic,pylibkriging,rmse,1.32197 +python_Kriging_kernels-trends-interpolation,python,ishigami/matern3_2/quadratic,pylibkriging,q2,0.865305 +python_Kriging_kernels-trends-interpolation,python,ishigami/matern3_2/quadratic,pylibkriging,nlpd,1.46827 +python_Kriging_kernels-trends-interpolation,python,ishigami/matern5_2/constant,pylibkriging,fit_s,0.02212 +python_Kriging_kernels-trends-interpolation,python,ishigami/matern5_2/constant,pylibkriging,rmse,3.00566 +python_Kriging_kernels-trends-interpolation,python,ishigami/matern5_2/constant,pylibkriging,q2,0.303709 +python_Kriging_kernels-trends-interpolation,python,ishigami/matern5_2/constant,pylibkriging,nlpd,3.42085 +python_Kriging_kernels-trends-interpolation,python,ishigami/matern5_2/constant,pylibkriging BFGS10,rmse,1.33903 +python_Kriging_kernels-trends-interpolation,python,ishigami/matern5_2/constant,pylibkriging BFGS10,q2,0.861806 +python_Kriging_kernels-trends-interpolation,python,ishigami/matern5_2/constant,pylibkriging BFGS10,nlpd,1.66132 +python_Kriging_kernels-trends-interpolation,python,ishigami/matern5_2/linear,pylibkriging,fit_s,0.0472796 +python_Kriging_kernels-trends-interpolation,python,ishigami/matern5_2/linear,pylibkriging,rmse,1.3434 +python_Kriging_kernels-trends-interpolation,python,ishigami/matern5_2/linear,pylibkriging,q2,0.860901 +python_Kriging_kernels-trends-interpolation,python,ishigami/matern5_2/linear,pylibkriging,nlpd,1.65103 +python_Kriging_kernels-trends-interpolation,python,ishigami/matern5_2/interactive,pylibkriging,fit_s,0.0477624 +python_Kriging_kernels-trends-interpolation,python,ishigami/matern5_2/interactive,pylibkriging,rmse,1.34479 +python_Kriging_kernels-trends-interpolation,python,ishigami/matern5_2/interactive,pylibkriging,q2,0.860615 +python_Kriging_kernels-trends-interpolation,python,ishigami/matern5_2/interactive,pylibkriging,nlpd,1.64153 +python_Kriging_kernels-trends-interpolation,python,ishigami/matern5_2/quadratic,pylibkriging,fit_s,0.0483804 +python_Kriging_kernels-trends-interpolation,python,ishigami/matern5_2/quadratic,pylibkriging,rmse,1.37082 +python_Kriging_kernels-trends-interpolation,python,ishigami/matern5_2/quadratic,pylibkriging,q2,0.855166 +python_Kriging_kernels-trends-interpolation,python,ishigami/matern5_2/quadratic,pylibkriging,nlpd,1.62228 +python_Kriging_kernels-trends-interpolation,python,hartmann3/gauss/constant,pylibkriging,fit_s,0.00703788 +python_Kriging_kernels-trends-interpolation,python,hartmann3/gauss/constant,pylibkriging,rmse,0.156226 +python_Kriging_kernels-trends-interpolation,python,hartmann3/gauss/constant,pylibkriging,q2,0.971116 +python_Kriging_kernels-trends-interpolation,python,hartmann3/gauss/constant,pylibkriging,nlpd,-0.555623 +python_Kriging_kernels-trends-interpolation,python,hartmann3/gauss/linear,pylibkriging,fit_s,0.0116901 +python_Kriging_kernels-trends-interpolation,python,hartmann3/gauss/linear,pylibkriging,rmse,0.153297 +python_Kriging_kernels-trends-interpolation,python,hartmann3/gauss/linear,pylibkriging,q2,0.972189 +python_Kriging_kernels-trends-interpolation,python,hartmann3/gauss/linear,pylibkriging,nlpd,-0.568404 +python_Kriging_kernels-trends-interpolation,python,hartmann3/gauss/interactive,pylibkriging,fit_s,0.00962472 +python_Kriging_kernels-trends-interpolation,python,hartmann3/gauss/interactive,pylibkriging,rmse,0.712947 +python_Kriging_kernels-trends-interpolation,python,hartmann3/gauss/interactive,pylibkriging,q2,0.398455 +python_Kriging_kernels-trends-interpolation,python,hartmann3/gauss/interactive,pylibkriging,nlpd,1.74554 +python_Kriging_kernels-trends-interpolation,python,hartmann3/gauss/quadratic,pylibkriging,fit_s,0.00845146 +python_Kriging_kernels-trends-interpolation,python,hartmann3/gauss/quadratic,pylibkriging,rmse,0.73238 +python_Kriging_kernels-trends-interpolation,python,hartmann3/gauss/quadratic,pylibkriging,q2,0.365216 +python_Kriging_kernels-trends-interpolation,python,hartmann3/gauss/quadratic,pylibkriging,nlpd,2.14588 +python_Kriging_kernels-trends-interpolation,python,hartmann3/exp/constant,pylibkriging,fit_s,0.00484872 +python_Kriging_kernels-trends-interpolation,python,hartmann3/exp/constant,pylibkriging,rmse,0.347348 +python_Kriging_kernels-trends-interpolation,python,hartmann3/exp/constant,pylibkriging,q2,0.857215 +python_Kriging_kernels-trends-interpolation,python,hartmann3/exp/constant,pylibkriging,nlpd,0.352683 +python_Kriging_kernels-trends-interpolation,python,hartmann3/exp/linear,pylibkriging,fit_s,0.00318956 +python_Kriging_kernels-trends-interpolation,python,hartmann3/exp/linear,pylibkriging,rmse,0.347273 +python_Kriging_kernels-trends-interpolation,python,hartmann3/exp/linear,pylibkriging,q2,0.857276 +python_Kriging_kernels-trends-interpolation,python,hartmann3/exp/linear,pylibkriging,nlpd,0.343631 +python_Kriging_kernels-trends-interpolation,python,hartmann3/exp/interactive,pylibkriging,fit_s,0.00330567 +python_Kriging_kernels-trends-interpolation,python,hartmann3/exp/interactive,pylibkriging,rmse,0.378349 +python_Kriging_kernels-trends-interpolation,python,hartmann3/exp/interactive,pylibkriging,q2,0.83059 +python_Kriging_kernels-trends-interpolation,python,hartmann3/exp/interactive,pylibkriging,nlpd,0.385842 +python_Kriging_kernels-trends-interpolation,python,hartmann3/exp/quadratic,pylibkriging,fit_s,0.00355411 +python_Kriging_kernels-trends-interpolation,python,hartmann3/exp/quadratic,pylibkriging,rmse,0.355119 +python_Kriging_kernels-trends-interpolation,python,hartmann3/exp/quadratic,pylibkriging,q2,0.850755 +python_Kriging_kernels-trends-interpolation,python,hartmann3/exp/quadratic,pylibkriging,nlpd,0.334371 +python_Kriging_kernels-trends-interpolation,python,hartmann3/matern3_2/constant,pylibkriging,fit_s,0.0107734 +python_Kriging_kernels-trends-interpolation,python,hartmann3/matern3_2/constant,pylibkriging,rmse,0.227521 +python_Kriging_kernels-trends-interpolation,python,hartmann3/matern3_2/constant,pylibkriging,q2,0.938737 +python_Kriging_kernels-trends-interpolation,python,hartmann3/matern3_2/constant,pylibkriging,nlpd,-0.181207 +python_Kriging_kernels-trends-interpolation,python,hartmann3/matern3_2/linear,pylibkriging,fit_s,0.0118749 +python_Kriging_kernels-trends-interpolation,python,hartmann3/matern3_2/linear,pylibkriging,rmse,0.23038 +python_Kriging_kernels-trends-interpolation,python,hartmann3/matern3_2/linear,pylibkriging,q2,0.937188 +python_Kriging_kernels-trends-interpolation,python,hartmann3/matern3_2/linear,pylibkriging,nlpd,-0.162203 +python_Kriging_kernels-trends-interpolation,python,hartmann3/matern3_2/interactive,pylibkriging,fit_s,0.00637007 +python_Kriging_kernels-trends-interpolation,python,hartmann3/matern3_2/interactive,pylibkriging,rmse,0.315752 +python_Kriging_kernels-trends-interpolation,python,hartmann3/matern3_2/interactive,pylibkriging,q2,0.88201 +python_Kriging_kernels-trends-interpolation,python,hartmann3/matern3_2/interactive,pylibkriging,nlpd,0.898253 +python_Kriging_kernels-trends-interpolation,python,hartmann3/matern3_2/quadratic,pylibkriging,fit_s,0.00598764 +python_Kriging_kernels-trends-interpolation,python,hartmann3/matern3_2/quadratic,pylibkriging,rmse,0.291393 +python_Kriging_kernels-trends-interpolation,python,hartmann3/matern3_2/quadratic,pylibkriging,q2,0.899513 +python_Kriging_kernels-trends-interpolation,python,hartmann3/matern3_2/quadratic,pylibkriging,nlpd,0.53743 +python_Kriging_kernels-trends-interpolation,python,hartmann3/matern5_2/constant,pylibkriging,fit_s,0.0100911 +python_Kriging_kernels-trends-interpolation,python,hartmann3/matern5_2/constant,pylibkriging,rmse,0.190287 +python_Kriging_kernels-trends-interpolation,python,hartmann3/matern5_2/constant,pylibkriging,q2,0.957148 +python_Kriging_kernels-trends-interpolation,python,hartmann3/matern5_2/constant,pylibkriging,nlpd,-0.403054 +python_Kriging_kernels-trends-interpolation,python,hartmann3/matern5_2/constant,pylibkriging BFGS10,rmse,0.190286 +python_Kriging_kernels-trends-interpolation,python,hartmann3/matern5_2/constant,pylibkriging BFGS10,q2,0.957148 +python_Kriging_kernels-trends-interpolation,python,hartmann3/matern5_2/constant,pylibkriging BFGS10,nlpd,-0.40306 +python_Kriging_kernels-trends-interpolation,python,hartmann3/matern5_2/linear,pylibkriging,fit_s,0.0106812 +python_Kriging_kernels-trends-interpolation,python,hartmann3/matern5_2/linear,pylibkriging,rmse,0.189633 +python_Kriging_kernels-trends-interpolation,python,hartmann3/matern5_2/linear,pylibkriging,q2,0.957442 +python_Kriging_kernels-trends-interpolation,python,hartmann3/matern5_2/linear,pylibkriging,nlpd,-0.410075 +python_Kriging_kernels-trends-interpolation,python,hartmann3/matern5_2/interactive,pylibkriging,fit_s,0.00839925 +python_Kriging_kernels-trends-interpolation,python,hartmann3/matern5_2/interactive,pylibkriging,rmse,0.307114 +python_Kriging_kernels-trends-interpolation,python,hartmann3/matern5_2/interactive,pylibkriging,q2,0.888377 +python_Kriging_kernels-trends-interpolation,python,hartmann3/matern5_2/interactive,pylibkriging,nlpd,1.50063 +python_Kriging_kernels-trends-interpolation,python,hartmann3/matern5_2/quadratic,pylibkriging,fit_s,0.00724792 +python_Kriging_kernels-trends-interpolation,python,hartmann3/matern5_2/quadratic,pylibkriging,rmse,0.285355 +python_Kriging_kernels-trends-interpolation,python,hartmann3/matern5_2/quadratic,pylibkriging,q2,0.903634 +python_Kriging_kernels-trends-interpolation,python,hartmann3/matern5_2/quadratic,pylibkriging,nlpd,0.898128 +python_Kriging_objectives-LL-LOO-LMP-gradients,python,branin/LL,pylibkriging,rmse,71.4801 +python_Kriging_objectives-LL-LOO-LMP-gradients,python,branin/LL,pylibkriging,q2,-1991.62 +python_Kriging_objectives-LL-LOO-LMP-gradients,python,branin/LL,pylibkriging,nlpd,13603.3 +python_Kriging_objectives-LL-LOO-LMP-gradients,python,branin/LOO,pylibkriging,rmse,71.5631 +python_Kriging_objectives-LL-LOO-LMP-gradients,python,branin/LOO,pylibkriging,q2,-1996.25 +python_Kriging_objectives-LL-LOO-LMP-gradients,python,branin/LOO,pylibkriging,nlpd,18836.2 +python_Kriging_objectives-LL-LOO-LMP-gradients,python,branin/LMP,pylibkriging,rmse,71.5295 +python_Kriging_objectives-LL-LOO-LMP-gradients,python,branin/LMP,pylibkriging,q2,-1994.38 +python_Kriging_objectives-LL-LOO-LMP-gradients,python,branin/LMP,pylibkriging,nlpd,14603.3 +python_Kriging_objectives-LL-LOO-LMP-gradients,python,hartmann3/LL,pylibkriging,rmse,1.30952 +python_Kriging_objectives-LL-LOO-LMP-gradients,python,hartmann3/LL,pylibkriging,q2,-2013.29 +python_Kriging_objectives-LL-LOO-LMP-gradients,python,hartmann3/LL,pylibkriging,nlpd,41.7128 +python_Kriging_objectives-LL-LOO-LMP-gradients,python,hartmann3/LOO,pylibkriging,rmse,1.31436 +python_Kriging_objectives-LL-LOO-LMP-gradients,python,hartmann3/LOO,pylibkriging,q2,-2028.22 +python_Kriging_objectives-LL-LOO-LMP-gradients,python,hartmann3/LOO,pylibkriging,nlpd,49.7135 +python_Kriging_objectives-LL-LOO-LMP-gradients,python,hartmann3/LMP,pylibkriging,rmse,1.31148 +python_Kriging_objectives-LL-LOO-LMP-gradients,python,hartmann3/LMP,pylibkriging,q2,-2019.34 +python_Kriging_objectives-LL-LOO-LMP-gradients,python,hartmann3/LMP,pylibkriging,nlpd,43.397 +python_Kriging_objectives-LL-LOO-LMP-gradients,python,ishigami/LL,pylibkriging,rmse,5.11103 +python_Kriging_objectives-LL-LOO-LMP-gradients,python,ishigami/LL,pylibkriging,q2,-1783.42 +python_Kriging_objectives-LL-LOO-LMP-gradients,python,ishigami/LL,pylibkriging,nlpd,10.4546 +python_Kriging_objectives-LL-LOO-LMP-gradients,python,ishigami/LOO,pylibkriging,rmse,5.81789 +python_Kriging_objectives-LL-LOO-LMP-gradients,python,ishigami/LOO,pylibkriging,q2,-2311.12 +python_Kriging_objectives-LL-LOO-LMP-gradients,python,ishigami/LOO,pylibkriging,nlpd,40.8323 +python_Kriging_objectives-LL-LOO-LMP-gradients,python,ishigami/LMP,pylibkriging,rmse,5.18495 +python_Kriging_objectives-LL-LOO-LMP-gradients,python,ishigami/LMP,pylibkriging,q2,-1835.41 +python_Kriging_objectives-LL-LOO-LMP-gradients,python,ishigami/LMP,pylibkriging,nlpd,11.5894 +python_Kriging_objectives-LL-LOO-LMP-gradients,python,friedman/LL,pylibkriging,rmse,6.87533 +python_Kriging_objectives-LL-LOO-LMP-gradients,python,friedman/LL,pylibkriging,q2,-1991.33 +python_Kriging_objectives-LL-LOO-LMP-gradients,python,friedman/LL,pylibkriging,nlpd,651.706 +python_Kriging_objectives-LL-LOO-LMP-gradients,python,friedman/LOO,pylibkriging,rmse,6.87323 +python_Kriging_objectives-LL-LOO-LMP-gradients,python,friedman/LOO,pylibkriging,q2,-1990.12 +python_Kriging_objectives-LL-LOO-LMP-gradients,python,friedman/LOO,pylibkriging,nlpd,484.115 +python_Kriging_objectives-LL-LOO-LMP-gradients,python,friedman/LMP,pylibkriging,rmse,6.87647 +python_Kriging_objectives-LL-LOO-LMP-gradients,python,friedman/LMP,pylibkriging,q2,-1991.99 +python_Kriging_objectives-LL-LOO-LMP-gradients,python,friedman/LMP,pylibkriging,nlpd,658.808 +python_Kriging-nugget-noise_noisy-data,python,forrester,nugget,rmse,0.198091 +python_Kriging-nugget-noise_noisy-data,python,forrester,nugget,q2,0.998087 +python_Kriging-nugget-noise_noisy-data,python,forrester,nugget,nlpd,0.286631 +python_Kriging-nugget-noise_noisy-data,python,forrester,noise-free fit,rmse,0.487916 +python_Kriging-nugget-noise_noisy-data,python,forrester,noise-free fit,q2,0.988392 +python_Kriging-nugget-noise_noisy-data,python,forrester,noise-free fit,nlpd,862.687 +python_Kriging-nugget-noise_noisy-data,python,friedman,nugget,rmse,0.467335 +python_Kriging-nugget-noise_noisy-data,python,friedman,nugget,q2,0.99089 +python_Kriging-nugget-noise_noisy-data,python,friedman,nugget,nlpd,1.07073 +python_Kriging-nugget-noise_noisy-data,python,friedman,noise-free fit,rmse,1.27156 +python_Kriging-nugget-noise_noisy-data,python,friedman,noise-free fit,q2,0.932559 +python_Kriging-nugget-noise_noisy-data,python,friedman,noise-free fit,nlpd,1.64495 +python_Kriging-nugget-noise_noisy-data,python,gramacy_lee_2012/hetero,known variances,rmse,0.168741 +python_Kriging-nugget-noise_noisy-data,python,gramacy_lee_2012/hetero,known variances,q2,0.982515 +python_Kriging-nugget-noise_noisy-data,python,gramacy_lee_2012/hetero,known variances,nlpd,-0.570574 +python_Kriging-nugget-noise_noisy-data,python,gramacy_lee_2012/hetero,nugget,rmse,0.170945 +python_Kriging-nugget-noise_noisy-data,python,gramacy_lee_2012/hetero,nugget,q2,0.982055 +python_Kriging-nugget-noise_noisy-data,python,gramacy_lee_2012/hetero,nugget,nlpd,-0.221568 +python_WarpKriging_warpings-categorical-ordinal,python,gl2012,Kriging,rmse,0.167263 +python_WarpKriging_warpings-categorical-ordinal,python,gl2012,Kriging,q2,0.981176 +python_WarpKriging_warpings-categorical-ordinal,python,gl2012,Kriging,nlpd,-1.08917 +python_WarpKriging_warpings-categorical-ordinal,python,gl2012,affine,rmse,0.167263 +python_WarpKriging_warpings-categorical-ordinal,python,gl2012,affine,q2,0.981176 +python_WarpKriging_warpings-categorical-ordinal,python,gl2012,affine,nlpd,-1.08917 +python_WarpKriging_warpings-categorical-ordinal,python,gl2012,boxcox,rmse,0.16759 +python_WarpKriging_warpings-categorical-ordinal,python,gl2012,boxcox,q2,0.981103 +python_WarpKriging_warpings-categorical-ordinal,python,gl2012,boxcox,nlpd,282.422 +python_WarpKriging_warpings-categorical-ordinal,python,gl2012,kumaraswamy,rmse,0.168286 +python_WarpKriging_warpings-categorical-ordinal,python,gl2012,kumaraswamy,q2,0.980945 +python_WarpKriging_warpings-categorical-ordinal,python,gl2012,kumaraswamy,nlpd,2.04577e+06 +python_WarpKriging_warpings-categorical-ordinal,python,gl2012,knots(5),rmse,0.16791 +python_WarpKriging_warpings-categorical-ordinal,python,gl2012,knots(5),q2,0.98103 +python_WarpKriging_warpings-categorical-ordinal,python,gl2012,knots(5),nlpd,2.04376e+06 +python_WarpKriging_warpings-categorical-ordinal,python,gl2012,neural_mono(8),rmse,0.170824 +python_WarpKriging_warpings-categorical-ordinal,python,gl2012,neural_mono(8),q2,0.980366 +python_WarpKriging_warpings-categorical-ordinal,python,gl2012,neural_mono(8),nlpd,-0.265141 +python_WarpKriging_warpings-categorical-ordinal,python,gl2008,Kriging,rmse,0.0534649 +python_WarpKriging_warpings-categorical-ordinal,python,gl2008,Kriging,q2,0.585636 +python_WarpKriging_warpings-categorical-ordinal,python,gl2008,Kriging,nlpd,-1.74569 +python_WarpKriging_warpings-categorical-ordinal,python,gl2008,kumaraswamy,rmse,0.0535298 +python_WarpKriging_warpings-categorical-ordinal,python,gl2008,kumaraswamy,q2,0.58463 +python_WarpKriging_warpings-categorical-ordinal,python,gl2008,kumaraswamy,nlpd,-1.74552 +python_WarpKriging_warpings-categorical-ordinal,python,mixed,Kriging (level as number),rmse,16.7693 +python_WarpKriging_warpings-categorical-ordinal,python,mixed,Kriging (level as number),q2,0.838453 +python_WarpKriging_warpings-categorical-ordinal,python,mixed,Kriging (level as number),nlpd,3.81063 +python_WarpKriging_warpings-categorical-ordinal,python,mixed,"categorical(5,2)",rmse,16.1438 +python_WarpKriging_warpings-categorical-ordinal,python,mixed,"categorical(5,2)",q2,0.85028 +python_WarpKriging_warpings-categorical-ordinal,python,mixed,"categorical(5,2)",nlpd,3.3206 +python_WarpKriging_warpings-categorical-ordinal,python,ordinal,Kriging (rank as number),rmse,0.213845 +python_WarpKriging_warpings-categorical-ordinal,python,ordinal,Kriging (rank as number),q2,0.953177 +python_WarpKriging_warpings-categorical-ordinal,python,ordinal,Kriging (rank as number),nlpd,0.810969 +python_WarpKriging_warpings-categorical-ordinal,python,ordinal,ordinal(6),rmse,0.213043 +python_WarpKriging_warpings-categorical-ordinal,python,ordinal,ordinal(6),q2,0.953528 +python_WarpKriging_warpings-categorical-ordinal,python,ordinal,ordinal(6),nlpd,0.78888 +python_Kriging_high-dimension-normalize-screening,python,otl_circuit,unit cube,fit_s,0.00796986 +python_Kriging_high-dimension-normalize-screening,python,otl_circuit,unit cube,rmse,0.0150514 +python_Kriging_high-dimension-normalize-screening,python,otl_circuit,unit cube,q2,0.999836 +python_Kriging_high-dimension-normalize-screening,python,otl_circuit,unit cube,nlpd,-2.90452 +python_Kriging_high-dimension-normalize-screening,python,otl_circuit,native + normalize,rmse,0.0150514 +python_Kriging_high-dimension-normalize-screening,python,otl_circuit,native + normalize,q2,0.999836 +python_Kriging_high-dimension-normalize-screening,python,otl_circuit,native + normalize,nlpd,-2.90452 +python_Kriging_high-dimension-normalize-screening,python,piston,unit cube,fit_s,0.023371 +python_Kriging_high-dimension-normalize-screening,python,piston,unit cube,rmse,0.0104258 +python_Kriging_high-dimension-normalize-screening,python,piston,unit cube,q2,0.994462 +python_Kriging_high-dimension-normalize-screening,python,piston,unit cube,nlpd,-3.49038 +python_Kriging_high-dimension-normalize-screening,python,piston,native + normalize,rmse,0.0104258 +python_Kriging_high-dimension-normalize-screening,python,piston,native + normalize,q2,0.994462 +python_Kriging_high-dimension-normalize-screening,python,piston,native + normalize,nlpd,-3.49038 +python_Kriging_high-dimension-normalize-screening,python,borehole,unit cube,fit_s,0.0214174 +python_Kriging_high-dimension-normalize-screening,python,borehole,unit cube,rmse,0.893341 +python_Kriging_high-dimension-normalize-screening,python,borehole,unit cube,q2,0.999599 +python_Kriging_high-dimension-normalize-screening,python,borehole,unit cube,nlpd,1.08433 +python_Kriging_high-dimension-normalize-screening,python,borehole,native + normalize,rmse,0.893341 +python_Kriging_high-dimension-normalize-screening,python,borehole,native + normalize,q2,0.999599 +python_Kriging_high-dimension-normalize-screening,python,borehole,native + normalize,nlpd,1.08433 +python_Kriging_high-dimension-normalize-screening,python,robot_arm,unit cube,fit_s,0.187081 +python_Kriging_high-dimension-normalize-screening,python,robot_arm,unit cube,rmse,0.358507 +python_Kriging_high-dimension-normalize-screening,python,robot_arm,unit cube,q2,0.571154 +python_Kriging_high-dimension-normalize-screening,python,robot_arm,unit cube,nlpd,0.355106 +python_Kriging_high-dimension-normalize-screening,python,robot_arm,native + normalize,rmse,0.358507 +python_Kriging_high-dimension-normalize-screening,python,robot_arm,native + normalize,q2,0.571154 +python_Kriging_high-dimension-normalize-screening,python,robot_arm,native + normalize,nlpd,0.355106 +python_Kriging_high-dimension-normalize-screening,python,dette_pepelyshev,unit cube,fit_s,0.0181198 +python_Kriging_high-dimension-normalize-screening,python,dette_pepelyshev,unit cube,rmse,0.424908 +python_Kriging_high-dimension-normalize-screening,python,dette_pepelyshev,unit cube,q2,0.997757 +python_Kriging_high-dimension-normalize-screening,python,dette_pepelyshev,unit cube,nlpd,0.510276 +python_Kriging_high-dimension-normalize-screening,python,dette_pepelyshev,native + normalize,rmse,0.424908 +python_Kriging_high-dimension-normalize-screening,python,dette_pepelyshev,native + normalize,q2,0.997757 +python_Kriging_high-dimension-normalize-screening,python,dette_pepelyshev,native + normalize,nlpd,0.510276 +python_Kriging_high-dimension-normalize-screening,python,wing_weight,unit cube,fit_s,0.0303686 +python_Kriging_high-dimension-normalize-screening,python,wing_weight,unit cube,rmse,0.809743 +python_Kriging_high-dimension-normalize-screening,python,wing_weight,unit cube,q2,0.999716 +python_Kriging_high-dimension-normalize-screening,python,wing_weight,unit cube,nlpd,1.07161 +python_Kriging_high-dimension-normalize-screening,python,wing_weight,native + normalize,rmse,0.809743 +python_Kriging_high-dimension-normalize-screening,python,wing_weight,native + normalize,q2,0.999716 +python_Kriging_high-dimension-normalize-screening,python,wing_weight,native + normalize,nlpd,1.07161 +python_Kriging_high-dimension-normalize-screening,python,welch,unit cube,fit_s,0.500499 +python_Kriging_high-dimension-normalize-screening,python,welch,unit cube,rmse,0.167729 +python_Kriging_high-dimension-normalize-screening,python,welch,unit cube,q2,0.993452 +python_Kriging_high-dimension-normalize-screening,python,welch,unit cube,nlpd,-0.429138 +python_Kriging_high-dimension-normalize-screening,python,welch,native + normalize,rmse,0.167692 +python_Kriging_high-dimension-normalize-screening,python,welch,native + normalize,q2,0.993454 +python_Kriging_high-dimension-normalize-screening,python,welch,native + normalize,nlpd,-0.429291 +python_MLPKriging_deep-kernel-projected-branin,python,projected_branin_10d,Kriging,rmse,4.42284 +python_MLPKriging_deep-kernel-projected-branin,python,projected_branin_10d,Kriging,q2,0.959122 +python_MLPKriging_deep-kernel-projected-branin,python,projected_branin_10d,Kriging,nlpd,2.89997 +python_MLPKriging_deep-kernel-projected-branin,python,projected_branin_10d,MLPKriging tanh BFGS,fit_s,0.171914 +python_MLPKriging_deep-kernel-projected-branin,python,projected_branin_10d,MLPKriging tanh BFGS,rmse,59.5068 +python_MLPKriging_deep-kernel-projected-branin,python,projected_branin_10d,MLPKriging tanh BFGS,q2,-6.39973 +python_MLPKriging_deep-kernel-projected-branin,python,projected_branin_10d,MLPKriging tanh BFGS,nlpd,100.745 +python_MLPKriging_deep-kernel-projected-branin,python,projected_branin_10d,MLPKriging tanh BFGS10,fit_s,0.53444 +python_MLPKriging_deep-kernel-projected-branin,python,projected_branin_10d,MLPKriging tanh BFGS10,rmse,4.41161 +python_MLPKriging_deep-kernel-projected-branin,python,projected_branin_10d,MLPKriging tanh BFGS10,q2,0.95933 +python_MLPKriging_deep-kernel-projected-branin,python,projected_branin_10d,MLPKriging tanh BFGS10,nlpd,28.4236 +python_MLPKriging_deep-kernel-projected-branin,python,projected_branin_10d,MLPKriging selu BFGS,fit_s,0.145419 +python_MLPKriging_deep-kernel-projected-branin,python,projected_branin_10d,MLPKriging selu BFGS,rmse,58.9844 +python_MLPKriging_deep-kernel-projected-branin,python,projected_branin_10d,MLPKriging selu BFGS,q2,-6.27038 +python_MLPKriging_deep-kernel-projected-branin,python,projected_branin_10d,MLPKriging selu BFGS,nlpd,46.9347 +python_MLPKriging_deep-kernel-projected-branin,python,projected_branin_10d,MLPKriging selu BFGS10,fit_s,0.518803 +python_MLPKriging_deep-kernel-projected-branin,python,projected_branin_10d,MLPKriging selu BFGS10,rmse,3.79093 +python_MLPKriging_deep-kernel-projected-branin,python,projected_branin_10d,MLPKriging selu BFGS10,q2,0.969969 +python_MLPKriging_deep-kernel-projected-branin,python,projected_branin_10d,MLPKriging selu BFGS10,nlpd,12.6814 +python_Kriging-vecchia-nystrom_large-n,python,hartmann3/n=1500,exact,fit_s,11.2644 +python_NestedKriging_aggregations-large-n,python,hartmann3/n=1500,exact,fit_s,11.2644 +python_Kriging-vecchia-nystrom_large-n,python,hartmann3/n=1500,exact,rmse,0.000259233 +python_NestedKriging_aggregations-large-n,python,hartmann3/n=1500,exact,rmse,0.000259233 +python_Kriging-vecchia-nystrom_large-n,python,hartmann3/n=1500,exact,q2,1 +python_NestedKriging_aggregations-large-n,python,hartmann3/n=1500,exact,q2,1 +python_Kriging-vecchia-nystrom_large-n,python,hartmann3/n=1500,exact,nlpd,-6.92184 +python_NestedKriging_aggregations-large-n,python,hartmann3/n=1500,exact,nlpd,-6.92184 +python_NestedKriging_aggregations-large-n,python,hartmann3/n=1500,nested NK (20 groups),fit_s,0.324342 +python_NestedKriging_aggregations-large-n,python,hartmann3/n=1500,nested NK (20 groups),rmse,0.0029908 +python_NestedKriging_aggregations-large-n,python,hartmann3/n=1500,nested NK (20 groups),q2,0.999991 +python_NestedKriging_aggregations-large-n,python,hartmann3/n=1500,nested NK (20 groups),nlpd,-4.85946 +python_NestedKriging_aggregations-large-n,python,hartmann3/n=5000,nested NK (20 groups),fit_s,3.25297 +python_NestedKriging_aggregations-large-n,python,hartmann3/n=5000,nested NK (20 groups),rmse,0.000222655 +python_NestedKriging_aggregations-large-n,python,hartmann3/n=5000,nested NK (20 groups),q2,1 +python_NestedKriging_aggregations-large-n,python,hartmann3/n=5000,nested NK (20 groups),nlpd,-7.44453 +python_NestedKriging_aggregations-large-n,python,hartmann3/n=1500,nested rBCM (20 groups),fit_s,0.273419 +python_NestedKriging_aggregations-large-n,python,hartmann3/n=1500,nested rBCM (20 groups),rmse,0.00304431 +python_NestedKriging_aggregations-large-n,python,hartmann3/n=1500,nested rBCM (20 groups),q2,0.99999 +python_NestedKriging_aggregations-large-n,python,hartmann3/n=1500,nested rBCM (20 groups),nlpd,-4.50201 +python_NestedKriging_aggregations-large-n,python,hartmann3/n=5000,nested rBCM (20 groups),fit_s,2.67811 +python_NestedKriging_aggregations-large-n,python,hartmann3/n=5000,nested rBCM (20 groups),rmse,0.000224517 +python_NestedKriging_aggregations-large-n,python,hartmann3/n=5000,nested rBCM (20 groups),q2,1 +python_NestedKriging_aggregations-large-n,python,hartmann3/n=5000,nested rBCM (20 groups),nlpd,-7.18182 +python_Kriging-vecchia-nystrom_large-n,python,hartmann3/n=1500,subset of data (500),fit_s,0.685586 +python_Kriging-vecchia-nystrom_large-n,python,hartmann3/n=1500,subset of data (500),rmse,0.00269153 +python_Kriging-vecchia-nystrom_large-n,python,hartmann3/n=1500,subset of data (500),q2,0.999992 +python_Kriging-vecchia-nystrom_large-n,python,hartmann3/n=1500,subset of data (500),nlpd,-4.68641 +python_Kriging-vecchia-nystrom_large-n,python,hartmann3/n=5000,subset of data (500),fit_s,0.906366 +python_Kriging-vecchia-nystrom_large-n,python,hartmann3/n=5000,subset of data (500),rmse,0.00247552 +python_Kriging-vecchia-nystrom_large-n,python,hartmann3/n=5000,subset of data (500),q2,0.999994 +python_Kriging-vecchia-nystrom_large-n,python,hartmann3/n=5000,subset of data (500),nlpd,-4.71359 +python_Kriging-vecchia-nystrom_large-n,python,otl_circuit/n=1500,exact,fit_s,10.9405 +python_NestedKriging_aggregations-large-n,python,otl_circuit/n=1500,exact,fit_s,10.9405 +python_Kriging-vecchia-nystrom_large-n,python,otl_circuit/n=1500,exact,rmse,9.81598e-05 +python_NestedKriging_aggregations-large-n,python,otl_circuit/n=1500,exact,rmse,9.81598e-05 +python_Kriging-vecchia-nystrom_large-n,python,otl_circuit/n=1500,exact,q2,1 +python_NestedKriging_aggregations-large-n,python,otl_circuit/n=1500,exact,q2,1 +python_Kriging-vecchia-nystrom_large-n,python,otl_circuit/n=1500,exact,nlpd,-8.21962 +python_NestedKriging_aggregations-large-n,python,otl_circuit/n=1500,exact,nlpd,-8.21962 +python_NestedKriging_aggregations-large-n,python,otl_circuit/n=1500,nested NK (20 groups),fit_s,0.279359 +python_NestedKriging_aggregations-large-n,python,otl_circuit/n=1500,nested NK (20 groups),rmse,0.00226873 +python_NestedKriging_aggregations-large-n,python,otl_circuit/n=1500,nested NK (20 groups),q2,0.999996 +python_NestedKriging_aggregations-large-n,python,otl_circuit/n=1500,nested NK (20 groups),nlpd,-4.90542 +python_NestedKriging_aggregations-large-n,python,otl_circuit/n=5000,nested NK (20 groups),fit_s,3.08712 +python_NestedKriging_aggregations-large-n,python,otl_circuit/n=5000,nested NK (20 groups),rmse,0.000261686 +python_NestedKriging_aggregations-large-n,python,otl_circuit/n=5000,nested NK (20 groups),q2,1 +python_NestedKriging_aggregations-large-n,python,otl_circuit/n=5000,nested NK (20 groups),nlpd,-7.20623 +python_NestedKriging_aggregations-large-n,python,otl_circuit/n=1500,nested rBCM (20 groups),fit_s,0.251989 +python_NestedKriging_aggregations-large-n,python,otl_circuit/n=1500,nested rBCM (20 groups),rmse,0.00291869 +python_NestedKriging_aggregations-large-n,python,otl_circuit/n=1500,nested rBCM (20 groups),q2,0.999993 +python_NestedKriging_aggregations-large-n,python,otl_circuit/n=1500,nested rBCM (20 groups),nlpd,-2.55512 +python_NestedKriging_aggregations-large-n,python,otl_circuit/n=5000,nested rBCM (20 groups),fit_s,3.67768 +python_NestedKriging_aggregations-large-n,python,otl_circuit/n=5000,nested rBCM (20 groups),rmse,0.000278149 +python_NestedKriging_aggregations-large-n,python,otl_circuit/n=5000,nested rBCM (20 groups),q2,1 +python_NestedKriging_aggregations-large-n,python,otl_circuit/n=5000,nested rBCM (20 groups),nlpd,-4.34927 +python_Kriging-vecchia-nystrom_large-n,python,otl_circuit/n=1500,subset of data (500),fit_s,0.709236 +python_Kriging-vecchia-nystrom_large-n,python,otl_circuit/n=1500,subset of data (500),rmse,0.0004286 +python_Kriging-vecchia-nystrom_large-n,python,otl_circuit/n=1500,subset of data (500),q2,1 +python_Kriging-vecchia-nystrom_large-n,python,otl_circuit/n=1500,subset of data (500),nlpd,-6.72678 +python_Kriging-vecchia-nystrom_large-n,python,otl_circuit/n=5000,subset of data (500),fit_s,0.905534 +python_Kriging-vecchia-nystrom_large-n,python,otl_circuit/n=5000,subset of data (500),rmse,0.000446879 +python_Kriging-vecchia-nystrom_large-n,python,otl_circuit/n=5000,subset of data (500),q2,1 +python_Kriging-vecchia-nystrom_large-n,python,otl_circuit/n=5000,subset of data (500),nlpd,-6.78844 +python_NestedKriging_aggregations-large-n,python,branin/nested,NK,rmse,0.0343984 +python_NestedKriging_aggregations-large-n,python,branin/nested,NK,q2,1 +python_NestedKriging_aggregations-large-n,python,branin/nested,NK,nlpd,2.00083e+14 +python_NestedKriging_aggregations-large-n,python,branin/nested,PoE,rmse,0.0354265 +python_NestedKriging_aggregations-large-n,python,branin/nested,PoE,q2,1 +python_NestedKriging_aggregations-large-n,python,branin/nested,PoE,nlpd,-1.64533 +python_NestedKriging_aggregations-large-n,python,branin/nested,gPoE,rmse,0.0354265 +python_NestedKriging_aggregations-large-n,python,branin/nested,gPoE,q2,1 +python_NestedKriging_aggregations-large-n,python,branin/nested,gPoE,nlpd,-0.621836 +python_NestedKriging_aggregations-large-n,python,branin/nested,BCM,rmse,0.0354196 +python_NestedKriging_aggregations-large-n,python,branin/nested,BCM,q2,1 +python_NestedKriging_aggregations-large-n,python,branin/nested,BCM,nlpd,-1.64533 +python_NestedKriging_aggregations-large-n,python,branin/nested,rBCM,rmse,0.035287 +python_NestedKriging_aggregations-large-n,python,branin/nested,rBCM,q2,1 +python_NestedKriging_aggregations-large-n,python,branin/nested,rBCM,nlpd,-2.61369 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,branin,pylibkriging,q2,0.999944 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,branin,pylibkriging,nlpd,0.137225 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,branin,pylibkriging,fit_s,0.00151396 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,branin,pylibkriging BFGS10,q2,0.999945 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,branin,pylibkriging BFGS10,nlpd,0.137254 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,branin,pylibkriging BFGS10,fit_s,0.0161211 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,branin,scikit-learn,q2,0.999597 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,branin,scikit-learn,nlpd,0.928508 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,branin,scikit-learn,fit_s,0.147773 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,branin,SMT,q2,0.999944 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,branin,SMT,nlpd,0.137259 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,branin,SMT,fit_s,0.792991 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,branin,OpenTURNS,q2,0.998221 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,branin,OpenTURNS,nlpd,1.55599 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,branin,OpenTURNS,fit_s,0.00512815 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,six_hump_camel,pylibkriging,q2,0.999418 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,six_hump_camel,pylibkriging,nlpd,0.0429346 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,six_hump_camel,pylibkriging,fit_s,0.00358009 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,six_hump_camel,pylibkriging BFGS10,q2,0.999418 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,six_hump_camel,pylibkriging BFGS10,nlpd,0.0429347 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,six_hump_camel,pylibkriging BFGS10,fit_s,0.0500369 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,six_hump_camel,scikit-learn,q2,0.0163311 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,six_hump_camel,scikit-learn,nlpd,4.63763 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,six_hump_camel,scikit-learn,fit_s,0.156482 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,six_hump_camel,SMT,q2,0.999594 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,six_hump_camel,SMT,nlpd,-0.121007 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,six_hump_camel,SMT,fit_s,0.941661 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,six_hump_camel,OpenTURNS,q2,0.956713 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,six_hump_camel,OpenTURNS,nlpd,2.55446 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,six_hump_camel,OpenTURNS,fit_s,0.00930476 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann3,pylibkriging,q2,0.986148 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann3,pylibkriging,nlpd,-0.847247 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann3,pylibkriging,fit_s,0.023098 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann3,pylibkriging BFGS10,q2,0.986148 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann3,pylibkriging BFGS10,nlpd,-0.847249 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann3,pylibkriging BFGS10,fit_s,0.113887 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann3,scikit-learn,q2,0.983889 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann3,scikit-learn,nlpd,-0.782697 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann3,scikit-learn,fit_s,0.208279 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann3,SMT,q2,0.986148 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann3,SMT,nlpd,-0.847249 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann3,SMT,fit_s,1.35887 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann3,OpenTURNS,q2,0.984712 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann3,OpenTURNS,nlpd,-0.809851 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann3,OpenTURNS,fit_s,0.0169773 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,ishigami,pylibkriging,q2,0.259556 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,ishigami,pylibkriging,nlpd,4.38991 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,ishigami,pylibkriging,fit_s,0.0264924 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,ishigami,pylibkriging BFGS10,q2,0.935722 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,ishigami,pylibkriging BFGS10,nlpd,0.857443 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,ishigami,pylibkriging BFGS10,fit_s,0.322963 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,ishigami,scikit-learn,q2,0.831142 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,ishigami,scikit-learn,nlpd,1.5455 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,ishigami,scikit-learn,fit_s,0.512728 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,ishigami,SMT,q2,0.51196 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,ishigami,SMT,nlpd,3.31349 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,ishigami,SMT,fit_s,2.70852 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,ishigami,OpenTURNS,q2,0.887251 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,ishigami,OpenTURNS,nlpd,1.39348 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,ishigami,OpenTURNS,fit_s,0.0510182 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,friedman,pylibkriging,q2,0.999771 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,friedman,pylibkriging,nlpd,-1.29726 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,friedman,pylibkriging,fit_s,0.0165708 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,friedman,pylibkriging BFGS10,q2,0.999771 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,friedman,pylibkriging BFGS10,nlpd,-1.29727 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,friedman,pylibkriging BFGS10,fit_s,0.279774 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,friedman,scikit-learn,q2,0.999754 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,friedman,scikit-learn,nlpd,-1.37702 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,friedman,scikit-learn,fit_s,0.308127 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,friedman,SMT,q2,0.999899 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,friedman,SMT,nlpd,-1.871 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,friedman,SMT,fit_s,4.71948 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,friedman,OpenTURNS,q2,0.990862 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,friedman,OpenTURNS,nlpd,0.469828 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,friedman,OpenTURNS,fit_s,0.0511642 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,otl_circuit,pylibkriging,q2,0.999922 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,otl_circuit,pylibkriging,nlpd,-3.46228 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,otl_circuit,pylibkriging,fit_s,0.0136361 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,otl_circuit,pylibkriging BFGS10,q2,0.999922 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,otl_circuit,pylibkriging BFGS10,nlpd,-3.46227 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,otl_circuit,pylibkriging BFGS10,fit_s,0.168652 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,otl_circuit,scikit-learn,q2,0.999906 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,otl_circuit,scikit-learn,nlpd,-3.43062 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,otl_circuit,scikit-learn,fit_s,0.64982 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,otl_circuit,SMT,q2,0.999954 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,otl_circuit,SMT,nlpd,-3.35016 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,otl_circuit,SMT,fit_s,6.837 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,otl_circuit,OpenTURNS,q2,0.99777 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,otl_circuit,OpenTURNS,nlpd,-1.61723 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,otl_circuit,OpenTURNS,fit_s,0.0662544 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,piston,pylibkriging,q2,0.997498 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,piston,pylibkriging,nlpd,-3.80667 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,piston,pylibkriging,fit_s,0.03055 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,piston,pylibkriging BFGS10,q2,0.997498 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,piston,pylibkriging BFGS10,nlpd,-3.8067 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,piston,pylibkriging BFGS10,fit_s,0.326413 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,piston,scikit-learn,q2,0.998684 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,piston,scikit-learn,nlpd,-4.10497 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,piston,scikit-learn,fit_s,0.856336 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,piston,SMT,q2,0.997511 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,piston,SMT,nlpd,-3.86618 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,piston,SMT,fit_s,12.0067 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,piston,OpenTURNS,q2,0.990631 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,piston,OpenTURNS,nlpd,-2.9906 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,piston,OpenTURNS,fit_s,0.137117 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,borehole,pylibkriging,q2,0.999694 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,borehole,pylibkriging,nlpd,1.00118 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,borehole,pylibkriging,fit_s,0.0166521 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,borehole,pylibkriging BFGS10,q2,0.999694 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,borehole,pylibkriging BFGS10,nlpd,1.00121 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,borehole,pylibkriging BFGS10,fit_s,0.200685 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,borehole,scikit-learn,q2,0.999929 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,borehole,scikit-learn,nlpd,0.194455 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,borehole,scikit-learn,fit_s,1.18352 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,borehole,SMT,q2,0.999925 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,borehole,SMT,nlpd,0.279122 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,borehole,SMT,fit_s,12.3314 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,borehole,OpenTURNS,q2,0.996206 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,borehole,OpenTURNS,nlpd,2.55431 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,borehole,OpenTURNS,fit_s,0.121907 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,wing_weight,pylibkriging,q2,0.99972 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,wing_weight,pylibkriging,nlpd,1.11517 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,wing_weight,pylibkriging,fit_s,0.0297599 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,wing_weight,pylibkriging BFGS10,q2,0.99972 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,wing_weight,pylibkriging BFGS10,nlpd,1.11512 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,wing_weight,pylibkriging BFGS10,fit_s,0.335854 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,wing_weight,scikit-learn,q2,0.998048 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,wing_weight,scikit-learn,nlpd,2.57304 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,wing_weight,scikit-learn,fit_s,0.833252 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,wing_weight,SMT,q2,0.999898 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,wing_weight,SMT,nlpd,0.604023 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,wing_weight,SMT,fit_s,26.7889 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,wing_weight,OpenTURNS,q2,0.993145 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,wing_weight,OpenTURNS,nlpd,2.83627 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,wing_weight,OpenTURNS,fit_s,0.253929 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann6,pylibkriging,q2,0.422939 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann6,pylibkriging,nlpd,0.0965158 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann6,pylibkriging,fit_s,0.068651 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann6,pylibkriging BFGS10,q2,0.72352 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann6,pylibkriging BFGS10,nlpd,-0.156898 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann6,pylibkriging BFGS10,fit_s,0.877097 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann6,scikit-learn,q2,0.69049 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann6,scikit-learn,nlpd,-0.137845 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann6,scikit-learn,fit_s,1.43153 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann6,SMT,q2,0.708463 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann6,SMT,nlpd,-0.0177081 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann6,SMT,fit_s,11.2226 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann6,OpenTURNS,q2,0.696177 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann6,OpenTURNS,nlpd,-0.132838 +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann6,OpenTURNS,fit_s,0.141383 diff --git a/tools/install/julia.sh b/tools/install/julia.sh new file mode 100755 index 0000000..01bd170 --- /dev/null +++ b/tools/install/julia.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# Install JLibKriging.jl (compiled against a libKriging checkout) and the Julia Jupyter +# kernel into the active Julia environment. +# tools/install/julia.sh LIBKRIGING_SRC [KERNEL_NAME] (default kernel name: julia-1.11, +# the one of the notebooks) +# JLibKriging.jl's build (deps/build.jl) compiles the checkout given by +# LIBKRIGING_SRC_DIR and generates the module from its bindings/Julia/jlibkriging. +# Already installed (e.g. a restored ~/.julia depot): only the kernel is (re)installed, +# and LIBKRIGING_SRC need not exist. +set -euo pipefail +kernel=${2:-julia-1.11} +python -m pip install nbclient nbformat ipykernel jupyter_client +if ! julia -e 'using JLibKriging, IJulia, DelimitedFiles' 2> /dev/null; then + LIBKRIGING_SRC_DIR=$(cd "$1" && pwd); export LIBKRIGING_SRC_DIR + julia -e 'using Pkg + Pkg.add(url="https://github.com/libKriging/JLibKriging.jl") + Pkg.add(["IJulia", "DelimitedFiles"])' +fi +julia -e "using IJulia; installkernel(\"Julia\"; specname=\"$kernel\") + using JLibKriging; println(\"JLibKriging OK\")" diff --git a/tools/install/octave.sh b/tools/install/octave.sh new file mode 100755 index 0000000..9155b4a --- /dev/null +++ b/tools/install/octave.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +# Build mLibKriging (the Octave binding) from a libKriging checkout into OUT_DIR, which +# then goes on OCTAVE_PATH; install the Octave Jupyter kernel. +# tools/install/octave.sh LIBKRIGING_SRC OUT_DIR +# Built from source for both channels (master: libKriging master; release: the last +# release tag): the published binaries target another Octave version than Ubuntu's. +# Needs octave, octave-dev, liblapack-dev, libopenblas-dev. +set -euo pipefail +src=$(cd "$1" && pwd); out=$(mkdir -p "$2" && cd "$2" && pwd) +if [ ! -f "$out/mLibKriging.mex" ]; then + cmake -S "$src" -B "$src/build-octave" -DCMAKE_BUILD_TYPE=Release -DENABLE_OCTAVE_BINDING=on \ + -DENABLE_PYTHON_BINDING=off -DENABLE_MATLAB_BINDING=off -DBUILD_SHARED_LIBS=off -DSTATIC_LIB=on + cmake --build "$src/build-octave" --target mLibKriging -j "$(nproc)" + cp "$src/build-octave/bindings/Octave/mlibkriging/mLibKriging.mex" "$src"/bindings/Octave/mlibkriging/*.m "$out/" +fi +python -m pip install nbclient nbformat ipykernel octave_kernel +OCTAVE_PATH="$out" octave --no-gui --eval 'k = Kriging([0; 1; 0.5], [0; 1; 0.5], "gauss"); printf("mLibKriging OK (theta %g)\n", k.theta())' diff --git a/tools/install/python.sh b/tools/install/python.sh new file mode 100755 index 0000000..5411b5c --- /dev/null +++ b/tools/install/python.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# Install pylibkriging, then the notebooks' helpers (tools/lkbench) and runner. +# tools/install/python.sh wheel LIBKRIGING_SRC OUT_DIR only build a wheel from a libKriging checkout +# tools/install/python.sh master WHEEL_DIR|LIBKRIGING_SRC install the wheel of WHEEL_DIR, or build it first +# tools/install/python.sh release the last release, from PyPI +# The wheel needs BLAS/LAPACK headers (Ubuntu: liblapack-dev libopenblas-dev). +set -euo pipefail +tools=$(cd "$(dirname "$0")/.." && pwd) +build_wheel() { # SRC OUT + python -m pip install --upgrade pip + python -m pip install -r "$1/bindings/Python/pylibkriging/requirements.txt" \ + -r "$1/bindings/Python/pylibkriging/dev-requirements.txt" cmake + CMAKE_BUILD_PARALLEL_LEVEL=${CMAKE_BUILD_PARALLEL_LEVEL:-$(nproc)} \ + python -m pip wheel --no-build-isolation --no-deps -w "$2" "$1" +} +case "$1" in + wheel) build_wheel "$2" "$3"; exit 0 ;; + master) + if ! ls "$2"/pylibkriging-*.whl > /dev/null 2>&1; then + build_wheel "$2" "$2/dist"; set -- "$1" "$2/dist" + fi + python -m pip install "$2"/pylibkriging-*.whl ;; + release) python -m pip install pylibkriging ;; + *) echo "usage: $0 wheel SRC OUT | master WHEEL_DIR|SRC | release" >&2; exit 2 ;; +esac +python -m pip install -e "${tools}[notebooks]" +python -c "import pylibkriging as lk; print('pylibkriging', lk.__version__, 'from', lk.__file__)" diff --git a/tools/install/r.sh b/tools/install/r.sh new file mode 100755 index 0000000..e2f7fa2 --- /dev/null +++ b/tools/install/r.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# Install rlibkriging into the library directory LIB (added to R_LIBS_USER by the caller), +# and the R Jupyter kernel (IRkernel). +# tools/install/r.sh master LIB LIBKRIGING_SRC build from a libKriging checkout (C++ core + bindings/R) +# tools/install/r.sh release LIB the last release, from CRAN +# The master build follows libKriging's tools/r-linux-macos/build.sh: the core is compiled +# with R's own compilers as a static library, then bindings/R is built against it. +set -euo pipefail +channel=$1 lib=$(mkdir -p "$2" && cd "$2" && pwd) +export R_LIBS_USER="$lib" +if [ "$channel" = release ]; then + Rscript -e 'install.packages("rlibkriging", lib = Sys.getenv("R_LIBS_USER"))' +else + src=$(cd "$3" && pwd) + # Rcpp & co. (bindings/R/rlibkriging/install/install.R would also install test-only packages) + Rscript -e 'for (p in c("Rcpp", "RcppArmadillo", "RhpcBLASctl")) + if (!requireNamespace(p, quietly = TRUE)) install.packages(p, lib = Sys.getenv("R_LIBS_USER"))' + CC="$(R CMD config CC)" CXX="$(R CMD config CXX)" cmake -S "$src" -B "$src/build-r" -G "Unix Makefiles" \ + -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_LIBDIR=lib -DBUILD_SHARED_LIBS=off -DSTATIC_LIB=on \ + -DENABLE_OCTAVE_BINDING=off -DENABLE_MATLAB_BINDING=off -DENABLE_PYTHON_BINDING=off + # install.lib also installs the bench/bench-* executables, which it does not build: build them + # first (without the tests, which the full install target would also build) + mapfile -t benches < <(cmake --build "$src/build-r" --target help | sed -n 's/^\.\.\. \(bench-[A-Za-z0-9_-]*\).*/\1/p') + cmake --build "$src/build-r" --target "${benches[@]}" -j "$(nproc)" + cmake --build "$src/build-r" --target install.lib -j "$(nproc)" + export LIBKRIGING_PATH="$src/build-r/installed" + cd "$src/bindings/R" + mkdir -p Rlibs && touch Rlibs/.prepared # dependencies already installed above + R_LIBS="$lib" Rscript -e "Rcpp::compileAttributes(pkgdir = 'rlibkriging')" + R_LIBS="$lib" MAKE_SHARED_LIBS=off STATIC_LIB=on MAKEFLAGS="-j$(nproc)" R CMD INSTALL --no-multiarch -l "$lib" rlibkriging +fi +python -m pip install nbclient nbformat ipykernel jupyter_client +Rscript -e 'if (!requireNamespace("IRkernel", quietly = TRUE)) install.packages("IRkernel", lib = Sys.getenv("R_LIBS_USER")) + IRkernel::installspec(user = TRUE) + cat("rlibkriging", as.character(packageVersion("rlibkriging")), "in", find.package("rlibkriging"), "\n")' diff --git a/notebooks/julia/checks.jl b/tools/julia/checks.jl similarity index 82% rename from notebooks/julia/checks.jl rename to tools/julia/checks.jl index 88c1645..7de62f8 100644 --- a/notebooks/julia/checks.jl +++ b/tools/julia/checks.jl @@ -1,18 +1,15 @@ # Same CSV outputs as lkbench/checks.py (see there), for the Julia notebooks. +# Paths are relative to the repository root, where the notebooks are executed. using DelimitedFiles, Printf -const RESULTS_DIR = get(ENV, "LKBENCH_RESULTS", joinpath("..", "..", "results")) +const RESULTS_DIR = get(ENV, "LKBENCH_RESULTS", "results") mkpath(RESULTS_DIR) function read_known() - f = joinpath("..", "..", "known_issues.csv") - isfile(f) || return Dict{Tuple{String,String},String}() - d = Dict{Tuple{String,String},String}() - for l in readlines(f)[2:end] - p = split(l, ","; limit=3) - length(p) == 3 && (d[(p[1], p[2])] = p[3]) - end - d + f = joinpath("tools", "known_issues.csv") + isfile(f) || error("run the notebooks from the repository root ($f not found)") + rows = readdlm(f, ',', String; header=true, quotes=true)[1] # quoted fields may hold commas + Dict{Tuple{String,String},String}((rows[i, 1], rows[i, 2]) => rows[i, 3] for i in 1:size(rows, 1)) end mutable struct Suite @@ -71,5 +68,5 @@ function finalize!(s::Suite) isempty(failed) || error("$(length(failed)) check(s) failed in $(s.name): " * join(first(failed, 20), "; ")) end -read_data(name, stem) = readdlm(joinpath("..", "..", "data", "$(name)_$(stem).csv"), ',', Float64) +read_data(name, stem) = readdlm(joinpath("data", "$(name)_$(stem).csv"), ',', Float64) q2(y, m) = 1 - sum((y .- m) .^ 2) / sum((y .- sum(y) / length(y)) .^ 2) diff --git a/tools/known_issues.csv b/tools/known_issues.csv new file mode 100644 index 0000000..c69a1b4 --- /dev/null +++ b/tools/known_issues.csv @@ -0,0 +1,11 @@ +suite,check,issue +python_Kriging-vecchia-nystrom_large-n,vecchia(n-1) LL == exact LL (same theta),With m >= n-1 neighbours the Vecchia likelihood should be the exact one; it matches for n=20 but not for n=40 (-111.349 vs -110.834) +python_Kriging-vecchia-nystrom_large-n,hartmann3/n=1500/nystrom(100): Q2,Nystrom predictor degrades as n grows at fixed k (theta estimates are fine: the exact predictor at the Nystrom theta has Q2 ~ 1) +python_Kriging-vecchia-nystrom_large-n,hartmann3/n=5000/nystrom(100): Q2,Nystrom predictor degrades as n grows at fixed k (theta estimates are fine: the exact predictor at the Nystrom theta has Q2 ~ 1) +python_Kriging-vecchia-nystrom_large-n,otl_circuit/n=1500/nystrom(100): Q2,Nystrom predictor degrades as n grows at fixed k (theta estimates are fine: the exact predictor at the Nystrom theta has Q2 ~ 1) +python_Kriging-vecchia-nystrom_large-n,otl_circuit/n=5000/nystrom(100): Q2,Nystrom predictor degrades as n grows at fixed k (theta estimates are fine: the exact predictor at the Nystrom theta has Q2 ~ 1) +python_Kriging_robustness-invalid-inputs,NaN in y is rejected,Kriging accepts NaN in y without raising +python_Kriging_robustness-invalid-inputs,Inf in y is rejected,Kriging accepts Inf in y without raising +python_Kriging_robustness-invalid-inputs,WarpKriging with 1 spec for 2 columns is rejected,WarpKriging accepts fewer warping specs than columns of X +python_Kriging_robustness-invalid-inputs,predict(stdev) time growth exponent in q (8000 vs 4000),"predict(return_stdev=True, return_cov=False) is O(q^2) in time and memory (10 000 points: 1.6 GB; 30 000: out of memory)" +python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,sklearn RBF == libKriging gauss: simple kriging stdev (beta fixed),With beta fixed (is_beta_estim=False) predict still adds the trend-estimation variance (universal kriging) instead of the simple kriging variance diff --git a/lkbench/__init__.py b/tools/lkbench/__init__.py similarity index 100% rename from lkbench/__init__.py rename to tools/lkbench/__init__.py diff --git a/lkbench/checks.py b/tools/lkbench/checks.py similarity index 85% rename from lkbench/checks.py rename to tools/lkbench/checks.py index 6e960e9..05770f8 100644 --- a/lkbench/checks.py +++ b/tools/lkbench/checks.py @@ -13,12 +13,12 @@ packages, where a threshold is a judgement call rather than a correctness bound. Statuses: PASS, FAIL (error-level), WARN (warn-level), SKIP (feature missing in -the installed version), KNOWN (a failure listed in ``known_issues.csv``: reported -with its issue, but does not fail the run). A listed check that passes is +the installed version), KNOWN (a failing check, of either severity, listed in +``tools/known_issues.csv``: reported with its issue, but does not fail the run). A listed check that passes is reported as FIXED by ``tools/report.py``, so that the list does not rot. The R, Octave and Julia notebooks write the same two CSV formats with a few lines -of their own code (see notebooks/*/checks.*), so that ``tools/report.py`` can +of their own code (tools/r/checks.R, tools/octave/lkb_*.m, tools/julia/checks.jl), so that ``tools/report.py`` can aggregate every language in one report. """ import csv @@ -31,17 +31,33 @@ import numpy as np -ROOT = Path(__file__).resolve().parents[1] + +def _find_root(): + """The repository root: $LKBENCH_ROOT, or the first directory, from the working + directory up, that holds tools/known_issues.csv (notebooks run from the root). + Not derived from this file's location, which is site-packages for a non-editable + install: the known issues would then be silently ignored.""" + if os.environ.get("LKBENCH_ROOT"): + return Path(os.environ["LKBENCH_ROOT"]).resolve() + for d in [Path.cwd(), *Path.cwd().parents]: + if (d / "tools" / "known_issues.csv").is_file(): + return d + raise RuntimeError("libKriging benchmarks repository not found from the working directory " + f"{Path.cwd()}: run the notebooks from the repository root, or set LKBENCH_ROOT") + + +ROOT = _find_root() RESULTS_DIR = Path(os.environ.get("LKBENCH_RESULTS", ROOT / "results")) -KNOWN_ISSUES_FILE = ROOT / "known_issues.csv" +KNOWN_ISSUES_FILE = ROOT / "tools" / "known_issues.csv" +DATA_DIR = ROOT / "data" def known_issues(): - """{(suite, check): issue} from known_issues.csv.""" - if not KNOWN_ISSUES_FILE.exists(): - return {} + """{(suite, check): issue} from tools/known_issues.csv.""" with open(KNOWN_ISSUES_FILE, newline="") as fh: return {(r["suite"], r["check"]): r["issue"] for r in csv.DictReader(fh)} + + CHECK_FIELDS = ["suite", "lang", "check", "severity", "status", "value", "expected", "detail"] METRIC_FIELDS = ["suite", "lang", "case", "package", "metric", "value"] diff --git a/lkbench/compat.py b/tools/lkbench/compat.py similarity index 100% rename from lkbench/compat.py rename to tools/lkbench/compat.py diff --git a/lkbench/design.py b/tools/lkbench/design.py similarity index 100% rename from lkbench/design.py rename to tools/lkbench/design.py diff --git a/lkbench/functions.py b/tools/lkbench/functions.py similarity index 99% rename from lkbench/functions.py rename to tools/lkbench/functions.py index c96ff1c..87537b2 100644 --- a/lkbench/functions.py +++ b/tools/lkbench/functions.py @@ -3,7 +3,7 @@ Every function takes ``X`` of shape (n, d) in its *native* domain and returns a length-n vector. ``TESTS`` gathers the metadata (dimension, domain, a reference -point with its known value) used by ``notebooks/python/00_test_functions.ipynb`` +point with its known value) used by ``python_no-model_test-functions-reference-values.ipynb`` to check the implementations themselves before anything is benchmarked on them. """ from dataclasses import dataclass, field diff --git a/lkbench/metrics.py b/tools/lkbench/metrics.py similarity index 100% rename from lkbench/metrics.py rename to tools/lkbench/metrics.py diff --git a/notebooks/octave/lkb_check.m b/tools/octave/lkb_check.m similarity index 100% rename from notebooks/octave/lkb_check.m rename to tools/octave/lkb_check.m diff --git a/notebooks/octave/lkb_close.m b/tools/octave/lkb_close.m similarity index 100% rename from notebooks/octave/lkb_close.m rename to tools/octave/lkb_close.m diff --git a/notebooks/octave/lkb_finalize.m b/tools/octave/lkb_finalize.m similarity index 92% rename from notebooks/octave/lkb_finalize.m rename to tools/octave/lkb_finalize.m index f878779..5a90860 100644 --- a/notebooks/octave/lkb_finalize.m +++ b/tools/octave/lkb_finalize.m @@ -1,6 +1,6 @@ function lkb_finalize(s) s = lkb_metric(s, "_suite", "_", "elapsed_s", toc(s.t0)); - d = getenv("LKBENCH_RESULTS"); if isempty(d), d = fullfile("..", "..", "results"); end + d = getenv("LKBENCH_RESULTS"); if isempty(d), d = "results"; end if ~exist(d, "dir"), mkdir(d); end write_csv(fullfile(d, [s.name ".checks.csv"]), {"suite", "lang", "check", "severity", "status", "value", "expected", "detail"}, s.checks); write_csv(fullfile(d, [s.name ".metrics.csv"]), {"suite", "lang", "case", "package", "metric", "value"}, s.metrics); diff --git a/notebooks/octave/lkb_metric.m b/tools/octave/lkb_metric.m similarity index 100% rename from notebooks/octave/lkb_metric.m rename to tools/octave/lkb_metric.m diff --git a/tools/octave/lkb_suite.m b/tools/octave/lkb_suite.m new file mode 100644 index 0000000..1cecb2e --- /dev/null +++ b/tools/octave/lkb_suite.m @@ -0,0 +1,39 @@ +function s = lkb_suite(name) + % Same CSV outputs as lkbench/checks.py (see there), for the Octave notebooks. + % A suite is a struct passed around by value: s = lkb_close(s, ...). + % Paths are relative to the repository root, where the notebooks are executed. + s.name = name; + s.checks = {}; + s.metrics = {}; + s.t0 = tic(); + s.known = cell(0, 3); + f = fullfile("tools", "known_issues.csv"); + if ~exist(f, "file"), error("run the notebooks from the repository root (%s not found)", f); end + fid = fopen(f, "r"); fgetl(fid); + while true + l = fgetl(fid); + if ~ischar(l), break; end + p = csv_fields(l); + if numel(p) >= 3, s.known(end + 1, :) = p(1:3); end + end + fclose(fid); +end + +function f = csv_fields(l) + % one CSV line into fields (RFC 4180 quoting: "a,b" and "" inside quotes) + f = {}; cur = ""; q = false; i = 1; + while i <= numel(l) + c = l(i); + if q && c == '"' && i < numel(l) && l(i + 1) == '"' + cur(end + 1) = '"'; i = i + 1; + elseif c == '"' + q = ~q; + elseif c == "," && ~q + f{end + 1} = cur; cur = ""; + else + cur(end + 1) = c; + end + i = i + 1; + end + f{end + 1} = cur; +end diff --git a/tools/octave/read_data.m b/tools/octave/read_data.m new file mode 100644 index 0000000..ada51ee --- /dev/null +++ b/tools/octave/read_data.m @@ -0,0 +1,3 @@ +function A = read_data(name, stem) + A = dlmread(fullfile("data", [name "_" stem ".csv"]), ","); +end diff --git a/tools/py2nb.py b/tools/py2nb.py index 06851b2..ecf4d7f 100644 --- a/tools/py2nb.py +++ b/tools/py2nb.py @@ -1,11 +1,12 @@ #!/usr/bin/env python3 -"""Convert a "percent" script (cells separated by ``# %%`` / ``# %% [markdown]``) into a -notebook without outputs. The notebooks of this repository are written this way -(sources in ``notebooks/**/src/``) so that they review as plain diffs; the -``.ipynb`` files are generated, and CI checks that they are in sync. +"""Convert a "percent" script (cells separated by ``# %%`` / ``# %% [markdown]``, or +``% %%`` in Octave) into a notebook without outputs. The notebooks of this repository +are written this way (sources in ``tools/src/``) so that they review as plain diffs; +the ``.ipynb`` files, at the repository root, are generated, and CI checks that they +are in sync (and that no notebook lacks its source). -Usage: python tools/py2nb.py SRC... [--check] - each ``notebooks//src/.`` gives ``notebooks//.ipynb``. +Usage: python tools/py2nb.py [--check] [SRC...] (default: every tools/src/* file) + each ``tools/src/.`` gives ``.ipynb`` at the repository root. """ import argparse import re @@ -14,6 +15,9 @@ import nbformat +ROOT = Path(__file__).resolve().parents[1] +SRC = ROOT / "tools" / "src" + KERNELS = { ".py": ({"name": "python3", "display_name": "Python 3", "language": "python"}, {"name": "python"}, "#"), @@ -57,12 +61,16 @@ def flush(): def main(argv=None): p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) - p.add_argument("sources", nargs="+", type=Path) + p.add_argument("sources", nargs="*", type=Path) p.add_argument("--check", action="store_true", help="fail if a notebook differs from its source") args = p.parse_args(argv) + sources = args.sources or sorted(f for f in SRC.iterdir() if f.suffix in KERNELS) stale = [] - for src in args.sources: - out = src.parent.parent / (src.stem + ".ipynb") + if args.check and not args.sources: + orphans = sorted({nb.stem for nb in ROOT.glob("*.ipynb")} - {s.stem for s in sources}) + stale += [ROOT / f"{o}.ipynb (no source in tools/src)" for o in orphans] + for src in sources: + out = ROOT / (src.stem + ".ipynb") text = nbformat.writes(convert(src)) + "\n" if args.check: if not out.exists() or out.read_text() != text: diff --git a/pyproject.toml b/tools/pyproject.toml similarity index 63% rename from pyproject.toml rename to tools/pyproject.toml index 4e92615..3e980ef 100644 --- a/pyproject.toml +++ b/tools/pyproject.toml @@ -7,11 +7,12 @@ name = "lkbench" version = "0.1.0" description = "Helpers (test functions, designs, metrics, checks) for the libKriging benchmark notebooks" requires-python = ">=3.9" -dependencies = ["numpy", "scipy", "pandas", "matplotlib"] +dependencies = ["numpy", "scipy", "pandas", "matplotlib", "jinja2"] # jinja2: DataFrame.style [project.optional-dependencies] +# the comparison packages (scikit-learn, SMT, OpenTURNS, GPy, ...) are installed by the +# notebooks that use them, not here notebooks = ["nbclient", "nbformat", "ipykernel"] -compare = ["scikit-learn", "smt", "openturns"] [tool.setuptools] packages = ["lkbench"] diff --git a/notebooks/r/checks.R b/tools/r/checks.R similarity index 88% rename from notebooks/r/checks.R rename to tools/r/checks.R index 272b1ea..9b28a67 100644 --- a/notebooks/r/checks.R +++ b/tools/r/checks.R @@ -1,10 +1,11 @@ # Same CSV outputs as lkbench/checks.py (see there), for the R notebooks. -# Statuses: PASS, FAIL (error-level), WARN (warn-level), SKIP, KNOWN (listed in known_issues.csv). -results_dir <- Sys.getenv("LKBENCH_RESULTS", file.path("..", "..", "results")) +# Statuses: PASS, FAIL (error-level), WARN (warn-level), SKIP, KNOWN (listed in tools/known_issues.csv). +# Paths are relative to the repository root, where the notebooks are executed. +results_dir <- Sys.getenv("LKBENCH_RESULTS", "results") dir.create(results_dir, showWarnings = FALSE, recursive = TRUE) -known_file <- file.path("..", "..", "known_issues.csv") -.known <- if (file.exists(known_file)) read.csv(known_file, stringsAsFactors = FALSE) else - data.frame(suite = character(), check = character(), issue = character()) +known_file <- file.path("tools", "known_issues.csv") +if (!file.exists(known_file)) stop("run the notebooks from the repository root (", known_file, " not found)") +.known <- read.csv(known_file, stringsAsFactors = FALSE) new_suite <- function(name) { s <- new.env() @@ -68,4 +69,4 @@ finalize <- function(s) { rmse <- function(y, m) sqrt(mean((y - m)^2)) q2 <- function(y, m) 1 - sum((y - m)^2) / sum((y - mean(y))^2) -read_data <- function(name, stem) as.matrix(read.csv(file.path("..", "..", "data", paste0(name, "_", stem, ".csv")), header = FALSE)) +read_data <- function(name, stem) as.matrix(read.csv(file.path("data", paste0(name, "_", stem, ".csv")), header = FALSE)) diff --git a/tools/report.py b/tools/report.py index 19c46c3..874a082 100644 --- a/tools/report.py +++ b/tools/report.py @@ -12,21 +12,24 @@ 1. a status table per notebook (PASS / FAIL / WARN / KNOWN / SKIP); 2. every FAIL, WARN and KNOWN check, with its value and threshold; - 3. known issues whose check now passes ("FIXED?": remove them from known_issues.csv); + 3. known issues whose check now passes ("FIXED?": remove them from + tools/known_issues.csv); 4. cross-binding consistency: each language against Python, per quantity, with - the tolerances of notebooks/python/11_cross_binding.ipynb; + the tolerances of python_Kriging_cross-binding-reference.ipynb; 5. drifts of the accuracy metrics (q2, rmse, nlpd) and of the fit times against a - baseline (baseline/metrics.csv), when one is given. + baseline (tools/baseline/metrics-.csv), when one is given. Exit status 1 if any check FAILed, if a cross-binding quantity of error level differs, or if --expect lists a suite that produced no output (crashed before finalize). The Markdown goes to stdout and, with --summary, is appended to a file -(e.g. $GITHUB_STEP_SUMMARY). +(e.g. $GITHUB_STEP_SUMMARY). The suite of a notebook is its file name without +extension; `--expect python,R` expects one suite per `python_*.ipynb` and +`R_*.ipynb` notebook of the repository root (names are also accepted). Usage: - python tools/report.py [--results results] [--baseline baseline/metrics.csv] - [--write-baseline baseline/metrics.csv] [--summary FILE] - [--expect py00_test_functions,...] + python tools/report.py [--results results] [--baseline tools/baseline/metrics-master.csv] + [--write-baseline new-baseline/metrics-master.csv] [--summary FILE] + [--expect python,R,octave,julia] """ import argparse import csv @@ -136,7 +139,8 @@ def main(argv=None): p.add_argument("--baseline", help="metrics CSV of a reference run (drift report)") p.add_argument("--write-baseline", help="write the current metrics as a new baseline") p.add_argument("--summary", help="also append the report to this file") - p.add_argument("--expect", default="", help="comma-separated suites that must have produced output") + p.add_argument("--expect", default="", + help="comma-separated suites, or binding prefixes (python, R, octave, julia), that must have output") args = p.parse_args(argv) checks = read_all(args.results, "*.checks.csv") @@ -156,7 +160,10 @@ def main(argv=None): lines += ["## Notebooks\n", md_table(tab)] exit_code |= int((checks["status"] == "FAIL").any()) - expected = [s for s in args.expect.split(",") if s] + expected = [] + for item in filter(None, args.expect.split(",")): + stems = sorted(nb.stem for nb in ROOT.glob(f"{item}_*.ipynb")) + expected += stems or [item] missing = sorted(set(expected) - set(checks.get("suite", pd.Series(dtype=str)))) if missing: lines += ["## Notebooks without output (crashed before `finalize`)\n", md_table(pd.DataFrame({"suite": missing}))] @@ -170,12 +177,12 @@ def main(argv=None): lines += [f"## {title} ({len(sub)})\n", md_table(sub)] # 3. known issues that now pass - known_file = ROOT / "known_issues.csv" + known_file = ROOT / "tools" / "known_issues.csv" if known_file.exists(): known = pd.read_csv(known_file, dtype=str, keep_default_na=False) passing = known.merge(checks[checks["status"] == "PASS"][["suite", "check"]], on=["suite", "check"]) if not passing.empty: - lines += ["## Known issues that now pass — FIXED? (remove them from known_issues.csv)\n", + lines += ["## Known issues that now pass — FIXED? (remove them from tools/known_issues.csv)\n", md_table(passing)] # 4. cross-binding diff --git a/tools/run_notebooks.py b/tools/run_notebooks.py index fd2be35..7c835c6 100644 --- a/tools/run_notebooks.py +++ b/tools/run_notebooks.py @@ -1,9 +1,10 @@ #!/usr/bin/env python3 """Execute Jupyter notebooks and fail if any cell raises. -Used by CI (.github/workflows/notebooks.yml) to execute every notebook against the -installed libKriging and, with `--inplace`, to save the executed notebook with its -outputs. A notebook that fails is saved too, with the outputs of the cells run up +Each notebook runs in its own directory (the repository root: the paths in the +notebooks are relative to it). Used by CI (.github/workflows/notebooks.yml) to +execute every notebook against the installed libKriging and, with `--inplace`, to +save the executed notebook with its outputs. A notebook that fails is saved too, with the outputs of the cells run up to the failing one (the error included): that is the run worth looking at. `--skip-setup` does not run the `%%bash` cells (nor a cell starting with @@ -12,8 +13,8 @@ kernel (Python: installed; R: library; Octave: `OCTAVE_PATH`; Julia: environment). Usage: - python3 tools/run_notebooks.py notebooks/python/*.ipynb - python3 tools/run_notebooks.py --inplace --timeout 3600 --kernel julia-1.11 notebooks/julia/*.ipynb + python3 tools/run_notebooks.py python_*.ipynb + python3 tools/run_notebooks.py --inplace --timeout 3600 --kernel julia-1.11 julia_Kriging_cross-binding.ipynb python3 tools/run_notebooks.py --skip-cell '^repo_root = ' --prelude 'addpath(...)' nb.ipynb Requires nbclient, nbformat and ipykernel. diff --git a/notebooks/r/src/01_cross_binding.R b/tools/src/R_Kriging_cross-binding.R similarity index 89% rename from notebooks/r/src/01_cross_binding.R rename to tools/src/R_Kriging_cross-binding.R index 2950207..785c8a8 100644 --- a/notebooks/r/src/01_cross_binding.R +++ b/tools/src/R_Kriging_cross-binding.R @@ -1,14 +1,14 @@ # %% [markdown] -# # R 01 — Cross-binding consistency: the R side +# # R · Kriging — cross-binding consistency: the R side # -# Same protocol as `notebooks/python/11_cross_binding.ipynb` (see there), on the +# Same protocol as `python_Kriging_cross-binding-reference.ipynb` (see there), on the # shared datasets of `data/`; writes `results/xbind_R.csv`, compared with the # Python values by `tools/report.py`. # %% suppressPackageStartupMessages(library(rlibkriging)) -source("checks.R") -s <- new_suite("r01_cross_binding") +source("tools/r/checks.R") +s <- new_suite("R_Kriging_cross-binding") cat("rlibkriging", as.character(packageVersion("rlibkriging")), "\n") rows <- list() put <- function(dataset, quantity, values) { diff --git a/notebooks/r/src/02_vs_dicekriging_robustgasp.R b/tools/src/R_Kriging_vs-DiceKriging-RobustGaSP.R similarity index 84% rename from notebooks/r/src/02_vs_dicekriging_robustgasp.R rename to tools/src/R_Kriging_vs-DiceKriging-RobustGaSP.R index bd20b33..4fffb7f 100644 --- a/notebooks/r/src/02_vs_dicekriging_robustgasp.R +++ b/tools/src/R_Kriging_vs-DiceKriging-RobustGaSP.R @@ -1,5 +1,5 @@ # %% [markdown] -# # R 02 — rlibkriging vs DiceKriging and RobustGaSP +# # R · Kriging vs DiceKriging and RobustGaSP # # DiceKriging is the reference implementation libKriging's `Kriging` follows # (same product-form kernels and range parametrization, same concentrated @@ -13,14 +13,32 @@ # Borehole), with RobustGaSP as a third package: Q², fit time # (warn-level comparisons). +# %% [markdown] +# ## Installing DiceKriging and RobustGaSP +# +# From CRAN (the configured repository, else cloud.r-project.org), into the +# first library of `.libPaths()`, unless already installed. DiceKriging is +# required; without RobustGaSP its comparison is skipped (warn-level). + +# %% +repos <- getOption("repos") +if (is.null(repos) || any(repos == "@CRAN@")) repos <- c(CRAN = "https://cloud.r-project.org") +for (p in c("DiceKriging", "RobustGaSP")) { + if (!requireNamespace(p, quietly = TRUE)) { + message("installing ", p) + tryCatch(install.packages(p, repos = repos, lib = .libPaths()[1]), + error = function(e) message(p, " could not be installed: ", conditionMessage(e))) + } +} + # %% suppressPackageStartupMessages({ library(rlibkriging) library(DiceKriging) }) has_rgasp <- requireNamespace("RobustGaSP", quietly = TRUE) -source("checks.R") -s <- new_suite("r02_vs_dicekriging_robustgasp") +source("tools/r/checks.R") +s <- new_suite("R_Kriging_vs-DiceKriging-RobustGaSP") cat("rlibkriging", as.character(packageVersion("rlibkriging")), "| DiceKriging", as.character(packageVersion("DiceKriging")), "| RobustGaSP", if (has_rgasp) as.character(packageVersion("RobustGaSP")) else "absent", "\n") DATASETS <- c("branin", "hartmann3", "borehole") diff --git a/tools/src/R_all-classes_api-tour.R b/tools/src/R_all-classes_api-tour.R new file mode 100644 index 0000000..5d7b48f --- /dev/null +++ b/tools/src/R_all-classes_api-tour.R @@ -0,0 +1,205 @@ +# %% [markdown] +# # R · all classes — every libKriging model class and fit method through rlibkriging +# +# One problem (Branin on [0,1]², Q² on 500 test points), every class and every fit +# method of the R binding, each with the checks that hold for it whatever the +# data: +# +# | model | checks (error-level unless stated) | +# |---|---| +# | `Kriging` (noise-free) | interpolation, Q² floor, conditional simulations through the data, `update` = fit on all points (same θ), `save`/`load` | +# | `Kriging(noise = "nugget")` | nugget > 0, smoothing next to the design points, single-start optimum as good as the multistart one (warn), `save`/`load` | +# | `Kriging(noise = )` | the stored variances, finite predictions | +# | `Kriging(method = "vecchia(m)")`, `"nystrom(k)"` | `k$method()`, `predict(method = ...)` finite, Q² floor (warn) | +# | `WarpKriging` (continuous and categorical warpings) | finite predictions, Q² floor (warn), `save`/`load` | +# | `MLPKriging` | finite predictions, Q² floor (warn), default optimizer as good as a multistart BFGS (warn), `save`/`load` | +# | `NestedKriging` (`NK`, `PoE`) | `NK` interpolates, stdev ≥ 0, `save`/`load` | +# +# A class missing from the installed rlibkriging (release channel) is recorded as +# SKIP. + +# %% +suppressPackageStartupMessages(library(rlibkriging)) +source("tools/r/checks.R") +s <- new_suite("R_all-classes_api-tour") +cat("rlibkriging", as.character(packageVersion("rlibkriging")), "\n") + +branin <- function(U) { # on [0,1]^2 + x1 <- 15 * U[, 1] - 5; x2 <- 15 * U[, 2] + (x2 - 5.1 / (4 * pi^2) * x1^2 + 5 / pi * x1 - 6)^2 + 10 * (1 - 1 / (8 * pi)) * cos(x1) + 10 +} +set.seed(1) +lhs <- function(n, d) sapply(1:d, function(j) (sample(n) - runif(n)) / n) +X <- lhs(30, 2); y <- branin(X) +Xt <- matrix(runif(1000), ncol = 2); yt <- branin(Xt) +tmp <- tempdir() +has_class <- function(name) exists(name, mode = "function") +skip <- function(s, check, reason) { + s$checks[[length(s$checks) + 1]] <- data.frame(suite = s$name, lang = "R", check = check, severity = "info", + status = "SKIP", value = "", expected = "", detail = reason, stringsAsFactors = FALSE) + message("[SKIP] ", check, ": ", reason) +} +roundtrip_same_mean <- function(s, label, k, Xn) { + f <- file.path(tmp, paste0(gsub("[^A-Za-z0-9]", "_", label), ".json")) + check_runs(s, paste0(label, ": save + load"), { + save(k, f) + k2 <- rlibkriging::load(f) + check_close(s, paste0(label, ": load(save(k)) predicts the same mean"), + predict(k2, Xn, return_stdev = FALSE)$mean, predict(k, Xn, return_stdev = FALSE)$mean, rtol = 1e-10) + }) +} + +# %% [markdown] +# ## `Kriging`, noise-free + +# %% +k <- Kriging(y, X, "matern5_2") +print(k) +p <- predict(k, Xt, return_stdev = TRUE) +add_metric(s, "branin", "Kriging", q2 = q2(yt, p$mean), rmse = rmse(yt, p$mean)) +check_greater(s, "Kriging: Q2", q2(yt, p$mean), 0.95) +check_less(s, "Kriging: interpolates (max |m(X) - y| / sd(y))", + max(abs(predict(k, X, return_stdev = FALSE)$mean - y)) / sd(y), 1e-6) +sims <- simulate(k, nsim = 5, seed = 2, x = X) +check_less(s, "Kriging: conditional simulations go through the data (max |sim(X) - y| / sigma)", + max(abs(sims - y)) / sqrt(k$sigma2()), 1e-4) +check_greater(s, "Kriging: stdev > 0 off the design", min(p$stdev), 0) +# update with 5 new points (no refit) = a fit on the 35 points at the same theta +Xu <- lhs(5, 2); yu <- branin(Xu) +ku <- copy(k) +update(ku, yu, Xu, refit = FALSE) +k35 <- Kriging(c(y, yu), rbind(X, Xu), "matern5_2", optim = "none", + parameters = list(theta = matrix(k$theta(), nrow = 1), is_theta_estim = FALSE, + sigma2 = k$sigma2(), is_sigma2_estim = FALSE)) +check_close(s, "Kriging: update(refit = FALSE) == fit on all points (same theta, sigma2)", + predict(ku, Xt, return_stdev = FALSE)$mean, predict(k35, Xt, return_stdev = FALSE)$mean, + rtol = 1e-6, atol = 1e-8 * sd(y)) +roundtrip_same_mean(s, "Kriging", k, Xt) + +# %% [markdown] +# ## `Kriging` with noise: estimated nugget, known variances +# +# Noise of standard deviation 5% of sd(y). The nugget model is fitted with 10 +# restarts (`optim = "BFGS10"`); the default single start is compared with it +# (warn-level: a lower likelihood means a local optimum). As DiceKriging's +# nugget, the predictor returns y exactly *at* the design points (the nugget is +# a micro-scale variation): smoothing is checked next to them. + +# %% +sd_noise <- 0.05 * sd(y) +yn <- y + rnorm(length(y), sd = sd_noise) +kn <- Kriging(yn, X, "matern5_2", noise = "nugget", optim = "BFGS10") +pn <- predict(kn, Xt, return_stdev = TRUE) +add_metric(s, "branin noisy", "Kriging nugget BFGS10", q2 = q2(yt, pn$mean), nugget = kn$nugget(), + ll = logLikelihood(kn)) +check_greater(s, "Kriging(nugget): estimated nugget > 0", kn$nugget(), 0) +check_greater(s, "Kriging(nugget): Q2", q2(yt, pn$mean), 0.95) +near <- pmin(X + 1e-7, 1) +check_greater(s, "Kriging(nugget): smooths the noisy data next to the design points (rms residual / noise sd)", + rmse(yn, predict(kn, near, return_stdev = FALSE)$mean) / sd_noise, 0.3) +kn1 <- Kriging(yn, X, "matern5_2", noise = "nugget") +add_metric(s, "branin noisy", "Kriging nugget BFGS", q2 = q2(yt, predict(kn1, Xt, return_stdev = FALSE)$mean), + nugget = kn1$nugget(), ll = logLikelihood(kn1)) +check_greater(s, "Kriging(nugget): LL(single start) - LL(10 restarts)", logLikelihood(kn1) - logLikelihood(kn), -1, + severity = "warn") +roundtrip_same_mean(s, "Kriging(nugget)", kn, Xt) +kh <- Kriging(yn, X, "matern5_2", noise = rep(sd_noise^2, length(yn))) +check_close(s, "Kriging(noise = variances): noise() returns them", kh$noise(), rep(sd_noise^2, length(yn)), + rtol = 1e-12) +ph <- predict(kh, Xt, return_stdev = TRUE) +check_less(s, "Kriging(noise = variances): non-finite predictions", sum(!is.finite(c(ph$mean, ph$stdev))), 1) +add_metric(s, "branin noisy", "Kriging known noise", q2 = q2(yt, ph$mean)) + +# %% [markdown] +# ## `Kriging` with an approximate fit: Vecchia, Nyström (n = 400) + +# %% +Xl <- lhs(400, 2); yl <- branin(Xl) +for (m in c("vecchia(20)", "nystrom(60)")) { + short <- sub("\\(.*", "", m) + check_runs(s, paste0("Kriging(method = '", m, "'): fit + predict"), { + ka <- Kriging(yl, Xl, "matern5_2", method = m) + add_check(s, paste0("Kriging(method = '", m, "'): method()"), identical(ka$method(), m), ka$method(), m) + pa <- predict(ka, Xt, return_stdev = TRUE, method = short) + check_less(s, paste0("Kriging(method = '", m, "'): non-finite predictions"), + sum(!is.finite(c(pa$mean, pa$stdev))), 1) + add_metric(s, "branin n=400", paste("Kriging", m), q2 = q2(yt, pa$mean)) + check_greater(s, paste0("Kriging(method = '", m, "'): Q2"), q2(yt, pa$mean), 0.95, severity = "warn") + }) +} + +# %% [markdown] +# ## `WarpKriging`: continuous warpings, then a categorical input +# +# The categorical case: x₂ restricted to 5 levels (coded 0, …, 4), embedded in ℝ². + +# %% +if (has_class("WarpKriging")) { + check_runs(s, "WarpKriging(kumaraswamy, kumaraswamy): fit + predict", { + kw <- WarpKriging(y, X, warping = c("kumaraswamy", "kumaraswamy"), kernel = "matern5_2") + pw <- predict(kw, Xt, return_stdev = TRUE) + check_less(s, "WarpKriging(kumaraswamy): non-finite predictions", sum(!is.finite(c(pw$mean, pw$stdev))), 1) + add_metric(s, "branin", "WarpKriging kumaraswamy", q2 = q2(yt, pw$mean)) + check_greater(s, "WarpKriging(kumaraswamy): Q2", q2(yt, pw$mean), 0.9, severity = "warn") + roundtrip_same_mean(s, "WarpKriging(kumaraswamy)", kw, Xt) + }) + lev <- function(U) cbind(U[, 1], round(U[, 2] * 4)) # level codes 0..4 + unlev <- function(L) cbind(L[, 1], L[, 2] / 4) + Xc <- lev(lhs(40, 2)); yc <- branin(unlev(Xc)) + Xtc <- lev(Xt); ytc <- branin(unlev(Xtc)) + check_runs(s, "WarpKriging(none, categorical(5,2)): fit + predict", { + kc <- WarpKriging(yc, Xc, warping = c("none", "categorical(5,2)"), kernel = "matern5_2") + pc <- predict(kc, Xtc, return_stdev = TRUE) + check_less(s, "WarpKriging(categorical): non-finite predictions", sum(!is.finite(c(pc$mean, pc$stdev))), 1) + add_metric(s, "branin, x2 categorical", "WarpKriging categorical", q2 = q2(ytc, pc$mean)) + check_greater(s, "WarpKriging(categorical): Q2", q2(ytc, pc$mean), 0.8, severity = "warn") + roundtrip_same_mean(s, "WarpKriging(categorical)", kc, Xtc) + }) +} else skip(s, "WarpKriging", "not in this rlibkriging") + +# %% [markdown] +# ## `MLPKriging`: a joint feature map ℝ² → ℝ² before the kernel + +# %% +if (has_class("MLPKriging")) { + check_runs(s, "MLPKriging: fit + predict", { + km <- MLPKriging(y, X, hidden_dims = c(8), d_out = 2, activation = "tanh", kernel = "matern5_2", optim = "BFGS10") + pm <- predict(km, Xt, return_stdev = TRUE) + check_less(s, "MLPKriging: non-finite predictions", sum(!is.finite(c(pm$mean, pm$stdev))), 1) + add_metric(s, "branin", "MLPKriging tanh [8] -> 2, BFGS10", q2 = q2(yt, pm$mean)) + check_greater(s, "MLPKriging: Q2", q2(yt, pm$mean), 0.9, severity = "warn") + roundtrip_same_mean(s, "MLPKriging", km, Xt) + # the default optimizer ("BFGS+Adam") + kd <- MLPKriging(y, X, hidden_dims = c(8), d_out = 2, activation = "tanh", kernel = "matern5_2") + qd <- q2(yt, predict(kd, Xt, return_stdev = FALSE)$mean) + add_metric(s, "branin", "MLPKriging tanh [8] -> 2, default optim", q2 = qd) + check_greater(s, "MLPKriging: Q2(default optim) - Q2(BFGS10)", qd - q2(yt, pm$mean), -0.05, severity = "warn") + }) +} else skip(s, "MLPKriging", "not in this rlibkriging") + +# %% [markdown] +# ## `NestedKriging`: 4 groups of 100 points, `NK` and `PoE` aggregations + +# %% +if (has_class("NestedKriging")) { + for (agg in c("NK", "PoE")) { + check_runs(s, paste0("NestedKriging(", agg, "): fit + predict"), { + kk <- NestedKriging(yl, Xl, kernel = "matern5_2", nb_groups = 4, aggregation = agg, seed = 3) + pk <- predict(kk, Xt, return_stdev = TRUE) + check_less(s, paste0("NestedKriging(", agg, "): stdev < 0"), sum(pk$stdev < 0), 1) + add_metric(s, "branin n=400", paste("NestedKriging", agg), q2 = q2(yt, pk$mean)) + check_greater(s, paste0("NestedKriging(", agg, "): Q2"), q2(yt, pk$mean), 0.95, severity = "warn") + if (agg == "NK") + check_less(s, "NestedKriging(NK): interpolates (max |m(X) - y| / sd(y))", + max(abs(predict(kk, Xl, return_stdev = FALSE)$mean - yl)) / sd(yl), 1e-3) + roundtrip_same_mean(s, paste0("NestedKriging(", agg, ")"), kk, Xt) + }) + } +} else skip(s, "NestedKriging", "not in this rlibkriging") + +# %% +me <- do.call(rbind, s$metrics) +print(me[me$metric == "q2", c("case", "package", "value")], row.names = FALSE) + +# %% +finalize(s) diff --git a/tools/src/julia_Kriging_cross-binding.jl b/tools/src/julia_Kriging_cross-binding.jl new file mode 100644 index 0000000..6e16558 --- /dev/null +++ b/tools/src/julia_Kriging_cross-binding.jl @@ -0,0 +1,52 @@ +# %% [markdown] +# # Julia · Kriging — cross-binding consistency: the Julia side +# +# Same protocol as `python_Kriging_cross-binding-reference.ipynb` (see there), on +# the shared datasets of `data/`; writes `results/xbind_julia.csv`, compared with +# the Python values by `tools/report.py`. +# +# The module is `JLibKriging` (the packaged binding, [JLibKriging.jl](https://github.com/libKriging/JLibKriging.jl)), +# imported under a short name so that its exported accessors (`X`, `y`, `theta`, ...) +# do not clash with the variables of the notebook. + +# %% +import JLibKriging as LK +include("tools/julia/checks.jl") +s = Suite("julia_Kriging_cross-binding") +rows = Vector{Vector{String}}() +record!(dataset, quantity, values) = + for (i, v) in enumerate(vec(Float64.(collect(values)))) + push!(rows, ["julia", dataset, quantity, string(i), @sprintf("%.17g", v)]) + end + +# %% +for name in ("branin", "hartmann3", "borehole") + Xd = read_data(name, "X"); yd = vec(read_data(name, "y")); Xt = read_data(name, "Xt")[1:50, :] + d = size(Xd, 2) + theta0 = fill(0.3, d) + kf = LK.Kriging(yd, Xd, "matern5_2"; regmodel="constant", optim="none", theta=theta0, is_theta_estim=false) + record!(name, "fixed_sigma2", LK.sigma2(kf)) + record!(name, "fixed_beta", LK.beta(kf)) + record!(name, "fixed_ll", LK.log_likelihood(kf)) + record!(name, "fixed_loo", LK.leave_one_out(kf)) + p = LK.predict(kf, Xt; return_stdev=true) + record!(name, "fixed_mean", p.mean) + record!(name, "fixed_stdev", p.stdev) + llf = LK.log_likelihood_fun(kf, theta0; return_grad=true) + record!(name, "llfun", llf.ll) + record!(name, "llfun_grad", llf.grad) + check_close!(s, "$name: log_likelihood() == log_likelihood_fun(theta0)", LK.log_likelihood(kf), llf.ll; rtol=1e-10) + k = LK.Kriging(yd, Xd, "matern5_2") + record!(name, "fit_theta", LK.theta(k)) + record!(name, "fit_ll", LK.log_likelihood(k)) + record!(name, "fit_mean", LK.predict(k, Xt; return_stdev=false).mean) + record!(name, "sim", LK.simulate(kf, 3, 123, Xt[1:5, :])) +end +open(joinpath(RESULTS_DIR, "xbind_julia.csv"), "w") do io + println(io, "lang,dataset,quantity,index,value") + for r in rows; println(io, join(r, ",")); end +end +println(length(rows), " values written") + +# %% +finalize!(s) diff --git a/tools/src/julia_Kriging_vs-GaussianProcesses.jl b/tools/src/julia_Kriging_vs-GaussianProcesses.jl new file mode 100644 index 0000000..e5f6a01 --- /dev/null +++ b/tools/src/julia_Kriging_vs-GaussianProcesses.jl @@ -0,0 +1,71 @@ +# %% [markdown] +# # Julia · Kriging vs GaussianProcesses.jl +# +# Default fits on the shared datasets of `data/` (Branin, Hartmann 3, Borehole): +# JLibKriging (`matern5_2`, constant trend, single start and 10 restarts) against +# [GaussianProcesses.jl](https://github.com/STOR-i/GaussianProcesses.jl) +# (Matérn 5/2 ARD, constant mean, MLE), warn-level comparison of Q². +# +# The module is `JLibKriging` (the packaged binding, [JLibKriging.jl](https://github.com/libKriging/JLibKriging.jl)), +# imported under a short name (as GaussianProcesses.jl): both export `predict`, +# `optimize!`, ... so every call is qualified. + +# %% +import JLibKriging as LK +include("tools/julia/checks.jl") +s = Suite("julia_Kriging_vs-GaussianProcesses") + +# %% [markdown] +# ## Installing GaussianProcesses.jl +# +# Added to the active Julia environment (with Optim.jl, its optimizer). Without +# it, the comparison is skipped (warn-level). + +# %% +import Pkg +have_gp = try + Pkg.add(["GaussianProcesses", "Optim"]) + @eval import GaussianProcesses as GP + @eval import Optim + true +catch e + println("GaussianProcesses.jl could not be installed: ", sprint(showerror, e)) + false +end + +# %% [markdown] +# ## Default fits + +# %% +for name in ("branin", "hartmann3", "borehole") + Xd = read_data(name, "X"); yd = vec(read_data(name, "y")); Xt = read_data(name, "Xt"); yt = vec(read_data(name, "yt")) + t = @elapsed k = LK.Kriging(yd, Xd, "matern5_2") + q_lk = q2(yt, LK.predict(k, Xt; return_stdev=false).mean) + add_metric!(s, name, "JLibKriging"; q2=q_lk, fit_s=t) + t = @elapsed k10 = LK.Kriging(yd, Xd, "matern5_2"; optim="BFGS10") + q_lk10 = q2(yt, LK.predict(k10, Xt; return_stdev=false).mean) + add_metric!(s, name, "JLibKriging BFGS10"; q2=q_lk10, fit_s=t) + add_check!(s, "$name: Q2(JLibKriging BFGS10) > 0.9", q_lk10 > 0.9; value=q_lk10, expected="> 0.9") + if have_gp + try + d = size(Xd, 2) + t = @elapsed begin + gp = GP.GP(permutedims(Xd), yd, GP.MeanConst(sum(yd) / length(yd)), + GP.Mat52Ard(zeros(d), log(sum((yd .- sum(yd) / length(yd)) .^ 2) / length(yd)) / 2), -8.0) + GP.optimize!(gp; noise=false) + end + mg, _ = GP.predict_f(gp, permutedims(Xt)) + q_gp = q2(yt, mg) + add_metric!(s, name, "GaussianProcesses.jl"; q2=q_gp, fit_s=t) + add_check!(s, "$name: Q2(JLibKriging) - Q2(GaussianProcesses.jl)", q_lk - q_gp > -0.02; + value=q_lk - q_gp, expected="> -0.02", severity="warn") + catch e + add_check!(s, "$name: GaussianProcesses.jl fit", false; value=string(typeof(e)), severity="warn") + end + end +end +have_gp || add_check!(s, "GaussianProcesses.jl available", false; value="absent", severity="warn") +foreach(r -> println(join(r[3:6], " ")), s.metrics) + +# %% +finalize!(s) diff --git a/tools/src/julia_all-classes_api-tour.jl b/tools/src/julia_all-classes_api-tour.jl new file mode 100644 index 0000000..5b743d1 --- /dev/null +++ b/tools/src/julia_all-classes_api-tour.jl @@ -0,0 +1,198 @@ +# %% [markdown] +# # Julia · all classes — every libKriging model class and fit method through JLibKriging +# +# Same tour as `R_all-classes_api-tour.ipynb` (see there for the checks): Branin +# on [0,1]², every class and fit method of the Julia binding, each with the +# checks that hold for it whatever the data. +# +# The module is `JLibKriging` (the packaged binding, [JLibKriging.jl](https://github.com/libKriging/JLibKriging.jl)), +# imported under a short name so that its exported accessors (`X`, `y`, `theta`, ...) +# do not clash with the variables of the notebook. + +# %% +import JLibKriging as LK +using Random +include("tools/julia/checks.jl") +s = Suite("julia_all-classes_api-tour") + +branin(U) = begin + x1 = 15 .* U[:, 1] .- 5; x2 = 15 .* U[:, 2] + (x2 .- 5.1 / (4π^2) .* x1 .^ 2 .+ 5 / π .* x1 .- 6) .^ 2 .+ 10 * (1 - 1 / (8π)) .* cos.(x1) .+ 10 +end +rng = MersenneTwister(1) +lhs(n, d) = hcat([(randperm(rng, n) .- rand(rng, n)) ./ n for _ in 1:d]...) +X = lhs(30, 2); y = branin(X) +Xt = rand(rng, 500, 2); yt = branin(Xt) +sd(v) = sqrt(sum((v .- sum(v) / length(v)) .^ 2) / (length(v) - 1)) +finite_count(v...) = count(!isfinite, vcat(v...)) + +function check_runs!(f, s, check) + try + f() + catch e + add_check!(s, check, false; value=string(typeof(e)), expected="no error", detail=first(sprint(showerror, e), 200)) + end +end + +function roundtrip!(s, label, k, Xn) + check_runs!(s, "$label: save + load") do + f = joinpath(mktempdir(), "k.json") + LK.save(k, f) + k2 = LK.load(f) + check_close!(s, "$label: load(save(k)) predicts the same mean", + LK.predict(k2, Xn; return_stdev=false).mean, LK.predict(k, Xn; return_stdev=false).mean; rtol=1e-10) + end +end + +# %% [markdown] +# ## `Kriging`, noise-free + +# %% +k = LK.Kriging(y, X, "matern5_2") +p = LK.predict(k, Xt; return_stdev=true) +add_metric!(s, "branin", "Kriging"; q2=q2(yt, p.mean)) +add_check!(s, "Kriging: Q2", q2(yt, p.mean) > 0.95; value=q2(yt, p.mean), expected="> 0.95") +e = maximum(abs.(LK.predict(k, X; return_stdev=false).mean .- y)) / sd(y) +add_check!(s, "Kriging: interpolates (max |m(X) - y| / sd(y))", e < 1e-6; value=e, expected="< 1e-6") +S = LK.simulate(k, 5, 2, X) +e = maximum(abs.(S .- y)) / sqrt(LK.sigma2(k)) +add_check!(s, "Kriging: conditional simulations go through the data (max |sim(X) - y| / sigma)", e < 1e-4; + value=e, expected="< 1e-4") +add_check!(s, "Kriging: stdev > 0 off the design", minimum(p.stdev) > 0; value=minimum(p.stdev), expected="> 0") +Xu = lhs(5, 2); yu = branin(Xu) +ku = copy(k) +LK.update!(ku, yu, Xu; refit=false) +k35 = LK.Kriging(vcat(y, yu), vcat(X, Xu), "matern5_2"; optim="none", + theta=vec(LK.theta(k)), is_theta_estim=false, sigma2=LK.sigma2(k), is_sigma2_estim=false) +check_close!(s, "Kriging: update!(refit=false) == fit on all points (same theta, sigma2)", + LK.predict(ku, Xt; return_stdev=false).mean, LK.predict(k35, Xt; return_stdev=false).mean; + rtol=1e-6, atol=1e-8 * sd(y)) +roundtrip!(s, "Kriging", k, Xt) + +# %% [markdown] +# ## `Kriging` with noise: estimated nugget, known variances +# +# Noise of standard deviation 5% of sd(y). The nugget model is fitted with 10 +# restarts (`optim="BFGS10"`); the default single start is compared with it +# (warn-level: a lower likelihood means a local optimum). As DiceKriging's +# nugget, the predictor returns y exactly *at* the design points (the nugget is +# a micro-scale variation): smoothing is checked next to them. + +# %% +sd_noise = 0.05 * sd(y) +yn = y .+ sd_noise .* randn(rng, length(y)) +kn = LK.Kriging(yn, X, "matern5_2"; noise="nugget", optim="BFGS10") +mn = LK.predict(kn, Xt; return_stdev=false).mean +add_metric!(s, "branin noisy", "Kriging nugget BFGS10"; q2=q2(yt, mn)) +add_check!(s, "Kriging(nugget): estimated nugget > 0", LK.nugget(kn) > 0; value=LK.nugget(kn), expected="> 0") +add_check!(s, "Kriging(nugget): Q2", q2(yt, mn) > 0.95; value=q2(yt, mn), expected="> 0.95") +r = sqrt(sum((LK.predict(kn, min.(X .+ 1e-7, 1.0); return_stdev=false).mean .- yn) .^ 2) / length(yn)) / sd_noise +add_check!(s, "Kriging(nugget): smooths the noisy data next to the design points (rms residual / noise sd)", r > 0.3; + value=r, expected="> 0.3") +kn1 = LK.Kriging(yn, X, "matern5_2"; noise="nugget") +add_metric!(s, "branin noisy", "Kriging nugget BFGS"; q2=q2(yt, LK.predict(kn1, Xt; return_stdev=false).mean)) +dll = LK.log_likelihood(kn1) - LK.log_likelihood(kn) +add_check!(s, "Kriging(nugget): LL(single start) - LL(10 restarts)", dll > -1; value=dll, expected="> -1", severity="warn") +roundtrip!(s, "Kriging(nugget)", kn, Xt) +v = fill(sd_noise^2, length(yn)) +kh = LK.Kriging(yn, X, "matern5_2"; noise=v) +check_close!(s, "Kriging(noise = variances): noise() returns them", LK.noise(kh), v; rtol=1e-12) +ph = LK.predict(kh, Xt; return_stdev=true) +add_check!(s, "Kriging(noise = variances): finite predictions", finite_count(ph.mean, ph.stdev) == 0; + value=finite_count(ph.mean, ph.stdev), expected="0 non-finite") +add_metric!(s, "branin noisy", "Kriging known noise"; q2=q2(yt, ph.mean)) + +# %% [markdown] +# ## `Kriging` with an approximate fit: Vecchia, Nyström (n = 400) + +# %% +Xl = lhs(400, 2); yl = branin(Xl) +for m in ("vecchia(20)", "nystrom(60)") + short = split(m, "(")[1] + label = "Kriging(method = '$m')" + check_runs!(s, "$label: fit + predict") do + ka = LK.Kriging(yl, Xl, "matern5_2"; method=m) + add_check!(s, "$label: method()", LK.method(ka) == m; value=LK.method(ka), expected=m) + pa = LK.predict(ka, Xt; return_stdev=true, method=String(short)) + add_check!(s, "$label: finite predictions", finite_count(pa.mean, pa.stdev) == 0; + value=finite_count(pa.mean, pa.stdev), expected="0 non-finite") + add_metric!(s, "branin n=400", "Kriging $m"; q2=q2(yt, pa.mean)) + add_check!(s, "$label: Q2", q2(yt, pa.mean) > 0.95; value=q2(yt, pa.mean), expected="> 0.95", severity="warn") + end +end + +# %% [markdown] +# ## `WarpKriging`: continuous warpings, then a categorical input +# +# The categorical case: x₂ restricted to 5 levels (coded 0, …, 4), embedded in ℝ². + +# %% +check_runs!(s, "WarpKriging(kumaraswamy, kumaraswamy): fit + predict") do + kw = LK.WarpKriging(y, X, ["kumaraswamy", "kumaraswamy"], "matern5_2") + pw = LK.predict(kw, Xt; return_stdev=true) + add_check!(s, "WarpKriging(kumaraswamy): finite predictions", finite_count(pw.mean, pw.stdev) == 0; + value=finite_count(pw.mean, pw.stdev), expected="0 non-finite") + add_metric!(s, "branin", "WarpKriging kumaraswamy"; q2=q2(yt, pw.mean)) + add_check!(s, "WarpKriging(kumaraswamy): Q2", q2(yt, pw.mean) > 0.9; value=q2(yt, pw.mean), expected="> 0.9", + severity="warn") + roundtrip!(s, "WarpKriging(kumaraswamy)", kw, Xt) +end +lev(U) = hcat(U[:, 1], round.(U[:, 2] .* 4)) +unlev(L) = hcat(L[:, 1], L[:, 2] ./ 4) +Xc = lev(lhs(40, 2)); yc = branin(unlev(Xc)) +Xtc = lev(Xt); ytc = branin(unlev(Xtc)) +check_runs!(s, "WarpKriging(none, categorical(5,2)): fit + predict") do + kc = LK.WarpKriging(yc, Xc, ["none", "categorical(5,2)"], "matern5_2") + pc = LK.predict(kc, Xtc; return_stdev=true) + add_check!(s, "WarpKriging(categorical): finite predictions", finite_count(pc.mean, pc.stdev) == 0; + value=finite_count(pc.mean, pc.stdev), expected="0 non-finite") + add_metric!(s, "branin, x2 categorical", "WarpKriging categorical"; q2=q2(ytc, pc.mean)) + add_check!(s, "WarpKriging(categorical): Q2", q2(ytc, pc.mean) > 0.8; value=q2(ytc, pc.mean), expected="> 0.8", + severity="warn") + roundtrip!(s, "WarpKriging(categorical)", kc, Xtc) +end + +# %% [markdown] +# ## `MLPKriging`: a joint feature map ℝ² → ℝ² before the kernel + +# %% +check_runs!(s, "MLPKriging: fit + predict") do + km = LK.MLPKriging(y, X, [8], 2; activation="tanh", kernel="matern5_2", optim="BFGS10") + pm = LK.predict(km, Xt; return_stdev=true) + add_check!(s, "MLPKriging: finite predictions", finite_count(pm.mean, pm.stdev) == 0; + value=finite_count(pm.mean, pm.stdev), expected="0 non-finite") + add_metric!(s, "branin", "MLPKriging tanh [8] -> 2, BFGS10"; q2=q2(yt, pm.mean)) + add_check!(s, "MLPKriging: Q2", q2(yt, pm.mean) > 0.9; value=q2(yt, pm.mean), expected="> 0.9", severity="warn") + roundtrip!(s, "MLPKriging", km, Xt) + kd = LK.MLPKriging(y, X, [8], 2; activation="tanh", kernel="matern5_2") # default optimizer + qd = q2(yt, LK.predict(kd, Xt; return_stdev=false).mean) + add_metric!(s, "branin", "MLPKriging tanh [8] -> 2, default optim"; q2=qd) + add_check!(s, "MLPKriging: Q2(default optim) - Q2(BFGS10)", qd - q2(yt, pm.mean) > -0.05; + value=qd - q2(yt, pm.mean), expected="> -0.05", severity="warn") +end + +# %% [markdown] +# ## `NestedKriging`: 4 groups of 100 points, `NK` and `PoE` aggregations + +# %% +for agg in ("NK", "PoE") + label = "NestedKriging($agg)" + check_runs!(s, "$label: fit + predict") do + kk = LK.NestedKriging(yl, Xl, "matern5_2", 4; aggregation=agg, seed=3) + pk = LK.predict(kk, Xt; return_stdev=true) + add_check!(s, "$label: stdev >= 0", all(pk.stdev .>= 0); value=minimum(pk.stdev), expected=">= 0") + add_metric!(s, "branin n=400", "NestedKriging $agg"; q2=q2(yt, pk.mean)) + add_check!(s, "$label: Q2", q2(yt, pk.mean) > 0.95; value=q2(yt, pk.mean), expected="> 0.95", severity="warn") + if agg == "NK" + e = maximum(abs.(LK.predict(kk, Xl; return_stdev=false).mean .- yl)) / sd(yl) + add_check!(s, "NestedKriging(NK): interpolates (max |m(X) - y| / sd(y))", e < 1e-3; value=e, expected="< 1e-3") + end + roundtrip!(s, label, kk, Xt) + end +end + +# %% +foreach(r -> r[5] == "q2" && println(rpad(r[3], 24), rpad(r[4], 28), r[6]), s.metrics) + +# %% +finalize!(s) diff --git a/notebooks/octave/src/01_cross_binding.m b/tools/src/octave_Kriging_cross-binding.m similarity index 86% rename from notebooks/octave/src/01_cross_binding.m rename to tools/src/octave_Kriging_cross-binding.m index a63524b..ad4cf57 100644 --- a/notebooks/octave/src/01_cross_binding.m +++ b/tools/src/octave_Kriging_cross-binding.m @@ -1,13 +1,14 @@ % %% [markdown] -% # Octave 01 — Cross-binding consistency: the Octave side +% # Octave · Kriging — cross-binding consistency: the Octave side % -% Same protocol as `notebooks/python/11_cross_binding.ipynb` (see there), on the +% Same protocol as `python_Kriging_cross-binding-reference.ipynb` (see there), on the % shared datasets of `data/`; writes `results/xbind_octave.csv`, compared with % the Python values by `tools/report.py`. mLibKriging must be on the path % (`OCTAVE_PATH` or `addpath`). % %% -s = lkb_suite("oct01_cross_binding"); +addpath(fullfile(pwd, "tools", "octave")); +s = lkb_suite("octave_Kriging_cross-binding"); out = {}; function out = put(out, dataset, quantity, values) v = double(values(:)); @@ -45,7 +46,7 @@ S = kf.simulate(int32(3), int32(123), Xt(1:5, :), false); out = put(out, name, "sim", S); end -d = getenv("LKBENCH_RESULTS"); if isempty(d), d = fullfile("..", "..", "results"); end +d = getenv("LKBENCH_RESULTS"); if isempty(d), d = "results"; end if ~exist(d, "dir"), mkdir(d); end fid = fopen(fullfile(d, "xbind_octave.csv"), "w"); fprintf(fid, "lang,dataset,quantity,index,value\n"); diff --git a/notebooks/octave/src/02_vs_stk.m b/tools/src/octave_Kriging_vs-STK.m similarity index 75% rename from notebooks/octave/src/02_vs_stk.m rename to tools/src/octave_Kriging_vs-STK.m index 806af40..9e10b6c 100644 --- a/notebooks/octave/src/02_vs_stk.m +++ b/tools/src/octave_Kriging_vs-STK.m @@ -1,5 +1,5 @@ % %% [markdown] -% # Octave 02 — mLibKriging vs STK +% # Octave · Kriging vs STK % % [STK](https://github.com/stk-kriging/stk) (Small Toolbox for Kriging) is the % reference kriging toolbox for Octave/MATLAB. @@ -14,17 +14,35 @@ % 2. **Default fits** on the shared datasets of `data/`: Matérn 5/2, each % toolbox's own estimation (warn-level comparison of Q²). % -% STK's location: the `STK_PATH` environment variable (skipped if unset). +% mLibKriging must be on the path (`OCTAVE_PATH` or `addpath`), see +% `tools/install/octave.sh`. + +% %% +addpath(fullfile(pwd, "tools", "octave")); +s = lkb_suite("octave_Kriging_vs-STK"); + +% %% [markdown] +% ## Installing STK +% +% STK is fetched here (shallow clone of its repository into a temporary +% directory), unless the `STK_PATH` environment variable already points to a copy. +% Without STK, the comparison is skipped (warn-level). % %% -s = lkb_suite("oct02_vs_stk"); stk_path = getenv("STK_PATH"); -have_stk = ~isempty(stk_path) && exist(fullfile(stk_path, "stk_init.m"), "file"); +if isempty(stk_path) || ~exist(fullfile(stk_path, "stk_init.m"), "file") + stk_path = fullfile(tempdir(), "stk"); + if ~exist(fullfile(stk_path, "stk_init.m"), "file") + [status, out] = system(sprintf("git clone --depth 1 https://github.com/stk-kriging/stk.git %s 2>&1", stk_path)); + printf("git clone stk: status %d\n%s", status, out); + end +end +have_stk = exist(fullfile(stk_path, "stk_init.m"), "file") == 2; if have_stk run(fullfile(stk_path, "stk_init.m")); - printf("STK %s\n", stk_version()); + printf("STK %s from %s\n", stk_version(), stk_path); else - s = lkb_check(s, "STK available", false, "absent", "STK_PATH", "STK not found: comparison skipped", "warn"); + s = lkb_check(s, "STK available", false, "absent", stk_path, "STK could not be installed: comparison skipped", "warn"); end q2 = @(y, m) 1 - sum((y - m).^2) / sum((y - mean(y)).^2); datasets = {"branin", "hartmann3", "borehole"}; diff --git a/tools/src/octave_all-classes_api-tour.m b/tools/src/octave_all-classes_api-tour.m new file mode 100644 index 0000000..fa88339 --- /dev/null +++ b/tools/src/octave_all-classes_api-tour.m @@ -0,0 +1,213 @@ +% %% [markdown] +% # Octave · all classes — every libKriging model class and fit method through mLibKriging +% +% Same tour as `R_all-classes_api-tour.ipynb` (see there for the checks): Branin +% on [0,1]², every class and fit method of the Octave binding (positional +% arguments), each with the checks that hold for it whatever the data. A class +% missing from the installed mLibKriging is recorded as SKIP. mLibKriging must +% be on the path (`OCTAVE_PATH` or `addpath`), see `tools/install/octave.sh`. + +% %% +addpath(fullfile(pwd, "tools", "octave")); +s = lkb_suite("octave_all-classes_api-tour"); +q2 = @(y, m) 1 - sum((y - m).^2) / sum((y - mean(y)).^2); +branin = @(U) ((15 * U(:, 2)) - 5.1 / (4 * pi^2) * (15 * U(:, 1) - 5).^2 + 5 / pi * (15 * U(:, 1) - 5) - 6).^2 ... + + 10 * (1 - 1 / (8 * pi)) * cos(15 * U(:, 1) - 5) + 10; +rand("seed", 1); randn("seed", 1); +function U = lhs(n, d) + U = zeros(n, d); + for j = 1:d, U(:, j) = (randperm(n)' - rand(n, 1)) / n; end +end +X = lhs(30, 2); y = branin(X); +Xt = rand(500, 2); yt = branin(Xt); +tmp = tempname(); mkdir(tmp); +has_class = @(name) exist(name) == 2; % a class file on the path + +function s = roundtrip(s, label, k, Xn, loader) + f = fullfile(tempdir(), [regexprep(label, "[^A-Za-z0-9]", "_") ".json"]); + try + k.save(f); + k2 = loader(f); + m1 = k.predict(Xn, false, false, false); + m2 = k2.predict(Xn, false, false, false); + s = lkb_close(s, [label ": load(save(k)) predicts the same mean"], m2, m1, 1e-10); + catch err + s = lkb_check(s, [label ": save + load"], false, "error", "no error", err.message); + end +end + +function s = skip(s, check, reason) + s.checks(end + 1, :) = {s.name, "octave", check, "info", "SKIP", "", "", reason}; + printf("[SKIP] %s: %s\n", check, reason); +end + +% %% [markdown] +% ## `Kriging`, noise-free + +% %% +k = Kriging(y, X, "matern5_2"); +[m, sd] = k.predict(Xt, true, false, false); +s = lkb_metric(s, "branin", "Kriging", "q2", q2(yt, m)); +s = lkb_check(s, "Kriging: Q2", q2(yt, m) > 0.95, q2(yt, m), "> 0.95"); +mX = k.predict(X, false, false, false); +e = max(abs(mX - y)) / std(y); +s = lkb_check(s, "Kriging: interpolates (max |m(X) - y| / sd(y))", e < 1e-6, e, "< 1e-6"); +S = k.simulate(int32(5), int32(2), X, false); +e = max(max(abs(S - y))) / sqrt(k.sigma2()); +s = lkb_check(s, "Kriging: conditional simulations go through the data (max |sim(X) - y| / sigma)", e < 1e-4, e, "< 1e-4"); +s = lkb_check(s, "Kriging: stdev > 0 off the design", min(sd) > 0, min(sd), "> 0"); +Xu = lhs(5, 2); yu = branin(Xu); +ku = k.copy(); +ku.update(yu, Xu, false); +th = k.theta(); +k35 = Kriging([y; yu], [X; Xu], "matern5_2", "constant", false, "none", "LL", ... + Params("theta", th(:)', "is_theta_estim", false, "sigma2", k.sigma2(), "is_sigma2_estim", false)); +s = lkb_close(s, "Kriging: update(refit = false) == fit on all points (same theta, sigma2)", ... + ku.predict(Xt, false, false, false), k35.predict(Xt, false, false, false), 1e-6, 1e-8 * std(y)); +s = roundtrip(s, "Kriging", k, Xt, @load_kriging); + +% %% [markdown] +% ## `Kriging` with noise: estimated nugget, known variances +% +% Noise of standard deviation 5% of sd(y). The nugget model is fitted with 10 +% restarts (`"BFGS10"`); the default single start is compared with it +% (warn-level: a lower likelihood means a local optimum). As DiceKriging's +% nugget, the predictor returns y exactly *at* the design points (the nugget is +% a micro-scale variation): smoothing is checked next to them. + +% %% +sd_noise = 0.05 * std(y); +yn = y + sd_noise * randn(size(y)); +kn = Kriging(yn, X, "matern5_2", "constant", false, "BFGS10", "LL", [], "nugget"); +mn = kn.predict(Xt, false, false, false); +nug = kn.nugget(); ll10 = kn.logLikelihood(); +s = lkb_metric(s, "branin noisy", "Kriging nugget BFGS10", "q2", q2(yt, mn)); +s = lkb_check(s, "Kriging(nugget): estimated nugget > 0", nug > 0, nug, "> 0"); +s = lkb_check(s, "Kriging(nugget): Q2", q2(yt, mn) > 0.95, q2(yt, mn), "> 0.95"); +r = sqrt(mean((kn.predict(min(X + 1e-7, 1), false, false, false) - yn).^2)) / sd_noise; +s = lkb_check(s, "Kriging(nugget): smooths the noisy data next to the design points (rms residual / noise sd)", r > 0.3, r, "> 0.3"); +kn1 = Kriging(yn, X, "matern5_2", "constant", false, "BFGS", "LL", [], "nugget"); +ll1 = kn1.logLikelihood(); +s = lkb_metric(s, "branin noisy", "Kriging nugget BFGS", "q2", q2(yt, kn1.predict(Xt, false, false, false))); +s = lkb_check(s, "Kriging(nugget): LL(single start) - LL(10 restarts)", ll1 - ll10 > -1, ll1 - ll10, "> -1", "", "warn"); +s = roundtrip(s, "Kriging(nugget)", kn, Xt, @load_kriging); +v = sd_noise^2 * ones(size(yn)); +kh = Kriging(yn, X, "matern5_2", "constant", false, "BFGS", "LL", [], "heterogeneous", v); +s = lkb_close(s, "Kriging(noise = variances): noise() returns them", kh.noise(), v, 1e-12); +[mh, sh] = kh.predict(Xt, true, false, false); +s = lkb_check(s, "Kriging(noise = variances): finite predictions", all(isfinite([mh; sh])), sum(~isfinite([mh; sh])), "0 non-finite"); +s = lkb_metric(s, "branin noisy", "Kriging known noise", "q2", q2(yt, mh)); + +% %% [markdown] +% ## `Kriging` with an approximate fit: Vecchia, Nyström (n = 400) + +% %% +Xl = lhs(400, 2); yl = branin(Xl); +for mth = {"vecchia(20)", "nystrom(60)"} + mth = mth{1}; short = regexprep(mth, "\\(.*", ""); + label = ["Kriging(method = '" mth "')"]; + try + ka = Kriging(yl, Xl, "matern5_2", "constant", false, "BFGS", "LL", [], "none", [], mth); + got = ka.method(); + s = lkb_check(s, [label ": method()"], strcmp(got, mth), got, mth); + [ma, sa] = ka.predict(Xt, true, false, false, short); + s = lkb_check(s, [label ": finite predictions"], all(isfinite([ma; sa])), sum(~isfinite([ma; sa])), "0 non-finite"); + s = lkb_metric(s, "branin n=400", ["Kriging " mth], "q2", q2(yt, ma)); + s = lkb_check(s, [label ": Q2"], q2(yt, ma) > 0.95, q2(yt, ma), "> 0.95", "", "warn"); + catch err + s = lkb_check(s, [label ": fit + predict"], false, "error", "no error", err.message); + end +end + +% %% [markdown] +% ## `WarpKriging`: continuous warpings, then a categorical input +% +% The categorical case: x₂ restricted to 5 levels (coded 0, …, 4), embedded in ℝ². + +% %% +if has_class("WarpKriging") + try + kw = WarpKriging(y, X, {"kumaraswamy", "kumaraswamy"}, "matern5_2"); + [mw, sw] = kw.predict(Xt, true, false, false); + s = lkb_check(s, "WarpKriging(kumaraswamy): finite predictions", all(isfinite([mw; sw])), sum(~isfinite([mw; sw])), "0 non-finite"); + s = lkb_metric(s, "branin", "WarpKriging kumaraswamy", "q2", q2(yt, mw)); + s = lkb_check(s, "WarpKriging(kumaraswamy): Q2", q2(yt, mw) > 0.9, q2(yt, mw), "> 0.9", "", "warn"); + s = roundtrip(s, "WarpKriging(kumaraswamy)", kw, Xt, @load_kriging); + catch err + s = lkb_check(s, "WarpKriging(kumaraswamy): fit + predict", false, "error", "no error", err.message); + end + lev = @(U) [U(:, 1), round(U(:, 2) * 4)]; + unlev = @(L) [L(:, 1), L(:, 2) / 4]; + Xc = lev(lhs(40, 2)); yc = branin(unlev(Xc)); + Xtc = lev(Xt); ytc = branin(unlev(Xtc)); + try + kc = WarpKriging(yc, Xc, {"none", "categorical(5,2)"}, "matern5_2"); + [mc, sc] = kc.predict(Xtc, true, false, false); + s = lkb_check(s, "WarpKriging(categorical): finite predictions", all(isfinite([mc; sc])), sum(~isfinite([mc; sc])), "0 non-finite"); + s = lkb_metric(s, "branin, x2 categorical", "WarpKriging categorical", "q2", q2(ytc, mc)); + s = lkb_check(s, "WarpKriging(categorical): Q2", q2(ytc, mc) > 0.8, q2(ytc, mc), "> 0.8", "", "warn"); + s = roundtrip(s, "WarpKriging(categorical)", kc, Xtc, @load_kriging); + catch err + s = lkb_check(s, "WarpKriging(categorical): fit + predict", false, "error", "no error", err.message); + end +else + s = skip(s, "WarpKriging", "not in this mLibKriging"); +end + +% %% [markdown] +% ## `MLPKriging`: a joint feature map ℝ² → ℝ² before the kernel + +% %% +if has_class("MLPKriging") + try + km = MLPKriging(y, X, [8], 2, "tanh", "matern5_2", "constant", false, "BFGS10"); + [mm, sm] = km.predict(Xt, true, false, false); + s = lkb_check(s, "MLPKriging: finite predictions", all(isfinite([mm; sm])), sum(~isfinite([mm; sm])), "0 non-finite"); + s = lkb_metric(s, "branin", "MLPKriging tanh [8] -> 2, BFGS10", "q2", q2(yt, mm)); + s = lkb_check(s, "MLPKriging: Q2", q2(yt, mm) > 0.9, q2(yt, mm), "> 0.9", "", "warn"); + s = roundtrip(s, "MLPKriging", km, Xt, @load_kriging); + kd = MLPKriging(y, X, [8], 2, "tanh", "matern5_2"); % default optimizer + qd = q2(yt, kd.predict(Xt, false, false, false)); + s = lkb_metric(s, "branin", "MLPKriging tanh [8] -> 2, default optim", "q2", qd); + s = lkb_check(s, "MLPKriging: Q2(default optim) - Q2(BFGS10)", qd - q2(yt, mm) > -0.05, qd - q2(yt, mm), "> -0.05", "", "warn"); + catch err + s = lkb_check(s, "MLPKriging: fit + predict", false, "error", "no error", err.message); + end +else + s = skip(s, "MLPKriging", "not in this mLibKriging"); +end + +% %% [markdown] +% ## `NestedKriging`: 4 groups of 100 points, `NK` and `PoE` aggregations + +% %% +if has_class("NestedKriging") + for agg = {"NK", "PoE"} + agg = agg{1}; label = ["NestedKriging(" agg ")"]; + try + kk = NestedKriging(yl, Xl, "matern5_2", 4, agg, "kmeans", 3); + [mk, sk] = kk.predict(Xt); % stdev returned when asked for (2 outputs) + s = lkb_check(s, [label ": stdev >= 0"], all(sk >= 0), min(sk), ">= 0"); + s = lkb_metric(s, "branin n=400", ["NestedKriging " agg], "q2", q2(yt, mk)); + s = lkb_check(s, [label ": Q2"], q2(yt, mk) > 0.95, q2(yt, mk), "> 0.95", "", "warn"); + if strcmp(agg, "NK") + e = max(abs(kk.predict(Xl) - yl)) / std(yl); + s = lkb_check(s, "NestedKriging(NK): interpolates (max |m(X) - y| / sd(y))", e < 1e-3, e, "< 1e-3"); + end + f = fullfile(tmp, ["nested_" agg ".json"]); + kk.save(f); k2 = load_kriging(f); + s = lkb_close(s, [label ": load(save(k)) predicts the same mean"], k2.predict(Xt), kk.predict(Xt), 1e-10); + catch err + s = lkb_check(s, [label ": fit + predict"], false, "error", "no error", err.message); + end + end +else + s = skip(s, "NestedKriging", "not in this mLibKriging"); +end + +% %% +q = strcmp(s.metrics(:, 5), "q2"); +disp(s.metrics(q, [3, 4, 6])) + +% %% +lkb_finalize(s); diff --git a/notebooks/python/src/03_noise.py b/tools/src/python_Kriging-nugget-noise_noisy-data.py similarity index 97% rename from notebooks/python/src/03_noise.py rename to tools/src/python_Kriging-nugget-noise_noisy-data.py index 8097f4f..e5497d6 100644 --- a/notebooks/python/src/03_noise.py +++ b/tools/src/python_Kriging-nugget-noise_noisy-data.py @@ -1,5 +1,5 @@ # %% [markdown] -# # 03 — Noisy observations: `noise="nugget"` and known noise variances +# # Python · Kriging with noise (`noise="nugget"`, known variances) — noisy observations # # `Kriging` handles noisy responses through its `noise` argument: `"nugget"` # estimates one homogeneous noise variance, a vector gives a known variance per @@ -35,7 +35,7 @@ import lkbench from lkbench import TESTS, design, metrics, compat -suite = lkbench.Suite("py03_noise") +suite = lkbench.Suite("python_Kriging-nugget-noise_noisy-data") print("pylibkriging", compat.version()) diff --git a/notebooks/python/src/07_scalability.py b/tools/src/python_Kriging-vecchia-nystrom_large-n.py similarity index 72% rename from notebooks/python/src/07_scalability.py rename to tools/src/python_Kriging-vecchia-nystrom_large-n.py index 29ab65a..6759b35 100644 --- a/notebooks/python/src/07_scalability.py +++ b/tools/src/python_Kriging-vecchia-nystrom_large-n.py @@ -1,16 +1,17 @@ # %% [markdown] -# # 07 — Large n: Vecchia, Nyström, NestedKriging, subset of data +# # Python · Kriging with `method="vecchia"` / `"nystrom"` — large n # -# The exact fit costs O(n³). libKriging offers: +# The exact fit costs O(n³). `Kriging` offers: # # * `method="vecchia(m)"`: Vecchia likelihood with m conditioning neighbours # (local, recommended in low dimension); # * `method="nystrom(k)"`: low-rank Nyström approximation with k landmarks # (global); -# * `NestedKriging`: a partition into groups, one sub-model per group, -# aggregated predictions (`NK`, `PoE`, `gPoE`, `BCM`, `rBCM`); # * `Kriging.subsetOfData`: a representative subset before an exact fit. # +# (`NestedKriging`, the partition-based alternative, on the same problems: +# `python_NestedKriging_aggregations-large-n.ipynb`.) +# # Problems: Hartmann 3 (3D, local structure: Vecchia's ground) and the OTL # circuit (6D, smooth: Nyström's ground), n = 1500 (exact fit still affordable, # for reference) and n = 5000 (approximations only). @@ -20,9 +21,7 @@ # * the method is explicit at prediction time: `predict(method="exact")` on a # Nyström/Vecchia fit raises, `method=` gives finite predictions; # * accuracy of each approximation close to the exact fit (warn-level gaps, -# error-level floors) and faster than the exact fit at n = 1500 (warn-level); -# * `NestedKriging` `NK` interpolates the data (it is a proper kriging -# predictor); the PoE family does not need to. +# error-level floors) and faster than the exact fit at n = 1500 (warn-level). # %% import time @@ -34,7 +33,7 @@ import lkbench from lkbench import TESTS, design, metrics, compat -suite = lkbench.Suite("py07_scalability") +suite = lkbench.Suite("python_Kriging-vecchia-nystrom_large-n") print("pylibkriging", compat.version()) HAS_METHOD = compat.has("method") if not HAS_METHOD: @@ -94,11 +93,6 @@ def timed(f): "vecchia(30)": lambda Xa, ya: (lk.Kriging(ya, Xa, "matern5_2", method="vecchia(30)"), "vecchia"), "nystrom(100)": lambda Xa, ya: (lk.Kriging(ya, Xa, "matern5_2", method="nystrom(100)"), "nystrom"), }) - if compat.has("NestedKriging"): - variants["nested NK (20 groups)"] = lambda Xa, ya: ( - lk.NestedKriging(ya, Xa, "matern5_2", nb_groups=20, aggregation="NK", seed=1), None) - variants["nested rBCM (20 groups)"] = lambda Xa, ya: ( - lk.NestedKriging(ya, Xa, "matern5_2", nb_groups=20, aggregation="rBCM", seed=1), None) if compat.has("subsetOfData"): def sod(Xa, ya): idx = np.ravel(lk.Kriging.subsetOfData(Xa, n_max=500, method="kmeans", seed=1)).astype(int) @@ -121,27 +115,5 @@ def sod(Xa, ya): suite.less(f"{case}/{label}: fit time / exact fit time", t / t_exact, 1.0, severity="warn") suite.table().round(4) -# %% [markdown] -# ## `NestedKriging` aggregations: interpolation and variance - -# %% -if compat.has("NestedKriging"): - tf = TESTS["branin"] - X, y, Xt, yt = design.dataset(tf, 200, n_test=500, seed=53) - for agg in ["NK", "PoE", "gPoE", "BCM", "rBCM"]: - with suite.runs(f"nested {agg}: fit"): - k = lk.NestedKriging(y, X, "matern5_2", nb_groups=8, aggregation=agg, seed=2) - m, s = (np.ravel(a) for a in k.predict(Xt, return_stdev=True)[:2]) - suite.true(f"nested {agg}: stdev >= 0", np.all(s >= 0)) - suite.metric("branin/nested", agg, **metrics.all_metrics(yt, m, s)) - if agg == "NK": - mX = np.ravel(k.predict(X, return_stdev=True)[0]) - suite.less("nested NK: interpolates (max |m(X)-y| / sd(y))", np.max(np.abs(mX - y)) / np.std(y), - 1e-4) - display(suite.table().loc["branin/nested"].round(4)) - suite.raises("nested NK with a linear trend is rejected", - lambda: lk.NestedKriging(y, X, "matern5_2", nb_groups=4, aggregation="NK", regmodel="linear"), - severity="warn") - # %% suite.finalize() diff --git a/notebooks/python/src/11_cross_binding.py b/tools/src/python_Kriging_cross-binding-reference.py similarity index 93% rename from notebooks/python/src/11_cross_binding.py rename to tools/src/python_Kriging_cross-binding-reference.py index 098a3ae..a61ac4f 100644 --- a/notebooks/python/src/11_cross_binding.py +++ b/tools/src/python_Kriging_cross-binding-reference.py @@ -1,5 +1,5 @@ # %% [markdown] -# # 11 — Cross-binding consistency: the Python side +# # Python · Kriging — cross-binding consistency: the Python reference # # All bindings (Python, R, Octave, Julia) wrap the same C++ library: on the # same data with the same options they must return the same numbers. Each @@ -21,18 +21,16 @@ # %% import csv -from pathlib import Path import numpy as np import pylibkriging as lk import lkbench from lkbench import compat -from lkbench.checks import RESULTS_DIR +from lkbench.checks import DATA_DIR as DATA, RESULTS_DIR -suite = lkbench.Suite("py11_cross_binding") +suite = lkbench.Suite("python_Kriging_cross-binding-reference") print("pylibkriging", compat.version()) -DATA = Path("../../data") DATASETS = ["branin", "hartmann3", "borehole"] rows = [] diff --git a/notebooks/python/src/06_high_dimension_mlp.py b/tools/src/python_Kriging_high-dimension-normalize-screening.py similarity index 57% rename from notebooks/python/src/06_high_dimension_mlp.py rename to tools/src/python_Kriging_high-dimension-normalize-screening.py index 7888da7..a127c64 100644 --- a/notebooks/python/src/06_high_dimension_mlp.py +++ b/tools/src/python_Kriging_high-dimension-normalize-screening.py @@ -1,5 +1,5 @@ # %% [markdown] -# # 06 — Moderate and high dimension: engineering functions, `normalize`, `MLPKriging` +# # Python · Kriging — moderate and high dimension: engineering functions, `normalize`, screening # # | function | d | character | # |---|---|---| @@ -17,11 +17,10 @@ # inputs in native units (Borehole spans 0.05 to 50 000) must not break a fit; # * Q² floors per function (regression detection); # * the fitted θ of Welch's **inactive** inputs are larger than those of the -# active ones (screening ability, warn-level); -# * `MLPKriging` (deep kernel) on a function with a low-dimensional structure: -# finite predictions; with a multistart optimizer, a Q² close to plain -# `Kriging`'s (error-level floor); the default single-start fit is compared -# with the multistart one (warn-level). +# active ones (screening ability, warn-level). +# +# `MLPKriging` on a high-dimensional function with a low-dimensional structure: +# `python_MLPKriging_deep-kernel-projected-branin.ipynb`. # %% import time @@ -33,7 +32,7 @@ import lkbench from lkbench import TESTS, design, metrics, compat -suite = lkbench.Suite("py06_high_dimension_mlp") +suite = lkbench.Suite("python_Kriging_high-dimension-normalize-screening") print("pylibkriging", compat.version()) @@ -91,50 +90,5 @@ def pred(k, X): severity="warn") pd.Series(theta, index=[f"x{i + 1}" for i in range(20)]).round(3).to_frame("theta").T -# %% [markdown] -# ## `MLPKriging`: deep kernel on a low-dimensional structure -# -# f(x) = Branin(Wx) for x ∈ [0,1]¹⁰ and a fixed 2 × 10 projection W: the -# function only varies in a 2D subspace, which a joint feature map can find -# whereas a per-axis kernel cannot. - -# %% -if compat.has("MLPKriging"): - rng = np.random.default_rng(43) - W = rng.normal(size=(2, 10)) - W /= np.abs(W).sum(axis=1, keepdims=True) - tfb = TESTS["branin"] - - def projected(X): - Z = X @ W.T - Z = (Z - (W.clip(max=0).sum(axis=1))) / np.abs(W).sum(axis=1) # back to [0,1]^2 - return tfb.unit(Z) - - X = design.lhs(120, 10, seed=44); Xt = design.uniform(2000, 10, seed=45) - y, yt = projected(X), projected(Xt) - kp = lk.Kriging(y, X, "matern5_2") - res_k = metrics.all_metrics(yt, *pred(kp, Xt)) - suite.metric("projected_branin_10d", "Kriging", **res_k) - for act in ["tanh", "selu"]: - for optim in ["BFGS", "BFGS10"]: - case = f"MLPKriging({act}, {optim})" - with suite.runs(f"{case}: fit"): - t0 = time.time() - km = lk.MLPKriging(y, X, hidden_dims=[16, 8], d_out=2, activation=act, kernel="matern5_2", - optim=optim) - fit_s = time.time() - t0 - m, s = pred(km, Xt) - suite.finite(f"{case}: predictions finite", m) - res = metrics.all_metrics(yt, m, s) - suite.metric("projected_branin_10d", f"MLPKriging {act} {optim}", fit_s=fit_s, - ll=float(km.logLikelihood()), **res) - if optim == "BFGS10": - suite.greater(f"{case}: Q2", res["q2"], 0.8) - else: - suite.greater(f"{case}: Q2 - Q2(Kriging)", res["q2"] - res_k["q2"], -0.1, severity="warn") - display(suite.table().loc["projected_branin_10d"].round(4)) -else: - suite.skip("MLPKriging", "not available in this pylibkriging") - # %% suite.finalize() diff --git a/notebooks/python/src/01_kriging_kernels_trends.py b/tools/src/python_Kriging_kernels-trends-interpolation.py similarity index 96% rename from notebooks/python/src/01_kriging_kernels_trends.py rename to tools/src/python_Kriging_kernels-trends-interpolation.py index eed51aa..f57e899 100644 --- a/notebooks/python/src/01_kriging_kernels_trends.py +++ b/tools/src/python_Kriging_kernels-trends-interpolation.py @@ -1,5 +1,5 @@ # %% [markdown] -# # 01 — Kriging: kernels × trends on 1D/2D/3D functions +# # Python · Kriging — kernels × trends, interpolation, predict (covariance, derivatives) # # Plain noise-free `Kriging`, every kernel (`gauss`, `exp`, `matern3_2`, # `matern5_2`) crossed with every trend (`constant`, `linear`, `interactive`, @@ -26,8 +26,8 @@ # `matern5_2` / `constant` with a multistart optimizer (`BFGS10`): a # regression shows up here first. The same floor for the default single-start # `BFGS` is warn-level: a single start can end in a poor local optimum of the -# likelihood (see notebook 02), which is worth seeing in the report but is not -# a defect of the model; +# likelihood (see `python_Kriging_objectives-LL-LOO-LMP-gradients.ipynb`), +# which is worth seeing in the report but is not a defect of the model; # * **sanity of the fitted hyperparameters**: finite, positive θ and σ²; # * **consistency**: `predict` on a batch equals `predict` point by point, the # returned covariance has the predicted variances on its diagonal, and is @@ -44,7 +44,7 @@ import lkbench from lkbench import TESTS, design, metrics, compat -suite = lkbench.Suite("py01_kernels_trends") +suite = lkbench.Suite("python_Kriging_kernels-trends-interpolation") print("pylibkriging", compat.version()) KERNELS = ["gauss", "exp", "matern3_2", "matern5_2"] diff --git a/notebooks/python/src/02_objectives_gradients.py b/tools/src/python_Kriging_objectives-LL-LOO-LMP-gradients.py similarity index 98% rename from notebooks/python/src/02_objectives_gradients.py rename to tools/src/python_Kriging_objectives-LL-LOO-LMP-gradients.py index 812e97f..38c8af4 100644 --- a/notebooks/python/src/02_objectives_gradients.py +++ b/tools/src/python_Kriging_objectives-LL-LOO-LMP-gradients.py @@ -1,5 +1,5 @@ # %% [markdown] -# # 02 — Objectives (LL, LOO, LMP), their gradients and optima +# # Python · Kriging — objectives (LL, LOO, LMP), their gradients and optima # # The hyperparameters θ of `Kriging` are estimated by maximizing the # log-likelihood (`LL`), maximizing the log marginal posterior (`LMP`), or @@ -32,7 +32,7 @@ import lkbench from lkbench import TESTS, design, metrics, compat -suite = lkbench.Suite("py02_objectives_gradients") +suite = lkbench.Suite("python_Kriging_objectives-LL-LOO-LMP-gradients") print("pylibkriging", compat.version()) CASES = {"branin": 25, "hartmann3": 40, "ishigami": 60, "friedman": 50} KERNELS = ["gauss", "exp", "matern3_2", "matern5_2"] diff --git a/notebooks/python/src/09_robustness.py b/tools/src/python_Kriging_robustness-invalid-inputs.py similarity index 98% rename from notebooks/python/src/09_robustness.py rename to tools/src/python_Kriging_robustness-invalid-inputs.py index 71d7000..fff6257 100644 --- a/notebooks/python/src/09_robustness.py +++ b/tools/src/python_Kriging_robustness-invalid-inputs.py @@ -1,5 +1,5 @@ # %% [markdown] -# # 09 — Robustness: degenerate and adversarial inputs +# # Python · Kriging — robustness: degenerate and adversarial inputs # # What a user eventually feeds a surrogate model, and what must happen: # @@ -27,7 +27,7 @@ import lkbench from lkbench import TESTS, design, compat -suite = lkbench.Suite("py09_robustness") +suite = lkbench.Suite("python_Kriging_robustness-invalid-inputs") print("pylibkriging", compat.version()) tf = TESTS["branin"] X, y, Xt, yt = design.dataset(tf, 20, n_test=200, seed=71) diff --git a/notebooks/python/src/04_update_simulate.py b/tools/src/python_Kriging_update-simulate.py similarity index 98% rename from notebooks/python/src/04_update_simulate.py rename to tools/src/python_Kriging_update-simulate.py index 2041635..0239782 100644 --- a/notebooks/python/src/04_update_simulate.py +++ b/tools/src/python_Kriging_update-simulate.py @@ -1,5 +1,5 @@ # %% [markdown] -# # 04 — `update`, `simulate`, `update_simulate` +# # Python · Kriging — `update`, `simulate`, `update_simulate` # # Sequential use of a fitted model (sequential design, Bayesian optimization): # @@ -24,7 +24,7 @@ import lkbench from lkbench import TESTS, design, compat -suite = lkbench.Suite("py04_update_simulate") +suite = lkbench.Suite("python_Kriging_update-simulate") print("pylibkriging", compat.version()) diff --git a/notebooks/python/src/10_compare_python_packages.py b/tools/src/python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy.py similarity index 90% rename from notebooks/python/src/10_compare_python_packages.py rename to tools/src/python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy.py index a291ec8..9ea756b 100644 --- a/notebooks/python/src/10_compare_python_packages.py +++ b/tools/src/python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy.py @@ -1,5 +1,5 @@ # %% [markdown] -# # 10 — Comparison with Python GP packages +# # Python · Kriging vs scikit-learn, SMT, OpenTURNS, GPy # # Two kinds of comparisons, which should not be confused: # @@ -15,8 +15,29 @@ # benchmark, and the checks are warn-level: libKriging should not be # markedly less accurate than the best package. # -# Packages (each optional: skipped if not installed): scikit-learn, SMT, -# OpenTURNS, GPy. +# Packages (each installed below, and skipped if that fails): scikit-learn, +# SMT, OpenTURNS, GPy. + +# %% [markdown] +# ## Installing scikit-learn, SMT, OpenTURNS, GPy +# +# With pip, into the kernel's environment, numpy pinned to its installed version +# (a package that would downgrade it is not installed: the libKriging extension +# is built against it). A package that cannot be installed is skipped, and +# reported. + +# %% +import importlib +import subprocess +import sys + +import numpy + +for package in ["scikit-learn", "smt", "openturns", "GPy"]: + r = subprocess.run([sys.executable, "-m", "pip", "install", "--quiet", package, f"numpy=={numpy.__version__}"], + capture_output=True, text=True) + print(f"{package}: " + ("installed" if r.returncode == 0 else "NOT installed\n" + r.stderr.strip()[-800:])) +importlib.invalidate_caches() # %% import time @@ -30,7 +51,7 @@ from lkbench import TESTS, design, metrics, compat warnings.filterwarnings("ignore") -suite = lkbench.Suite("py10_compare_python_packages") +suite = lkbench.Suite("python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy") sk = compat.optional("sklearn.gaussian_process") smt = compat.optional("smt.surrogate_models") ot = compat.optional("openturns") diff --git a/tools/src/python_MLPKriging_deep-kernel-projected-branin.py b/tools/src/python_MLPKriging_deep-kernel-projected-branin.py new file mode 100644 index 0000000..849a2f1 --- /dev/null +++ b/tools/src/python_MLPKriging_deep-kernel-projected-branin.py @@ -0,0 +1,70 @@ +# %% [markdown] +# # Python · MLPKriging — deep kernel on a 10D function with a 2D structure +# +# f(x) = Branin(Wx) for x ∈ [0,1]¹⁰ and a fixed 2 × 10 projection W: the +# function only varies in a 2D subspace, which a joint feature map (an MLP from +# ℝ¹⁰ to ℝ², then a stationary kernel) can find, whereas a per-axis kernel +# cannot. +# +# Checks: finite predictions; with a multistart optimizer, a Q² floor +# (error-level); the single-start fits compared with plain `Kriging` +# (warn-level), for two activations (`tanh`, smooth; `selu`, the default, with +# a kink at 0). + +# %% +import time + +import numpy as np +import pylibkriging as lk + +import lkbench +from lkbench import TESTS, design, metrics, compat + +suite = lkbench.Suite("python_MLPKriging_deep-kernel-projected-branin") +print("pylibkriging", compat.version()) + + +def pred(k, X): + m, s, *_ = k.predict(X, return_stdev=True) + return np.ravel(m), np.ravel(s) + +# %% +if compat.has("MLPKriging"): + rng = np.random.default_rng(43) + W = rng.normal(size=(2, 10)) + W /= np.abs(W).sum(axis=1, keepdims=True) + tfb = TESTS["branin"] + + def projected(X): + Z = X @ W.T + Z = (Z - (W.clip(max=0).sum(axis=1))) / np.abs(W).sum(axis=1) # back to [0,1]^2 + return tfb.unit(Z) + + X = design.lhs(120, 10, seed=44); Xt = design.uniform(2000, 10, seed=45) + y, yt = projected(X), projected(Xt) + kp = lk.Kriging(y, X, "matern5_2") + res_k = metrics.all_metrics(yt, *pred(kp, Xt)) + suite.metric("projected_branin_10d", "Kriging", **res_k) + for act in ["tanh", "selu"]: + for optim in ["BFGS", "BFGS10"]: + case = f"MLPKriging({act}, {optim})" + with suite.runs(f"{case}: fit"): + t0 = time.time() + km = lk.MLPKriging(y, X, hidden_dims=[16, 8], d_out=2, activation=act, kernel="matern5_2", + optim=optim) + fit_s = time.time() - t0 + m, s = pred(km, Xt) + suite.finite(f"{case}: predictions finite", m) + res = metrics.all_metrics(yt, m, s) + suite.metric("projected_branin_10d", f"MLPKriging {act} {optim}", fit_s=fit_s, + ll=float(km.logLikelihood()), **res) + if optim == "BFGS10": + suite.greater(f"{case}: Q2", res["q2"], 0.8) + else: + suite.greater(f"{case}: Q2 - Q2(Kriging)", res["q2"] - res_k["q2"], -0.1, severity="warn") + display(suite.table().loc["projected_branin_10d"].round(4)) +else: + suite.skip("MLPKriging", "not available in this pylibkriging") + +# %% +suite.finalize() diff --git a/tools/src/python_NestedKriging_aggregations-large-n.py b/tools/src/python_NestedKriging_aggregations-large-n.py new file mode 100644 index 0000000..f02862b --- /dev/null +++ b/tools/src/python_NestedKriging_aggregations-large-n.py @@ -0,0 +1,101 @@ +# %% [markdown] +# # Python · NestedKriging — aggregations, large n +# +# `NestedKriging` partitions the design into groups, fits one `Kriging` +# sub-model per group (common hyperparameters), and aggregates the sub-model +# predictions: `NK` (nested kriging: kriging of the sub-predictors, a proper +# kriging predictor), or the products of experts `PoE`, `gPoE`, `BCM`, `rBCM` +# (cheaper, variances not fully consistent). +# +# Problems: as in `python_Kriging-vecchia-nystrom_large-n.ipynb`, Hartmann 3 +# and the OTL circuit with n = 1500 (exact fit as reference) and n = 5000; then +# every aggregation on Branin. +# +# Checks: +# * accuracy close to the exact fit (warn-level gap, error-level floor), and +# faster than the exact fit at n = 1500 (warn-level); +# * `NK` interpolates the data (it is a proper kriging predictor); the PoE family +# does not need to; all stdev ≥ 0; +# * `NK` with a non-constant trend is rejected (simple-kriging theory). + +# %% +import time + +import numpy as np +import pandas as pd +import pylibkriging as lk + +import lkbench +from lkbench import TESTS, design, metrics, compat + +suite = lkbench.Suite("python_NestedKriging_aggregations-large-n") +print("pylibkriging", compat.version()) +if not compat.has("NestedKriging"): + suite.skip("NestedKriging", "not available in this pylibkriging") + + +def pred(k, X, **kw): + m, s, *_ = k.predict(X, return_stdev=True, **kw) + return np.ravel(m), np.ravel(s) + + +def timed(f): + t0 = time.time() + out = f() + return out, time.time() - t0 + +# %% [markdown] +# ## Accuracy and cost against the exact fit + +# %% +CASES = {"hartmann3": (1500, 5000), "otl_circuit": (1500, 5000)} +FLOORS = {"hartmann3": 0.98, "otl_circuit": 0.99} +if compat.has("NestedKriging"): + for fname, (n_mid, n_big) in CASES.items(): + tf = TESTS[fname] + X, y, Xt, yt = design.dataset(tf, n_big, n_test=2000, seed=52) + ke, t_exact = timed(lambda: lk.Kriging(y[:n_mid], X[:n_mid], "matern5_2")) + res_e = metrics.all_metrics(yt, *pred(ke, Xt)) + suite.metric(f"{fname}/n={n_mid}", "exact", fit_s=t_exact, **res_e) + for agg in ["NK", "rBCM"]: + label = f"nested {agg} (20 groups)" + for n in (n_mid, n_big): + case = f"{fname}/n={n}" + with suite.runs(f"{case}/{label}: fit + predict"): + k, t = timed(lambda: lk.NestedKriging(y[:n], X[:n], "matern5_2", nb_groups=20, aggregation=agg, + seed=1)) + m, s = pred(k, Xt) + suite.finite(f"{case}/{label}: finite predictions", m) + res = metrics.all_metrics(yt, m, s) + suite.metric(case, label, fit_s=t, **res) + suite.greater(f"{case}/{label}: Q2", res["q2"], FLOORS[fname] - 0.05) + if n == n_mid: + suite.greater(f"{case}/{label}: Q2 - Q2(exact)", res["q2"] - res_e["q2"], -0.02, + severity="warn") + suite.less(f"{case}/{label}: fit time / exact fit time", t / t_exact, 1.0, severity="warn") + display(suite.table().round(4)) + +# %% [markdown] +# ## `NestedKriging` aggregations: interpolation and variance + +# %% +if compat.has("NestedKriging"): + tf = TESTS["branin"] + X, y, Xt, yt = design.dataset(tf, 200, n_test=500, seed=53) + for agg in ["NK", "PoE", "gPoE", "BCM", "rBCM"]: + with suite.runs(f"nested {agg}: fit"): + k = lk.NestedKriging(y, X, "matern5_2", nb_groups=8, aggregation=agg, seed=2) + m, s = (np.ravel(a) for a in k.predict(Xt, return_stdev=True)[:2]) + suite.true(f"nested {agg}: stdev >= 0", np.all(s >= 0)) + suite.metric("branin/nested", agg, **metrics.all_metrics(yt, m, s)) + if agg == "NK": + mX = np.ravel(k.predict(X, return_stdev=True)[0]) + suite.less("nested NK: interpolates (max |m(X)-y| / sd(y))", np.max(np.abs(mX - y)) / np.std(y), + 1e-4) + display(suite.table().loc["branin/nested"].round(4)) + suite.raises("nested NK with a linear trend is rejected", + lambda: lk.NestedKriging(y, X, "matern5_2", nb_groups=4, aggregation="NK", regmodel="linear"), + severity="warn") + +# %% +suite.finalize() diff --git a/notebooks/python/src/05_warpkriging.py b/tools/src/python_WarpKriging_warpings-categorical-ordinal.py similarity index 97% rename from notebooks/python/src/05_warpkriging.py rename to tools/src/python_WarpKriging_warpings-categorical-ordinal.py index 05d4477..95388c7 100644 --- a/notebooks/python/src/05_warpkriging.py +++ b/tools/src/python_WarpKriging_warpings-categorical-ordinal.py @@ -1,5 +1,5 @@ # %% [markdown] -# # 05 — `WarpKriging`: non-stationarity, categorical and ordinal inputs +# # Python · WarpKriging — non-stationarity, categorical and ordinal inputs # # `WarpKriging` learns a transform per input variable jointly with the GP # hyperparameters. Problems where a plain stationary GP is the wrong model: @@ -24,7 +24,7 @@ import lkbench from lkbench import TESTS, design, metrics, compat -suite = lkbench.Suite("py05_warpkriging") +suite = lkbench.Suite("python_WarpKriging_warpings-categorical-ordinal") print("pylibkriging", compat.version()) if not compat.has("WarpKriging"): suite.skip("WarpKriging", "not available in this pylibkriging") diff --git a/notebooks/python/src/08_api_persistence.py b/tools/src/python_all-classes_save-load-pickle-sklearn.py similarity index 90% rename from notebooks/python/src/08_api_persistence.py rename to tools/src/python_all-classes_save-load-pickle-sklearn.py index bc362a5..ea2533a 100644 --- a/notebooks/python/src/08_api_persistence.py +++ b/tools/src/python_all-classes_save-load-pickle-sklearn.py @@ -1,5 +1,5 @@ # %% [markdown] -# # 08 — API: save/load, pickle, copy, accessors, scikit-learn estimators +# # Python · all classes — save/load, pickle, copy, accessors, scikit-learn estimators # # Persistence and introspection, for every model class: # @@ -25,7 +25,7 @@ import lkbench from lkbench import TESTS, design, compat -suite = lkbench.Suite("py08_api_persistence") +suite = lkbench.Suite("python_all-classes_save-load-pickle-sklearn") print("pylibkriging", compat.version()) tmp = Path(tempfile.mkdtemp()) @@ -135,6 +135,28 @@ def predict_any(k, Xn): suite.true("summary() is a non-empty string", isinstance(k.summary(), str) and len(k.summary()) > 20) suite.true("is_theta_estim() after an optimized fit", bool(k.is_theta_estim())) +# %% [markdown] +# ## Installing scikit-learn +# +# Needed by the estimators of `pylibkriging.sklearn` below. With pip, into the +# kernel's environment, numpy pinned to its installed version (a package that +# would downgrade it is not installed: the libKriging extension is built +# against it). A package that cannot be installed is skipped, and +# reported. + +# %% +import importlib +import subprocess +import sys + +import numpy + +for package in ["scikit-learn"]: + r = subprocess.run([sys.executable, "-m", "pip", "install", "--quiet", package, f"numpy=={numpy.__version__}"], + capture_output=True, text=True) + print(f"{package}: " + ("installed" if r.returncode == 0 else "NOT installed\n" + r.stderr.strip()[-800:])) +importlib.invalidate_caches() + # %% [markdown] # ## scikit-learn estimators diff --git a/notebooks/python/src/00_test_functions.py b/tools/src/python_no-model_test-functions-reference-values.py similarity index 94% rename from notebooks/python/src/00_test_functions.py rename to tools/src/python_no-model_test-functions-reference-values.py index d6a133c..9135f87 100644 --- a/notebooks/python/src/00_test_functions.py +++ b/tools/src/python_no-model_test-functions-reference-values.py @@ -1,5 +1,5 @@ # %% [markdown] -# # 00 — Test functions +# # Python · no model — the test functions against their published reference values # # Every benchmark of this repository relies on the test functions of # `lkbench.functions` (after the [Virtual Library of Simulation Experiments](https://www.sfu.ca/~ssurjano/)). @@ -15,7 +15,7 @@ import lkbench from lkbench import TESTS, design -suite = lkbench.Suite("py00_test_functions") +suite = lkbench.Suite("python_no-model_test-functions-reference-values") # %% [markdown] # ## Reference values From c188a9cacfeb8bbadcf4a4e2c3bcd5e9c03f4d5d Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 14 Oct 2026 13:36:22 +0000 Subject: [PATCH 04/16] Notebooks named __; one notebook per Kriging class and binding - every notebook (and its suite, known issues, baselines) renamed from __ to __; CI matrices and report.py --expect select a binding by suffix (*_python.ipynb, ...); - the R, Octave and Julia all-classes tours split into one notebook per class: interpolation-simulate-update-save_Kriging, noisy-data_Kriging-nugget-noise, large-n_Kriging-vecchia-nystrom, warpings-categorical_WarpKriging, deep-kernel_MLPKriging, aggregations_NestedKriging (x R, octave, julia); - the Python all-classes persistence notebook split: Kriging variants in save-load-pickle-copy-accessors_Kriging_python, the scikit-learn estimators (and the install of scikit-learn) in sklearn-estimators_Kriging_python, and the save/load/pickle/deepcopy checks of WarpKriging, MLPKriging and NestedKriging in their own notebooks, through a shared lkbench.persistence.check; - tools/known_issues.csv: the two warn-level findings of the R/Octave/Julia notebooks, reported as libKriging/libKriging#382 (MLPKriging single-start fit) and libKriging/libKriging#383 (nugget single-start fit), are now KNOWN; - README: tables per binding, grouped by Kriging type. Run locally against libKriging master (ef052e1): the 15 Python notebooks, the R and Octave notebooks except vs-DiceKriging-RobustGaSP_Kriging_R (CRAN unreachable here): 0 FAIL; cross-binding R and Octave vs Python 36/36 each. Julia not run. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01QiFEE514dpwS57mcFWDeZG --- .github/workflows/notebooks.yml | 6 +- README.md | 68 +- R_all-classes_api-tour.ipynb | 320 --- ...gations-large-n_NestedKriging_python.ipynb | 31 +- aggregations_NestedKriging_R.ipynb | 125 ++ aggregations_NestedKriging_julia.ipynb | 128 ++ aggregations_NestedKriging_octave.ipynb | 138 ++ ...oss-binding-reference_Kriging_python.ipynb | 2 +- ...ing.ipynb => cross-binding_Kriging_R.ipynb | 4 +- ...ipynb => cross-binding_Kriging_julia.ipynb | 4 +- ...pynb => cross-binding_Kriging_octave.ipynb | 4 +- ...l-projected-branin_MLPKriging_python.ipynb | 30 +- deep-kernel_MLPKriging_R.ipynb | 124 ++ deep-kernel_MLPKriging_julia.ipynb | 126 ++ deep-kernel_MLPKriging_octave.ipynb | 132 ++ ...n-normalize-screening_Kriging_python.ipynb | 4 +- ...ation-simulate-update-save_Kriging_R.ipynb | 131 ++ ...n-simulate-update-save_Kriging_julia.ipynb | 134 ++ ...-simulate-update-save_Kriging_octave.ipynb | 134 ++ julia_all-classes_api-tour.ipynb | 313 --- ...-trends-interpolation_Kriging_python.ipynb | 4 +- large-n_Kriging-vecchia-nystrom_R.ipynb | 120 ++ large-n_Kriging-vecchia-nystrom_julia.ipynb | 126 ++ large-n_Kriging-vecchia-nystrom_octave.ipynb | 128 ++ ...rge-n_Kriging-vecchia-nystrom_python.ipynb | 4 +- noisy-data_Kriging-nugget-noise_R.ipynb | 137 ++ noisy-data_Kriging-nugget-noise_julia.ipynb | 141 ++ noisy-data_Kriging-nugget-noise_octave.ipynb | 140 ++ ...isy-data_Kriging-nugget-noise_python.ipynb | 2 +- ...-LL-LOO-LMP-gradients_Kriging_python.ipynb | 2 +- octave_all-classes_api-tour.ipynb | 328 --- ...all-classes_save-load-pickle-sklearn.ipynb | 296 --- ...stness-invalid-inputs_Kriging_python.ipynb | 2 +- ...pickle-copy-accessors_Kriging_python.ipynb | 171 ++ sklearn-estimators_Kriging_python.ipynb | 148 ++ ...ons-reference-values_no-model_python.ipynb | 2 +- tools/baseline/metrics-master.csv | 1912 ++++++++--------- tools/baseline/metrics-release.csv | 1848 ++++++++-------- tools/known_issues.csv | 26 +- tools/lkbench/__init__.py | 2 +- tools/lkbench/functions.py | 2 +- tools/lkbench/persistence.py | 50 + tools/report.py | 10 +- tools/run_notebooks.py | 4 +- tools/src/R_all-classes_api-tour.R | 205 -- ...regations-large-n_NestedKriging_python.py} | 17 +- tools/src/aggregations_NestedKriging_R.R | 70 + tools/src/aggregations_NestedKriging_julia.jl | 73 + tools/src/aggregations_NestedKriging_octave.m | 83 + ...cross-binding-reference_Kriging_python.py} | 2 +- ...ss-binding.R => cross-binding_Kriging_R.R} | 4 +- ...ding.jl => cross-binding_Kriging_julia.jl} | 4 +- ...nding.m => cross-binding_Kriging_octave.m} | 4 +- ...nel-projected-branin_MLPKriging_python.py} | 16 +- tools/src/deep-kernel_MLPKriging_R.R | 69 + tools/src/deep-kernel_MLPKriging_julia.jl | 71 + tools/src/deep-kernel_MLPKriging_octave.m | 77 + ...ion-normalize-screening_Kriging_python.py} | 4 +- ...rpolation-simulate-update-save_Kriging_R.R | 76 + ...tion-simulate-update-save_Kriging_julia.jl | 79 + ...tion-simulate-update-save_Kriging_octave.m | 79 + tools/src/julia_all-classes_api-tour.jl | 198 -- ...ls-trends-interpolation_Kriging_python.py} | 4 +- tools/src/large-n_Kriging-vecchia-nystrom_R.R | 65 + .../large-n_Kriging-vecchia-nystrom_julia.jl | 71 + .../large-n_Kriging-vecchia-nystrom_octave.m | 73 + ...large-n_Kriging-vecchia-nystrom_python.py} | 4 +- tools/src/noisy-data_Kriging-nugget-noise_R.R | 82 + .../noisy-data_Kriging-nugget-noise_julia.jl | 86 + .../noisy-data_Kriging-nugget-noise_octave.m | 85 + ...noisy-data_Kriging-nugget-noise_python.py} | 2 +- ...es-LL-LOO-LMP-gradients_Kriging_python.py} | 2 +- tools/src/octave_all-classes_api-tour.m | 213 -- ...on_all-classes_save-load-pickle-sklearn.py | 200 -- ...bustness-invalid-inputs_Kriging_python.py} | 2 +- ...ad-pickle-copy-accessors_Kriging_python.py | 99 + .../src/sklearn-estimators_Kriging_python.py | 88 + ...tions-reference-values_no-model_python.py} | 2 +- ...e.py => update-simulate_Kriging_python.py} | 2 +- ... => vs-DiceKriging-RobustGaSP_Kriging_R.R} | 2 +- ... => vs-GaussianProcesses_Kriging_julia.jl} | 2 +- ...iging_vs-STK.m => vs-STK_Kriging_octave.m} | 2 +- ...learn-SMT-OpenTURNS-GPy_Kriging_python.py} | 2 +- ...categorical-ordinal_WarpKriging_python.py} | 15 +- .../src/warpings-categorical_WarpKriging_R.R | 78 + .../warpings-categorical_WarpKriging_julia.jl | 83 + .../warpings-categorical_WarpKriging_octave.m | 89 + ...nb => update-simulate_Kriging_python.ipynb | 2 +- ... vs-DiceKriging-RobustGaSP_Kriging_R.ipynb | 2 +- ...> vs-GaussianProcesses_Kriging_julia.ipynb | 2 +- ...s-STK.ipynb => vs-STK_Kriging_octave.ipynb | 2 +- ...arn-SMT-OpenTURNS-GPy_Kriging_python.ipynb | 2 +- ...tegorical-ordinal_WarpKriging_python.ipynb | 29 +- warpings-categorical_WarpKriging_R.ipynb | 133 ++ warpings-categorical_WarpKriging_julia.ipynb | 138 ++ warpings-categorical_WarpKriging_octave.ipynb | 144 ++ 96 files changed, 6445 insertions(+), 4056 deletions(-) delete mode 100644 R_all-classes_api-tour.ipynb rename python_NestedKriging_aggregations-large-n.ipynb => aggregations-large-n_NestedKriging_python.ipynb (86%) create mode 100644 aggregations_NestedKriging_R.ipynb create mode 100644 aggregations_NestedKriging_julia.ipynb create mode 100644 aggregations_NestedKriging_octave.ipynb rename python_Kriging_cross-binding-reference.ipynb => cross-binding-reference_Kriging_python.ipynb (98%) rename R_Kriging_cross-binding.ipynb => cross-binding_Kriging_R.ipynb (96%) rename julia_Kriging_cross-binding.ipynb => cross-binding_Kriging_julia.ipynb (96%) rename octave_Kriging_cross-binding.ipynb => cross-binding_Kriging_octave.ipynb (96%) rename python_MLPKriging_deep-kernel-projected-branin.ipynb => deep-kernel-projected-branin_MLPKriging_python.ipynb (79%) create mode 100644 deep-kernel_MLPKriging_R.ipynb create mode 100644 deep-kernel_MLPKriging_julia.ipynb create mode 100644 deep-kernel_MLPKriging_octave.ipynb rename python_Kriging_high-dimension-normalize-screening.ipynb => high-dimension-normalize-screening_Kriging_python.ipynb (97%) create mode 100644 interpolation-simulate-update-save_Kriging_R.ipynb create mode 100644 interpolation-simulate-update-save_Kriging_julia.ipynb create mode 100644 interpolation-simulate-update-save_Kriging_octave.ipynb delete mode 100644 julia_all-classes_api-tour.ipynb rename python_Kriging_kernels-trends-interpolation.ipynb => kernels-trends-interpolation_Kriging_python.ipynb (98%) create mode 100644 large-n_Kriging-vecchia-nystrom_R.ipynb create mode 100644 large-n_Kriging-vecchia-nystrom_julia.ipynb create mode 100644 large-n_Kriging-vecchia-nystrom_octave.ipynb rename python_Kriging-vecchia-nystrom_large-n.ipynb => large-n_Kriging-vecchia-nystrom_python.ipynb (98%) create mode 100644 noisy-data_Kriging-nugget-noise_R.ipynb create mode 100644 noisy-data_Kriging-nugget-noise_julia.ipynb create mode 100644 noisy-data_Kriging-nugget-noise_octave.ipynb rename python_Kriging-nugget-noise_noisy-data.ipynb => noisy-data_Kriging-nugget-noise_python.ipynb (99%) rename python_Kriging_objectives-LL-LOO-LMP-gradients.ipynb => objectives-LL-LOO-LMP-gradients_Kriging_python.ipynb (99%) delete mode 100644 octave_all-classes_api-tour.ipynb delete mode 100644 python_all-classes_save-load-pickle-sklearn.ipynb rename python_Kriging_robustness-invalid-inputs.ipynb => robustness-invalid-inputs_Kriging_python.ipynb (99%) create mode 100644 save-load-pickle-copy-accessors_Kriging_python.ipynb create mode 100644 sklearn-estimators_Kriging_python.ipynb rename python_no-model_test-functions-reference-values.ipynb => test-functions-reference-values_no-model_python.ipynb (98%) create mode 100644 tools/lkbench/persistence.py delete mode 100644 tools/src/R_all-classes_api-tour.R rename tools/src/{python_NestedKriging_aggregations-large-n.py => aggregations-large-n_NestedKriging_python.py} (86%) create mode 100644 tools/src/aggregations_NestedKriging_R.R create mode 100644 tools/src/aggregations_NestedKriging_julia.jl create mode 100644 tools/src/aggregations_NestedKriging_octave.m rename tools/src/{python_Kriging_cross-binding-reference.py => cross-binding-reference_Kriging_python.py} (98%) rename tools/src/{R_Kriging_cross-binding.R => cross-binding_Kriging_R.R} (95%) rename tools/src/{julia_Kriging_cross-binding.jl => cross-binding_Kriging_julia.jl} (95%) rename tools/src/{octave_Kriging_cross-binding.m => cross-binding_Kriging_octave.m} (95%) rename tools/src/{python_MLPKriging_deep-kernel-projected-branin.py => deep-kernel-projected-branin_MLPKriging_python.py} (79%) create mode 100644 tools/src/deep-kernel_MLPKriging_R.R create mode 100644 tools/src/deep-kernel_MLPKriging_julia.jl create mode 100644 tools/src/deep-kernel_MLPKriging_octave.m rename tools/src/{python_Kriging_high-dimension-normalize-screening.py => high-dimension-normalize-screening_Kriging_python.py} (96%) create mode 100644 tools/src/interpolation-simulate-update-save_Kriging_R.R create mode 100644 tools/src/interpolation-simulate-update-save_Kriging_julia.jl create mode 100644 tools/src/interpolation-simulate-update-save_Kriging_octave.m delete mode 100644 tools/src/julia_all-classes_api-tour.jl rename tools/src/{python_Kriging_kernels-trends-interpolation.py => kernels-trends-interpolation_Kriging_python.py} (98%) create mode 100644 tools/src/large-n_Kriging-vecchia-nystrom_R.R create mode 100644 tools/src/large-n_Kriging-vecchia-nystrom_julia.jl create mode 100644 tools/src/large-n_Kriging-vecchia-nystrom_octave.m rename tools/src/{python_Kriging-vecchia-nystrom_large-n.py => large-n_Kriging-vecchia-nystrom_python.py} (97%) create mode 100644 tools/src/noisy-data_Kriging-nugget-noise_R.R create mode 100644 tools/src/noisy-data_Kriging-nugget-noise_julia.jl create mode 100644 tools/src/noisy-data_Kriging-nugget-noise_octave.m rename tools/src/{python_Kriging-nugget-noise_noisy-data.py => noisy-data_Kriging-nugget-noise_python.py} (99%) rename tools/src/{python_Kriging_objectives-LL-LOO-LMP-gradients.py => objectives-LL-LOO-LMP-gradients_Kriging_python.py} (99%) delete mode 100644 tools/src/octave_all-classes_api-tour.m delete mode 100644 tools/src/python_all-classes_save-load-pickle-sklearn.py rename tools/src/{python_Kriging_robustness-invalid-inputs.py => robustness-invalid-inputs_Kriging_python.py} (99%) create mode 100644 tools/src/save-load-pickle-copy-accessors_Kriging_python.py create mode 100644 tools/src/sklearn-estimators_Kriging_python.py rename tools/src/{python_no-model_test-functions-reference-values.py => test-functions-reference-values_no-model_python.py} (97%) rename tools/src/{python_Kriging_update-simulate.py => update-simulate_Kriging_python.py} (99%) rename tools/src/{R_Kriging_vs-DiceKriging-RobustGaSP.R => vs-DiceKriging-RobustGaSP_Kriging_R.R} (99%) rename tools/src/{julia_Kriging_vs-GaussianProcesses.jl => vs-GaussianProcesses_Kriging_julia.jl} (98%) rename tools/src/{octave_Kriging_vs-STK.m => vs-STK_Kriging_octave.m} (99%) rename tools/src/{python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy.py => vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python.py} (99%) rename tools/src/{python_WarpKriging_warpings-categorical-ordinal.py => warpings-categorical-ordinal_WarpKriging_python.py} (92%) create mode 100644 tools/src/warpings-categorical_WarpKriging_R.R create mode 100644 tools/src/warpings-categorical_WarpKriging_julia.jl create mode 100644 tools/src/warpings-categorical_WarpKriging_octave.m rename python_Kriging_update-simulate.ipynb => update-simulate_Kriging_python.ipynb (99%) rename R_Kriging_vs-DiceKriging-RobustGaSP.ipynb => vs-DiceKriging-RobustGaSP_Kriging_R.ipynb (99%) rename julia_Kriging_vs-GaussianProcesses.ipynb => vs-GaussianProcesses_Kriging_julia.ipynb (98%) rename octave_Kriging_vs-STK.ipynb => vs-STK_Kriging_octave.ipynb (99%) rename python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy.ipynb => vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python.ipynb (99%) rename python_WarpKriging_warpings-categorical-ordinal.ipynb => warpings-categorical-ordinal_WarpKriging_python.ipynb (92%) create mode 100644 warpings-categorical_WarpKriging_R.ipynb create mode 100644 warpings-categorical_WarpKriging_julia.ipynb create mode 100644 warpings-categorical_WarpKriging_octave.ipynb diff --git a/.github/workflows/notebooks.yml b/.github/workflows/notebooks.yml index 1d4b641..a51f081 100644 --- a/.github/workflows/notebooks.yml +++ b/.github/workflows/notebooks.yml @@ -18,7 +18,7 @@ name: notebooks # # A notebook job fails when one of its error-level checks fails; the report job # fails when any notebook failed or crashed, or when the bindings disagree -# (cross-binding protocol, python_Kriging_cross-binding-reference.ipynb). +# (cross-binding protocol, cross-binding-reference_Kriging_python.ipynb). # The binding installations are the scripts of tools/install/, usable locally too. on: @@ -111,10 +111,10 @@ jobs: import json, os from pathlib import Path channels = os.environ["CHANNELS"].split(",") - for lang, prefix in [("python", "python"), ("r", "R"), ("octave", "octave"), ("julia", "julia")]: + for lang, binding in [("python", "python"), ("r", "R"), ("octave", "octave"), ("julia", "julia")]: chans = [c for c in channels if c == "master" or lang != "julia"] # no Julia release include = [{"channel": c, "notebook": nb.stem} for c in chans - for nb in sorted(Path(".").glob(f"{prefix}_*.ipynb"))] + for nb in sorted(Path(".").glob(f"*_{binding}.ipynb"))] print(f"{lang}={json.dumps({'include': include or [{'channel': 'none', 'notebook': 'none'}]})}") EOF cat "$GITHUB_OUTPUT" diff --git a/README.md b/README.md index f578f01..ce7e202 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ the run, warn-level ones are reported), and CI executes each of them against the and aggregates the findings of every language into one report. The notebooks are at the root of the repository, named -`__.ipynb`; everything else (notebook sources, +`__.ipynb`; everything else (notebook sources, helpers, binding installation, CI scripts, known issues, baselines) is under [`tools/`](tools), and the shared datasets under [`data/`](data). @@ -29,44 +29,60 @@ the aggregated report with all CSV results: | notebook | kriging type | theme: problems, features | compared with | executed | |---|---|---|---|---| -| [python_no-model_test-functions-reference-values](python_no-model_test-functions-reference-values.ipynb) | — | the 21 test functions of [`tools/lkbench/functions.py`](tools/lkbench/functions.py) against published optima / ranges | — | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-python_no-model_test-functions-reference-values.zip) | -| [python_Kriging_kernels-trends-interpolation](python_Kriging_kernels-trends-interpolation.ipynb) | `Kriging` | 4 kernels × 4 trends on Forrester, Gramacy & Lee, Branin, six-hump camel, log-Goldstein-Price, Lim, Ishigami, Hartmann 3: interpolation, `predict` (covariance, derivatives) | finite differences | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-python_Kriging_kernels-trends-interpolation.zip) | -| [python_Kriging_objectives-LL-LOO-LMP-gradients](python_Kriging_objectives-LL-LOO-LMP-gradients.ipynb) | `Kriging` | `LL` / `LOO` / `LMP` objectives and gradients, optimality of θ̂, `leaveOneOutVec`, multistart (Branin, Hartmann 3, Ishigami, Friedman) | finite differences, explicit refits | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-python_Kriging_objectives-LL-LOO-LMP-gradients.zip) | -| [python_Kriging-nugget-noise_noisy-data](python_Kriging-nugget-noise_noisy-data.ipynb) | `Kriging` with `noise="nugget"` or variances | estimated nugget, heteroscedastic noise, calibration (Forrester, Friedman, Gramacy & Lee, Branin with replicates) | true noise, replicate averaging | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-python_Kriging-nugget-noise_noisy-data.zip) | -| [python_Kriging_update-simulate](python_Kriging_update-simulate.ipynb) | `Kriging` | `update`, `simulate`, `update_simulate` (Branin, Hartmann 3, Hartmann 6) | fits from scratch, moments | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-python_Kriging_update-simulate.zip) | -| [python_WarpKriging_warpings-categorical-ordinal](python_WarpKriging_warpings-categorical-ordinal.ipynb) | `WarpKriging` | affine, boxcox, kumaraswamy, knots, neural_mono, categorical, ordinal warpings (Gramacy & Lee 2012 / 2008, mixed categorical Branin, ordinal Hartmann 3) | plain `Kriging` | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-python_WarpKriging_warpings-categorical-ordinal.zip) | -| [python_Kriging_high-dimension-normalize-screening](python_Kriging_high-dimension-normalize-screening.ipynb) | `Kriging` | `normalize`, screening, 6D to 20D engineering functions (OTL circuit, piston, borehole, robot arm, Dette-Pepelyshev, wing weight, Welch) | unit-cube fit | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-python_Kriging_high-dimension-normalize-screening.zip) | -| [python_MLPKriging_deep-kernel-projected-branin](python_MLPKriging_deep-kernel-projected-branin.ipynb) | `MLPKriging` | deep kernel on a 10D function with a 2D structure, activations, multistart | plain `Kriging` | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-python_MLPKriging_deep-kernel-projected-branin.zip) | -| [python_Kriging-vecchia-nystrom_large-n](python_Kriging-vecchia-nystrom_large-n.ipynb) | `Kriging` with `method="vecchia(m)"` / `"nystrom(k)"` | large n (Hartmann 3, OTL circuit, n up to 5000), limit cases, `subsetOfData` | exact fit | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-python_Kriging-vecchia-nystrom_large-n.zip) | -| [python_NestedKriging_aggregations-large-n](python_NestedKriging_aggregations-large-n.ipynb) | `NestedKriging` | `NK`, `PoE`, `gPoE`, `BCM`, `rBCM` aggregations, large n | exact fit | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-python_NestedKriging_aggregations-large-n.zip) | -| [python_all-classes_save-load-pickle-sklearn](python_all-classes_save-load-pickle-sklearn.ipynb) | all classes | `save`/`load`, pickle, copy, accessors, scikit-learn estimators (Hartmann 3) | — | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-python_all-classes_save-load-pickle-sklearn.zip) | -| [python_Kriging_robustness-invalid-inputs](python_Kriging_robustness-invalid-inputs.ipynb) | `Kriging` | invalid inputs, duplicates, constant y, ill-conditioning, scale equivariance, extrapolation, cost in q (Branin) | — | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-python_Kriging_robustness-invalid-inputs.zip) | -| [python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy](python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy.ipynb) | `Kriging` | default fits on 10 functions, 2D to 10D | scikit-learn, SMT, OpenTURNS, GPy (exact equality at fixed hyperparameters + accuracy/cost) | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy.zip) | -| [python_Kriging_cross-binding-reference](python_Kriging_cross-binding-reference.ipynb) | `Kriging` | the cross-binding protocol on the datasets of `data/`: the Python reference | the other bindings (report) | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-python_Kriging_cross-binding-reference.zip) | +| [test-functions-reference-values_no-model_python](test-functions-reference-values_no-model_python.ipynb) | — | the 21 test functions of [`tools/lkbench/functions.py`](tools/lkbench/functions.py) against published optima / ranges | — | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-test-functions-reference-values_no-model_python.zip) | +| [kernels-trends-interpolation_Kriging_python](kernels-trends-interpolation_Kriging_python.ipynb) | `Kriging` | 4 kernels × 4 trends on Forrester, Gramacy & Lee, Branin, six-hump camel, log-Goldstein-Price, Lim, Ishigami, Hartmann 3: interpolation, `predict` (covariance, derivatives) | finite differences | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-kernels-trends-interpolation_Kriging_python.zip) | +| [objectives-LL-LOO-LMP-gradients_Kriging_python](objectives-LL-LOO-LMP-gradients_Kriging_python.ipynb) | `Kriging` | `LL` / `LOO` / `LMP` objectives and gradients, optimality of θ̂, `leaveOneOutVec`, multistart (Branin, Hartmann 3, Ishigami, Friedman) | finite differences, explicit refits | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-objectives-LL-LOO-LMP-gradients_Kriging_python.zip) | +| [update-simulate_Kriging_python](update-simulate_Kriging_python.ipynb) | `Kriging` | `update`, `simulate`, `update_simulate` (Branin, Hartmann 3, Hartmann 6) | fits from scratch, moments | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-update-simulate_Kriging_python.zip) | +| [high-dimension-normalize-screening_Kriging_python](high-dimension-normalize-screening_Kriging_python.ipynb) | `Kriging` | `normalize`, screening, 6D to 20D engineering functions (OTL circuit, piston, borehole, robot arm, Dette-Pepelyshev, wing weight, Welch) | unit-cube fit | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-high-dimension-normalize-screening_Kriging_python.zip) | +| [robustness-invalid-inputs_Kriging_python](robustness-invalid-inputs_Kriging_python.ipynb) | `Kriging` | invalid inputs, duplicates, constant y, ill-conditioning, scale equivariance, extrapolation, cost in q (Branin) | — | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-robustness-invalid-inputs_Kriging_python.zip) | +| [save-load-pickle-copy-accessors_Kriging_python](save-load-pickle-copy-accessors_Kriging_python.ipynb) | `Kriging` | `save`/`load`, pickle, copy, accessors, for every noise model, objective and fit method (Hartmann 3) | — | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-save-load-pickle-copy-accessors_Kriging_python.zip) | +| [sklearn-estimators_Kriging_python](sklearn-estimators_Kriging_python.ipynb) | `Kriging` | `pylibkriging.sklearn.KrigingRegressor`: `Pipeline`, `GridSearchCV`, `clone`, `sample_y` (Hartmann 3) | the native class | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-sklearn-estimators_Kriging_python.zip) | +| [vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python](vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python.ipynb) | `Kriging` | default fits on 10 functions, 2D to 10D | scikit-learn, SMT, OpenTURNS, GPy (exact equality at fixed hyperparameters + accuracy/cost) | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python.zip) | +| [cross-binding-reference_Kriging_python](cross-binding-reference_Kriging_python.ipynb) | `Kriging` | the cross-binding protocol on the datasets of `data/`: the Python reference | the other bindings (report) | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-cross-binding-reference_Kriging_python.zip) | +| [noisy-data_Kriging-nugget-noise_python](noisy-data_Kriging-nugget-noise_python.ipynb) | `Kriging` with `noise="nugget"` or variances | estimated nugget, heteroscedastic noise, calibration (Forrester, Friedman, Gramacy & Lee, Branin with replicates) | true noise, replicate averaging | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-noisy-data_Kriging-nugget-noise_python.zip) | +| [large-n_Kriging-vecchia-nystrom_python](large-n_Kriging-vecchia-nystrom_python.ipynb) | `Kriging` with `method="vecchia(m)"` / `"nystrom(k)"` | large n (Hartmann 3, OTL circuit, n up to 5000), limit cases, `subsetOfData` | exact fit | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-large-n_Kriging-vecchia-nystrom_python.zip) | +| [warpings-categorical-ordinal_WarpKriging_python](warpings-categorical-ordinal_WarpKriging_python.ipynb) | `WarpKriging` | affine, boxcox, kumaraswamy, knots, neural_mono, categorical, ordinal warpings (Gramacy & Lee 2012 / 2008, mixed categorical Branin, ordinal Hartmann 3); save/load, pickle | plain `Kriging` | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-warpings-categorical-ordinal_WarpKriging_python.zip) | +| [deep-kernel-projected-branin_MLPKriging_python](deep-kernel-projected-branin_MLPKriging_python.ipynb) | `MLPKriging` | deep kernel on a 10D function with a 2D structure, activations, multistart; save/load, pickle | plain `Kriging` | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-deep-kernel-projected-branin_MLPKriging_python.zip) | +| [aggregations-large-n_NestedKriging_python](aggregations-large-n_NestedKriging_python.ipynb) | `NestedKriging` | `NK`, `PoE`, `gPoE`, `BCM`, `rBCM` aggregations, large n; save/load, pickle | exact fit | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-aggregations-large-n_NestedKriging_python.zip) | ### R (`rlibkriging`) | notebook | kriging type | theme: problems, features | compared with | executed | |---|---|---|---|---| -| [R_all-classes_api-tour](R_all-classes_api-tour.ipynb) | all classes | `Kriging` (noise-free, nugget, known noise, Vecchia, Nyström), `WarpKriging` (continuous, categorical), `MLPKriging`, `NestedKriging`, `update`, `simulate`, `save`/`load` (Branin) | — | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-R_all-classes_api-tour.zip) | -| [R_Kriging_cross-binding](R_Kriging_cross-binding.ipynb) | `Kriging` | the cross-binding protocol, R side | Python (report) | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-R_Kriging_cross-binding.zip) | -| [R_Kriging_vs-DiceKriging-RobustGaSP](R_Kriging_vs-DiceKriging-RobustGaSP.ipynb) | `Kriging` | default fits (Branin, Hartmann 3, Borehole) | DiceKriging (exact equality), RobustGaSP | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-R_Kriging_vs-DiceKriging-RobustGaSP.zip) | +| [interpolation-simulate-update-save_Kriging_R](interpolation-simulate-update-save_Kriging_R.ipynb) | `Kriging` | interpolation, conditional simulation, `update`, `save`/`load` (Branin) | fit on all points | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-interpolation-simulate-update-save_Kriging_R.zip) | +| [cross-binding_Kriging_R](cross-binding_Kriging_R.ipynb) | `Kriging` | the cross-binding protocol, R side | Python (report) | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-cross-binding_Kriging_R.zip) | +| [vs-DiceKriging-RobustGaSP_Kriging_R](vs-DiceKriging-RobustGaSP_Kriging_R.ipynb) | `Kriging` | default fits (Branin, Hartmann 3, Borehole) | DiceKriging (exact equality), RobustGaSP | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-vs-DiceKriging-RobustGaSP_Kriging_R.zip) | +| [noisy-data_Kriging-nugget-noise_R](noisy-data_Kriging-nugget-noise_R.ipynb) | `Kriging` with nugget or known variances | estimated nugget (10 restarts vs single start), known noise variances (Branin + noise) | — | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-noisy-data_Kriging-nugget-noise_R.zip) | +| [large-n_Kriging-vecchia-nystrom_R](large-n_Kriging-vecchia-nystrom_R.ipynb) | `Kriging` with Vecchia / Nyström | approximate fits, `method()`, prediction with the fit method (Branin, n = 400) | — | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-large-n_Kriging-vecchia-nystrom_R.zip) | +| [warpings-categorical_WarpKriging_R](warpings-categorical_WarpKriging_R.ipynb) | `WarpKriging` | kumaraswamy warpings, a categorical input (Branin) | — | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-warpings-categorical_WarpKriging_R.zip) | +| [deep-kernel_MLPKriging_R](deep-kernel_MLPKriging_R.ipynb) | `MLPKriging` | joint feature map, multistart vs default optimizer (Branin) | — | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-deep-kernel_MLPKriging_R.zip) | +| [aggregations_NestedKriging_R](aggregations_NestedKriging_R.ipynb) | `NestedKriging` | `NK` and `PoE` aggregations, interpolation, `save`/`load` (Branin, n = 400) | — | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-aggregations_NestedKriging_R.zip) | ### Octave (`mLibKriging`) | notebook | kriging type | theme: problems, features | compared with | executed | |---|---|---|---|---| -| [octave_all-classes_api-tour](octave_all-classes_api-tour.ipynb) | all classes | the R tour, with the positional arguments of the Octave binding | — | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-octave_all-classes_api-tour.zip) | -| [octave_Kriging_cross-binding](octave_Kriging_cross-binding.ipynb) | `Kriging` | the cross-binding protocol, Octave side | Python (report) | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-octave_Kriging_cross-binding.zip) | -| [octave_Kriging_vs-STK](octave_Kriging_vs-STK.ipynb) | `Kriging` | default fits (Branin, Hartmann 3, Borehole) | STK (exact equality, Gaussian kernel) | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-octave_Kriging_vs-STK.zip) | +| [interpolation-simulate-update-save_Kriging_octave](interpolation-simulate-update-save_Kriging_octave.ipynb) | `Kriging` | interpolation, conditional simulation, `update`, `save`/`load` (Branin) | fit on all points | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-interpolation-simulate-update-save_Kriging_octave.zip) | +| [cross-binding_Kriging_octave](cross-binding_Kriging_octave.ipynb) | `Kriging` | the cross-binding protocol, Octave side | Python (report) | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-cross-binding_Kriging_octave.zip) | +| [vs-STK_Kriging_octave](vs-STK_Kriging_octave.ipynb) | `Kriging` | default fits (Branin, Hartmann 3, Borehole) | STK (exact equality, Gaussian kernel) | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-vs-STK_Kriging_octave.zip) | +| [noisy-data_Kriging-nugget-noise_octave](noisy-data_Kriging-nugget-noise_octave.ipynb) | `Kriging` with nugget or known variances | estimated nugget (10 restarts vs single start), known noise variances (Branin + noise) | — | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-noisy-data_Kriging-nugget-noise_octave.zip) | +| [large-n_Kriging-vecchia-nystrom_octave](large-n_Kriging-vecchia-nystrom_octave.ipynb) | `Kriging` with Vecchia / Nyström | approximate fits, `method()`, prediction with the fit method (Branin, n = 400) | — | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-large-n_Kriging-vecchia-nystrom_octave.zip) | +| [warpings-categorical_WarpKriging_octave](warpings-categorical_WarpKriging_octave.ipynb) | `WarpKriging` | kumaraswamy warpings, a categorical input (Branin) | — | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-warpings-categorical_WarpKriging_octave.zip) | +| [deep-kernel_MLPKriging_octave](deep-kernel_MLPKriging_octave.ipynb) | `MLPKriging` | joint feature map, multistart vs default optimizer (Branin) | — | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-deep-kernel_MLPKriging_octave.zip) | +| [aggregations_NestedKriging_octave](aggregations_NestedKriging_octave.ipynb) | `NestedKriging` | `NK` and `PoE` aggregations, interpolation, `save`/`load` (Branin, n = 400) | — | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-aggregations_NestedKriging_octave.zip) | ### Julia (`JLibKriging`) | notebook | kriging type | theme: problems, features | compared with | executed | |---|---|---|---|---| -| [julia_all-classes_api-tour](julia_all-classes_api-tour.ipynb) | all classes | the R tour, with the Julia binding | — | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-julia_all-classes_api-tour.zip) | -| [julia_Kriging_cross-binding](julia_Kriging_cross-binding.ipynb) | `Kriging` | the cross-binding protocol, Julia side | Python (report) | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-julia_Kriging_cross-binding.zip) | -| [julia_Kriging_vs-GaussianProcesses](julia_Kriging_vs-GaussianProcesses.ipynb) | `Kriging` | default fits (Branin, Hartmann 3, Borehole) | GaussianProcesses.jl | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-julia_Kriging_vs-GaussianProcesses.zip) | +| [interpolation-simulate-update-save_Kriging_julia](interpolation-simulate-update-save_Kriging_julia.ipynb) | `Kriging` | interpolation, conditional simulation, `update`, `save`/`load` (Branin) | fit on all points | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-interpolation-simulate-update-save_Kriging_julia.zip) | +| [cross-binding_Kriging_julia](cross-binding_Kriging_julia.ipynb) | `Kriging` | the cross-binding protocol, Julia side | Python (report) | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-cross-binding_Kriging_julia.zip) | +| [vs-GaussianProcesses_Kriging_julia](vs-GaussianProcesses_Kriging_julia.ipynb) | `Kriging` | default fits (Branin, Hartmann 3, Borehole) | GaussianProcesses.jl | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-vs-GaussianProcesses_Kriging_julia.zip) | +| [noisy-data_Kriging-nugget-noise_julia](noisy-data_Kriging-nugget-noise_julia.ipynb) | `Kriging` with nugget or known variances | estimated nugget (10 restarts vs single start), known noise variances (Branin + noise) | — | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-noisy-data_Kriging-nugget-noise_julia.zip) | +| [large-n_Kriging-vecchia-nystrom_julia](large-n_Kriging-vecchia-nystrom_julia.ipynb) | `Kriging` with Vecchia / Nyström | approximate fits, `method()`, prediction with the fit method (Branin, n = 400) | — | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-large-n_Kriging-vecchia-nystrom_julia.zip) | +| [warpings-categorical_WarpKriging_julia](warpings-categorical_WarpKriging_julia.ipynb) | `WarpKriging` | kumaraswamy warpings, a categorical input (Branin) | — | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-warpings-categorical_WarpKriging_julia.zip) | +| [deep-kernel_MLPKriging_julia](deep-kernel_MLPKriging_julia.ipynb) | `MLPKriging` | joint feature map, multistart vs default optimizer (Branin) | — | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-deep-kernel_MLPKriging_julia.zip) | +| [aggregations_NestedKriging_julia](aggregations_NestedKriging_julia.ipynb) | `NestedKriging` | `NK` and `PoE` aggregations, interpolation, `save`/`load` (Branin, n = 400) | — | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-aggregations_NestedKriging_julia.zip) | A notebook that compares libKriging with another package installs that package itself, in its first cells (pip, CRAN, `Pkg.add`, or a clone of STK); a package @@ -106,7 +122,7 @@ tools/install/python.sh master ../libKriging # or: tools/install/python.sh tools/install/r.sh master ~/Rlib ../libKriging # then R_LIBS_USER=~/Rlib tools/install/octave.sh ../libKriging ~/mlib # then OCTAVE_PATH=~/mlib tools/install/julia.sh ../libKriging -python tools/run_notebooks.py --inplace python_Kriging_update-simulate.ipynb +python tools/run_notebooks.py --inplace update-simulate_Kriging_python.ipynb python tools/report.py --expect python ``` @@ -125,7 +141,7 @@ python tools/py2nb.py # regenerate every notebook from tools/src/ python tools/py2nb.py --check # CI: sources and notebooks in sync, no notebook without source ``` -A new notebook is a new source in `tools/src/` named `__`, +A new notebook is a new source in `tools/src/` named `__`, with its suite named the same, and a line in the tables above (CI checks that every notebook is linked from this README); CI picks it up automatically. diff --git a/R_all-classes_api-tour.ipynb b/R_all-classes_api-tour.ipynb deleted file mode 100644 index 507404f..0000000 --- a/R_all-classes_api-tour.ipynb +++ /dev/null @@ -1,320 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "c000", - "metadata": {}, - "source": [ - "# R · all classes — every libKriging model class and fit method through rlibkriging\n", - "\n", - "One problem (Branin on [0,1]², Q² on 500 test points), every class and every fit\n", - "method of the R binding, each with the checks that hold for it whatever the\n", - "data:\n", - "\n", - "| model | checks (error-level unless stated) |\n", - "|---|---|\n", - "| `Kriging` (noise-free) | interpolation, Q² floor, conditional simulations through the data, `update` = fit on all points (same θ), `save`/`load` |\n", - "| `Kriging(noise = \"nugget\")` | nugget > 0, smoothing next to the design points, single-start optimum as good as the multistart one (warn), `save`/`load` |\n", - "| `Kriging(noise = )` | the stored variances, finite predictions |\n", - "| `Kriging(method = \"vecchia(m)\")`, `\"nystrom(k)\"` | `k$method()`, `predict(method = ...)` finite, Q² floor (warn) |\n", - "| `WarpKriging` (continuous and categorical warpings) | finite predictions, Q² floor (warn), `save`/`load` |\n", - "| `MLPKriging` | finite predictions, Q² floor (warn), default optimizer as good as a multistart BFGS (warn), `save`/`load` |\n", - "| `NestedKriging` (`NK`, `PoE`) | `NK` interpolates, stdev ≥ 0, `save`/`load` |\n", - "\n", - "A class missing from the installed rlibkriging (release channel) is recorded as\n", - "SKIP." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c001", - "metadata": {}, - "outputs": [], - "source": [ - "suppressPackageStartupMessages(library(rlibkriging))\n", - "source(\"tools/r/checks.R\")\n", - "s <- new_suite(\"R_all-classes_api-tour\")\n", - "cat(\"rlibkriging\", as.character(packageVersion(\"rlibkriging\")), \"\\n\")\n", - "\n", - "branin <- function(U) { # on [0,1]^2\n", - " x1 <- 15 * U[, 1] - 5; x2 <- 15 * U[, 2]\n", - " (x2 - 5.1 / (4 * pi^2) * x1^2 + 5 / pi * x1 - 6)^2 + 10 * (1 - 1 / (8 * pi)) * cos(x1) + 10\n", - "}\n", - "set.seed(1)\n", - "lhs <- function(n, d) sapply(1:d, function(j) (sample(n) - runif(n)) / n)\n", - "X <- lhs(30, 2); y <- branin(X)\n", - "Xt <- matrix(runif(1000), ncol = 2); yt <- branin(Xt)\n", - "tmp <- tempdir()\n", - "has_class <- function(name) exists(name, mode = \"function\")\n", - "skip <- function(s, check, reason) {\n", - " s$checks[[length(s$checks) + 1]] <- data.frame(suite = s$name, lang = \"R\", check = check, severity = \"info\",\n", - " status = \"SKIP\", value = \"\", expected = \"\", detail = reason, stringsAsFactors = FALSE)\n", - " message(\"[SKIP] \", check, \": \", reason)\n", - "}\n", - "roundtrip_same_mean <- function(s, label, k, Xn) {\n", - " f <- file.path(tmp, paste0(gsub(\"[^A-Za-z0-9]\", \"_\", label), \".json\"))\n", - " check_runs(s, paste0(label, \": save + load\"), {\n", - " save(k, f)\n", - " k2 <- rlibkriging::load(f)\n", - " check_close(s, paste0(label, \": load(save(k)) predicts the same mean\"),\n", - " predict(k2, Xn, return_stdev = FALSE)$mean, predict(k, Xn, return_stdev = FALSE)$mean, rtol = 1e-10)\n", - " })\n", - "}" - ] - }, - { - "cell_type": "markdown", - "id": "c002", - "metadata": {}, - "source": [ - "## `Kriging`, noise-free" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c003", - "metadata": {}, - "outputs": [], - "source": [ - "k <- Kriging(y, X, \"matern5_2\")\n", - "print(k)\n", - "p <- predict(k, Xt, return_stdev = TRUE)\n", - "add_metric(s, \"branin\", \"Kriging\", q2 = q2(yt, p$mean), rmse = rmse(yt, p$mean))\n", - "check_greater(s, \"Kriging: Q2\", q2(yt, p$mean), 0.95)\n", - "check_less(s, \"Kriging: interpolates (max |m(X) - y| / sd(y))\",\n", - " max(abs(predict(k, X, return_stdev = FALSE)$mean - y)) / sd(y), 1e-6)\n", - "sims <- simulate(k, nsim = 5, seed = 2, x = X)\n", - "check_less(s, \"Kriging: conditional simulations go through the data (max |sim(X) - y| / sigma)\",\n", - " max(abs(sims - y)) / sqrt(k$sigma2()), 1e-4)\n", - "check_greater(s, \"Kriging: stdev > 0 off the design\", min(p$stdev), 0)\n", - "# update with 5 new points (no refit) = a fit on the 35 points at the same theta\n", - "Xu <- lhs(5, 2); yu <- branin(Xu)\n", - "ku <- copy(k)\n", - "update(ku, yu, Xu, refit = FALSE)\n", - "k35 <- Kriging(c(y, yu), rbind(X, Xu), \"matern5_2\", optim = \"none\",\n", - " parameters = list(theta = matrix(k$theta(), nrow = 1), is_theta_estim = FALSE,\n", - " sigma2 = k$sigma2(), is_sigma2_estim = FALSE))\n", - "check_close(s, \"Kriging: update(refit = FALSE) == fit on all points (same theta, sigma2)\",\n", - " predict(ku, Xt, return_stdev = FALSE)$mean, predict(k35, Xt, return_stdev = FALSE)$mean,\n", - " rtol = 1e-6, atol = 1e-8 * sd(y))\n", - "roundtrip_same_mean(s, \"Kriging\", k, Xt)" - ] - }, - { - "cell_type": "markdown", - "id": "c004", - "metadata": {}, - "source": [ - "## `Kriging` with noise: estimated nugget, known variances\n", - "\n", - "Noise of standard deviation 5% of sd(y). The nugget model is fitted with 10\n", - "restarts (`optim = \"BFGS10\"`); the default single start is compared with it\n", - "(warn-level: a lower likelihood means a local optimum). As DiceKriging's\n", - "nugget, the predictor returns y exactly *at* the design points (the nugget is\n", - "a micro-scale variation): smoothing is checked next to them." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c005", - "metadata": {}, - "outputs": [], - "source": [ - "sd_noise <- 0.05 * sd(y)\n", - "yn <- y + rnorm(length(y), sd = sd_noise)\n", - "kn <- Kriging(yn, X, \"matern5_2\", noise = \"nugget\", optim = \"BFGS10\")\n", - "pn <- predict(kn, Xt, return_stdev = TRUE)\n", - "add_metric(s, \"branin noisy\", \"Kriging nugget BFGS10\", q2 = q2(yt, pn$mean), nugget = kn$nugget(),\n", - " ll = logLikelihood(kn))\n", - "check_greater(s, \"Kriging(nugget): estimated nugget > 0\", kn$nugget(), 0)\n", - "check_greater(s, \"Kriging(nugget): Q2\", q2(yt, pn$mean), 0.95)\n", - "near <- pmin(X + 1e-7, 1)\n", - "check_greater(s, \"Kriging(nugget): smooths the noisy data next to the design points (rms residual / noise sd)\",\n", - " rmse(yn, predict(kn, near, return_stdev = FALSE)$mean) / sd_noise, 0.3)\n", - "kn1 <- Kriging(yn, X, \"matern5_2\", noise = \"nugget\")\n", - "add_metric(s, \"branin noisy\", \"Kriging nugget BFGS\", q2 = q2(yt, predict(kn1, Xt, return_stdev = FALSE)$mean),\n", - " nugget = kn1$nugget(), ll = logLikelihood(kn1))\n", - "check_greater(s, \"Kriging(nugget): LL(single start) - LL(10 restarts)\", logLikelihood(kn1) - logLikelihood(kn), -1,\n", - " severity = \"warn\")\n", - "roundtrip_same_mean(s, \"Kriging(nugget)\", kn, Xt)\n", - "kh <- Kriging(yn, X, \"matern5_2\", noise = rep(sd_noise^2, length(yn)))\n", - "check_close(s, \"Kriging(noise = variances): noise() returns them\", kh$noise(), rep(sd_noise^2, length(yn)),\n", - " rtol = 1e-12)\n", - "ph <- predict(kh, Xt, return_stdev = TRUE)\n", - "check_less(s, \"Kriging(noise = variances): non-finite predictions\", sum(!is.finite(c(ph$mean, ph$stdev))), 1)\n", - "add_metric(s, \"branin noisy\", \"Kriging known noise\", q2 = q2(yt, ph$mean))" - ] - }, - { - "cell_type": "markdown", - "id": "c006", - "metadata": {}, - "source": [ - "## `Kriging` with an approximate fit: Vecchia, Nyström (n = 400)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c007", - "metadata": {}, - "outputs": [], - "source": [ - "Xl <- lhs(400, 2); yl <- branin(Xl)\n", - "for (m in c(\"vecchia(20)\", \"nystrom(60)\")) {\n", - " short <- sub(\"\\\\(.*\", \"\", m)\n", - " check_runs(s, paste0(\"Kriging(method = '\", m, \"'): fit + predict\"), {\n", - " ka <- Kriging(yl, Xl, \"matern5_2\", method = m)\n", - " add_check(s, paste0(\"Kriging(method = '\", m, \"'): method()\"), identical(ka$method(), m), ka$method(), m)\n", - " pa <- predict(ka, Xt, return_stdev = TRUE, method = short)\n", - " check_less(s, paste0(\"Kriging(method = '\", m, \"'): non-finite predictions\"),\n", - " sum(!is.finite(c(pa$mean, pa$stdev))), 1)\n", - " add_metric(s, \"branin n=400\", paste(\"Kriging\", m), q2 = q2(yt, pa$mean))\n", - " check_greater(s, paste0(\"Kriging(method = '\", m, \"'): Q2\"), q2(yt, pa$mean), 0.95, severity = \"warn\")\n", - " })\n", - "}" - ] - }, - { - "cell_type": "markdown", - "id": "c008", - "metadata": {}, - "source": [ - "## `WarpKriging`: continuous warpings, then a categorical input\n", - "\n", - "The categorical case: x₂ restricted to 5 levels (coded 0, …, 4), embedded in ℝ²." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c009", - "metadata": {}, - "outputs": [], - "source": [ - "if (has_class(\"WarpKriging\")) {\n", - " check_runs(s, \"WarpKriging(kumaraswamy, kumaraswamy): fit + predict\", {\n", - " kw <- WarpKriging(y, X, warping = c(\"kumaraswamy\", \"kumaraswamy\"), kernel = \"matern5_2\")\n", - " pw <- predict(kw, Xt, return_stdev = TRUE)\n", - " check_less(s, \"WarpKriging(kumaraswamy): non-finite predictions\", sum(!is.finite(c(pw$mean, pw$stdev))), 1)\n", - " add_metric(s, \"branin\", \"WarpKriging kumaraswamy\", q2 = q2(yt, pw$mean))\n", - " check_greater(s, \"WarpKriging(kumaraswamy): Q2\", q2(yt, pw$mean), 0.9, severity = \"warn\")\n", - " roundtrip_same_mean(s, \"WarpKriging(kumaraswamy)\", kw, Xt)\n", - " })\n", - " lev <- function(U) cbind(U[, 1], round(U[, 2] * 4)) # level codes 0..4\n", - " unlev <- function(L) cbind(L[, 1], L[, 2] / 4)\n", - " Xc <- lev(lhs(40, 2)); yc <- branin(unlev(Xc))\n", - " Xtc <- lev(Xt); ytc <- branin(unlev(Xtc))\n", - " check_runs(s, \"WarpKriging(none, categorical(5,2)): fit + predict\", {\n", - " kc <- WarpKriging(yc, Xc, warping = c(\"none\", \"categorical(5,2)\"), kernel = \"matern5_2\")\n", - " pc <- predict(kc, Xtc, return_stdev = TRUE)\n", - " check_less(s, \"WarpKriging(categorical): non-finite predictions\", sum(!is.finite(c(pc$mean, pc$stdev))), 1)\n", - " add_metric(s, \"branin, x2 categorical\", \"WarpKriging categorical\", q2 = q2(ytc, pc$mean))\n", - " check_greater(s, \"WarpKriging(categorical): Q2\", q2(ytc, pc$mean), 0.8, severity = \"warn\")\n", - " roundtrip_same_mean(s, \"WarpKriging(categorical)\", kc, Xtc)\n", - " })\n", - "} else skip(s, \"WarpKriging\", \"not in this rlibkriging\")" - ] - }, - { - "cell_type": "markdown", - "id": "c010", - "metadata": {}, - "source": [ - "## `MLPKriging`: a joint feature map ℝ² → ℝ² before the kernel" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c011", - "metadata": {}, - "outputs": [], - "source": [ - "if (has_class(\"MLPKriging\")) {\n", - " check_runs(s, \"MLPKriging: fit + predict\", {\n", - " km <- MLPKriging(y, X, hidden_dims = c(8), d_out = 2, activation = \"tanh\", kernel = \"matern5_2\", optim = \"BFGS10\")\n", - " pm <- predict(km, Xt, return_stdev = TRUE)\n", - " check_less(s, \"MLPKriging: non-finite predictions\", sum(!is.finite(c(pm$mean, pm$stdev))), 1)\n", - " add_metric(s, \"branin\", \"MLPKriging tanh [8] -> 2, BFGS10\", q2 = q2(yt, pm$mean))\n", - " check_greater(s, \"MLPKriging: Q2\", q2(yt, pm$mean), 0.9, severity = \"warn\")\n", - " roundtrip_same_mean(s, \"MLPKriging\", km, Xt)\n", - " # the default optimizer (\"BFGS+Adam\")\n", - " kd <- MLPKriging(y, X, hidden_dims = c(8), d_out = 2, activation = \"tanh\", kernel = \"matern5_2\")\n", - " qd <- q2(yt, predict(kd, Xt, return_stdev = FALSE)$mean)\n", - " add_metric(s, \"branin\", \"MLPKriging tanh [8] -> 2, default optim\", q2 = qd)\n", - " check_greater(s, \"MLPKriging: Q2(default optim) - Q2(BFGS10)\", qd - q2(yt, pm$mean), -0.05, severity = \"warn\")\n", - " })\n", - "} else skip(s, \"MLPKriging\", \"not in this rlibkriging\")" - ] - }, - { - "cell_type": "markdown", - "id": "c012", - "metadata": {}, - "source": [ - "## `NestedKriging`: 4 groups of 100 points, `NK` and `PoE` aggregations" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c013", - "metadata": {}, - "outputs": [], - "source": [ - "if (has_class(\"NestedKriging\")) {\n", - " for (agg in c(\"NK\", \"PoE\")) {\n", - " check_runs(s, paste0(\"NestedKriging(\", agg, \"): fit + predict\"), {\n", - " kk <- NestedKriging(yl, Xl, kernel = \"matern5_2\", nb_groups = 4, aggregation = agg, seed = 3)\n", - " pk <- predict(kk, Xt, return_stdev = TRUE)\n", - " check_less(s, paste0(\"NestedKriging(\", agg, \"): stdev < 0\"), sum(pk$stdev < 0), 1)\n", - " add_metric(s, \"branin n=400\", paste(\"NestedKriging\", agg), q2 = q2(yt, pk$mean))\n", - " check_greater(s, paste0(\"NestedKriging(\", agg, \"): Q2\"), q2(yt, pk$mean), 0.95, severity = \"warn\")\n", - " if (agg == \"NK\")\n", - " check_less(s, \"NestedKriging(NK): interpolates (max |m(X) - y| / sd(y))\",\n", - " max(abs(predict(kk, Xl, return_stdev = FALSE)$mean - yl)) / sd(yl), 1e-3)\n", - " roundtrip_same_mean(s, paste0(\"NestedKriging(\", agg, \")\"), kk, Xt)\n", - " })\n", - " }\n", - "} else skip(s, \"NestedKriging\", \"not in this rlibkriging\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c014", - "metadata": {}, - "outputs": [], - "source": [ - "me <- do.call(rbind, s$metrics)\n", - "print(me[me$metric == \"q2\", c(\"case\", \"package\", \"value\")], row.names = FALSE)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c015", - "metadata": {}, - "outputs": [], - "source": [ - "finalize(s)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "R", - "language": "R", - "name": "ir" - }, - "language_info": { - "name": "R" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/python_NestedKriging_aggregations-large-n.ipynb b/aggregations-large-n_NestedKriging_python.ipynb similarity index 86% rename from python_NestedKriging_aggregations-large-n.ipynb rename to aggregations-large-n_NestedKriging_python.ipynb index 9d70d4b..bed6171 100644 --- a/python_NestedKriging_aggregations-large-n.ipynb +++ b/aggregations-large-n_NestedKriging_python.ipynb @@ -13,7 +13,7 @@ "kriging predictor), or the products of experts `PoE`, `gPoE`, `BCM`, `rBCM`\n", "(cheaper, variances not fully consistent).\n", "\n", - "Problems: as in `python_Kriging-vecchia-nystrom_large-n.ipynb`, Hartmann 3\n", + "Problems: as in `large-n_Kriging-vecchia-nystrom_python.ipynb`, Hartmann 3\n", "and the OTL circuit with n = 1500 (exact fit as reference) and n = 5000; then\n", "every aggregation on Branin.\n", "\n", @@ -41,7 +41,7 @@ "import lkbench\n", "from lkbench import TESTS, design, metrics, compat\n", "\n", - "suite = lkbench.Suite(\"python_NestedKriging_aggregations-large-n\")\n", + "suite = lkbench.Suite(\"aggregations-large-n_NestedKriging_python\")\n", "print(\"pylibkriging\", compat.version())\n", "if not compat.has(\"NestedKriging\"):\n", " suite.skip(\"NestedKriging\", \"not available in this pylibkriging\")\n", @@ -135,10 +135,35 @@ " severity=\"warn\")" ] }, + { + "cell_type": "markdown", + "id": "c006", + "metadata": {}, + "source": [ + "## save / load, pickle, copy\n", + "\n", + "On Hartmann 3 (n = 40): a model saved and loaded back (class-specific and\n", + "generic `pylibkriging.load`), pickled, or deep-copied predicts exactly as the\n", + "original." + ] + }, { "cell_type": "code", "execution_count": null, - "id": "c006", + "id": "c007", + "metadata": {}, + "outputs": [], + "source": [ + "if compat.has(\"NestedKriging\"):\n", + " X, y, Xt, yt = design.dataset(TESTS[\"hartmann3\"], 40, n_test=100, seed=61)\n", + " with suite.runs(\"NestedKriging: fit (persistence)\"):\n", + " lkbench.persistence.check(suite, \"NestedKriging\", lk.NestedKriging(y, X, \"matern5_2\", nb_groups=4, seed=1), Xt, pred)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c008", "metadata": {}, "outputs": [], "source": [ diff --git a/aggregations_NestedKriging_R.ipynb b/aggregations_NestedKriging_R.ipynb new file mode 100644 index 0000000..ff6fd14 --- /dev/null +++ b/aggregations_NestedKriging_R.ipynb @@ -0,0 +1,125 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "c000", + "metadata": {}, + "source": [ + "# R · `NestedKriging` — `NK` and `PoE` aggregations, 4 groups of 100 points\n", + "\n", + "Branin on [0,1]² (n = 30 LHS points unless stated, Q² on 500 uniform test\n", + "points), through rlibkriging. Checks (error-level unless stated):\n", + "\n", + "* stdev ≥ 0, Q² > 0.95 (warn-level)\n", + "* `NK` interpolates the data (it is a proper kriging predictor)\n", + "* `save` then `load`: the same predicted mean\n", + "\n", + "A class missing from the installed rlibkriging (release channel) is recorded as SKIP." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c001", + "metadata": {}, + "outputs": [], + "source": [ + "suppressPackageStartupMessages(library(rlibkriging))\n", + "source(\"tools/r/checks.R\")\n", + "s <- new_suite(\"aggregations_NestedKriging_R\")\n", + "cat(\"rlibkriging\", as.character(packageVersion(\"rlibkriging\")), \"\\n\")\n", + "\n", + "branin <- function(U) { # on [0,1]^2\n", + " x1 <- 15 * U[, 1] - 5; x2 <- 15 * U[, 2]\n", + " (x2 - 5.1 / (4 * pi^2) * x1^2 + 5 / pi * x1 - 6)^2 + 10 * (1 - 1 / (8 * pi)) * cos(x1) + 10\n", + "}\n", + "set.seed(1)\n", + "lhs <- function(n, d) sapply(1:d, function(j) (sample(n) - runif(n)) / n)\n", + "X <- lhs(30, 2); y <- branin(X)\n", + "Xt <- matrix(runif(1000), ncol = 2); yt <- branin(Xt)\n", + "tmp <- tempdir()\n", + "has_class <- function(name) exists(name, mode = \"function\")\n", + "skip <- function(s, check, reason) {\n", + " s$checks[[length(s$checks) + 1]] <- data.frame(suite = s$name, lang = \"R\", check = check, severity = \"info\",\n", + " status = \"SKIP\", value = \"\", expected = \"\", detail = reason, stringsAsFactors = FALSE)\n", + " message(\"[SKIP] \", check, \": \", reason)\n", + "}\n", + "roundtrip_same_mean <- function(s, label, k, Xn) {\n", + " f <- file.path(tmp, paste0(gsub(\"[^A-Za-z0-9]\", \"_\", label), \".json\"))\n", + " check_runs(s, paste0(label, \": save + load\"), {\n", + " save(k, f)\n", + " k2 <- rlibkriging::load(f)\n", + " check_close(s, paste0(label, \": load(save(k)) predicts the same mean\"),\n", + " predict(k2, Xn, return_stdev = FALSE)$mean, predict(k, Xn, return_stdev = FALSE)$mean, rtol = 1e-10)\n", + " })\n", + "}" + ] + }, + { + "cell_type": "markdown", + "id": "c002", + "metadata": {}, + "source": [ + "## `NestedKriging`: 4 groups of 100 points, `NK` and `PoE` aggregations" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c003", + "metadata": {}, + "outputs": [], + "source": [ + "Xl <- lhs(400, 2); yl <- branin(Xl)\n", + "if (has_class(\"NestedKriging\")) {\n", + " for (agg in c(\"NK\", \"PoE\")) {\n", + " check_runs(s, paste0(\"NestedKriging(\", agg, \"): fit + predict\"), {\n", + " kk <- NestedKriging(yl, Xl, kernel = \"matern5_2\", nb_groups = 4, aggregation = agg, seed = 3)\n", + " pk <- predict(kk, Xt, return_stdev = TRUE)\n", + " check_less(s, paste0(\"NestedKriging(\", agg, \"): stdev < 0\"), sum(pk$stdev < 0), 1)\n", + " add_metric(s, \"branin n=400\", paste(\"NestedKriging\", agg), q2 = q2(yt, pk$mean))\n", + " check_greater(s, paste0(\"NestedKriging(\", agg, \"): Q2\"), q2(yt, pk$mean), 0.95, severity = \"warn\")\n", + " if (agg == \"NK\")\n", + " check_less(s, \"NestedKriging(NK): interpolates (max |m(X) - y| / sd(y))\",\n", + " max(abs(predict(kk, Xl, return_stdev = FALSE)$mean - yl)) / sd(yl), 1e-3)\n", + " roundtrip_same_mean(s, paste0(\"NestedKriging(\", agg, \")\"), kk, Xt)\n", + " })\n", + " }\n", + "} else skip(s, \"NestedKriging\", \"not in this rlibkriging\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c004", + "metadata": {}, + "outputs": [], + "source": [ + "me <- do.call(rbind, s$metrics)\n", + "print(me[me$metric == \"q2\", c(\"case\", \"package\", \"value\")], row.names = FALSE)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c005", + "metadata": {}, + "outputs": [], + "source": [ + "finalize(s)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "R", + "language": "R", + "name": "ir" + }, + "language_info": { + "name": "R" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/aggregations_NestedKriging_julia.ipynb b/aggregations_NestedKriging_julia.ipynb new file mode 100644 index 0000000..f49993a --- /dev/null +++ b/aggregations_NestedKriging_julia.ipynb @@ -0,0 +1,128 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "c000", + "metadata": {}, + "source": [ + "# Julia · `NestedKriging` — `NK` and `PoE` aggregations, 4 groups of 100 points\n", + "\n", + "Branin on [0,1]² (n = 30 LHS points unless stated, Q² on 500 uniform test\n", + "points), through JLibKriging. Checks (error-level unless stated):\n", + "\n", + "* stdev ≥ 0, Q² > 0.95 (warn-level)\n", + "* `NK` interpolates the data (it is a proper kriging predictor)\n", + "* `save` then `load`: the same predicted mean\n", + "\n", + "The same checks as `aggregations_NestedKriging_R.ipynb`, with the Julia binding." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c001", + "metadata": {}, + "outputs": [], + "source": [ + "import JLibKriging as LK\n", + "using Random\n", + "include(\"tools/julia/checks.jl\")\n", + "s = Suite(\"aggregations_NestedKriging_julia\")\n", + "\n", + "branin(U) = begin\n", + " x1 = 15 .* U[:, 1] .- 5; x2 = 15 .* U[:, 2]\n", + " (x2 .- 5.1 / (4π^2) .* x1 .^ 2 .+ 5 / π .* x1 .- 6) .^ 2 .+ 10 * (1 - 1 / (8π)) .* cos.(x1) .+ 10\n", + "end\n", + "rng = MersenneTwister(1)\n", + "lhs(n, d) = hcat([(randperm(rng, n) .- rand(rng, n)) ./ n for _ in 1:d]...)\n", + "X = lhs(30, 2); y = branin(X)\n", + "Xt = rand(rng, 500, 2); yt = branin(Xt)\n", + "sd(v) = sqrt(sum((v .- sum(v) / length(v)) .^ 2) / (length(v) - 1))\n", + "finite_count(v...) = count(!isfinite, vcat(v...))\n", + "\n", + "function check_runs!(f, s, check)\n", + " try\n", + " f()\n", + " catch e\n", + " add_check!(s, check, false; value=string(typeof(e)), expected=\"no error\", detail=first(sprint(showerror, e), 200))\n", + " end\n", + "end\n", + "\n", + "function roundtrip!(s, label, k, Xn)\n", + " check_runs!(s, \"$label: save + load\") do\n", + " f = joinpath(mktempdir(), \"k.json\")\n", + " LK.save(k, f)\n", + " k2 = LK.load(f)\n", + " check_close!(s, \"$label: load(save(k)) predicts the same mean\",\n", + " LK.predict(k2, Xn; return_stdev=false).mean, LK.predict(k, Xn; return_stdev=false).mean; rtol=1e-10)\n", + " end\n", + "end" + ] + }, + { + "cell_type": "markdown", + "id": "c002", + "metadata": {}, + "source": [ + "## `NestedKriging`: 4 groups of 100 points, `NK` and `PoE` aggregations" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c003", + "metadata": {}, + "outputs": [], + "source": [ + "Xl = lhs(400, 2); yl = branin(Xl)\n", + "for agg in (\"NK\", \"PoE\")\n", + " label = \"NestedKriging($agg)\"\n", + " check_runs!(s, \"$label: fit + predict\") do\n", + " kk = LK.NestedKriging(yl, Xl, \"matern5_2\", 4; aggregation=agg, seed=3)\n", + " pk = LK.predict(kk, Xt; return_stdev=true)\n", + " add_check!(s, \"$label: stdev >= 0\", all(pk.stdev .>= 0); value=minimum(pk.stdev), expected=\">= 0\")\n", + " add_metric!(s, \"branin n=400\", \"NestedKriging $agg\"; q2=q2(yt, pk.mean))\n", + " add_check!(s, \"$label: Q2\", q2(yt, pk.mean) > 0.95; value=q2(yt, pk.mean), expected=\"> 0.95\", severity=\"warn\")\n", + " if agg == \"NK\"\n", + " e = maximum(abs.(LK.predict(kk, Xl; return_stdev=false).mean .- yl)) / sd(yl)\n", + " add_check!(s, \"NestedKriging(NK): interpolates (max |m(X) - y| / sd(y))\", e < 1e-3; value=e, expected=\"< 1e-3\")\n", + " end\n", + " roundtrip!(s, label, kk, Xt)\n", + " end\n", + "end" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c004", + "metadata": {}, + "outputs": [], + "source": [ + "foreach(r -> r[5] == \"q2\" && println(rpad(r[3], 24), rpad(r[4], 28), r[6]), s.metrics)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c005", + "metadata": {}, + "outputs": [], + "source": [ + "finalize!(s)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Julia 1.11", + "language": "julia", + "name": "julia-1.11" + }, + "language_info": { + "name": "julia" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/aggregations_NestedKriging_octave.ipynb b/aggregations_NestedKriging_octave.ipynb new file mode 100644 index 0000000..53c7c3a --- /dev/null +++ b/aggregations_NestedKriging_octave.ipynb @@ -0,0 +1,138 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "c000", + "metadata": {}, + "source": [ + "# Octave · `NestedKriging` — `NK` and `PoE` aggregations, 4 groups of 100 points\n", + "\n", + "Branin on [0,1]² (n = 30 LHS points unless stated, Q² on 500 uniform test\n", + "points), through mLibKriging. Checks (error-level unless stated):\n", + "\n", + "* stdev ≥ 0, Q² > 0.95 (warn-level)\n", + "* `NK` interpolates the data (it is a proper kriging predictor)\n", + "* `save` then `load`: the same predicted mean\n", + "\n", + "The same checks as `aggregations_NestedKriging_R.ipynb`, with the Octave binding.\n", + "\n", + "A class missing from the installed mLibKriging (release channel) is recorded as SKIP." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c001", + "metadata": {}, + "outputs": [], + "source": [ + "addpath(fullfile(pwd, \"tools\", \"octave\"));\n", + "s = lkb_suite(\"aggregations_NestedKriging_octave\");\n", + "q2 = @(y, m) 1 - sum((y - m).^2) / sum((y - mean(y)).^2);\n", + "branin = @(U) ((15 * U(:, 2)) - 5.1 / (4 * pi^2) * (15 * U(:, 1) - 5).^2 + 5 / pi * (15 * U(:, 1) - 5) - 6).^2 ...\n", + " + 10 * (1 - 1 / (8 * pi)) * cos(15 * U(:, 1) - 5) + 10;\n", + "rand(\"seed\", 1); randn(\"seed\", 1);\n", + "function U = lhs(n, d)\n", + " U = zeros(n, d);\n", + " for j = 1:d, U(:, j) = (randperm(n)' - rand(n, 1)) / n; end\n", + "end\n", + "X = lhs(30, 2); y = branin(X);\n", + "Xt = rand(500, 2); yt = branin(Xt);\n", + "tmp = tempname(); mkdir(tmp);\n", + "has_class = @(name) exist(name) == 2; % a class file on the path\n", + "\n", + "function s = roundtrip(s, label, k, Xn, loader)\n", + " f = fullfile(tempdir(), [regexprep(label, \"[^A-Za-z0-9]\", \"_\") \".json\"]);\n", + " try\n", + " k.save(f);\n", + " k2 = loader(f);\n", + " m1 = k.predict(Xn, false, false, false);\n", + " m2 = k2.predict(Xn, false, false, false);\n", + " s = lkb_close(s, [label \": load(save(k)) predicts the same mean\"], m2, m1, 1e-10);\n", + " catch err\n", + " s = lkb_check(s, [label \": save + load\"], false, \"error\", \"no error\", err.message);\n", + " end\n", + "end\n", + "\n", + "function s = skip(s, check, reason)\n", + " s.checks(end + 1, :) = {s.name, \"octave\", check, \"info\", \"SKIP\", \"\", \"\", reason};\n", + " printf(\"[SKIP] %s: %s\\n\", check, reason);\n", + "end" + ] + }, + { + "cell_type": "markdown", + "id": "c002", + "metadata": {}, + "source": [ + "## `NestedKriging`: 4 groups of 100 points, `NK` and `PoE` aggregations" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c003", + "metadata": {}, + "outputs": [], + "source": [ + "Xl = lhs(400, 2); yl = branin(Xl);\n", + "if has_class(\"NestedKriging\")\n", + " for agg = {\"NK\", \"PoE\"}\n", + " agg = agg{1}; label = [\"NestedKriging(\" agg \")\"];\n", + " try\n", + " kk = NestedKriging(yl, Xl, \"matern5_2\", 4, agg, \"kmeans\", 3);\n", + " [mk, sk] = kk.predict(Xt); % stdev returned when asked for (2 outputs)\n", + " s = lkb_check(s, [label \": stdev >= 0\"], all(sk >= 0), min(sk), \">= 0\");\n", + " s = lkb_metric(s, \"branin n=400\", [\"NestedKriging \" agg], \"q2\", q2(yt, mk));\n", + " s = lkb_check(s, [label \": Q2\"], q2(yt, mk) > 0.95, q2(yt, mk), \"> 0.95\", \"\", \"warn\");\n", + " if strcmp(agg, \"NK\")\n", + " e = max(abs(kk.predict(Xl) - yl)) / std(yl);\n", + " s = lkb_check(s, \"NestedKriging(NK): interpolates (max |m(X) - y| / sd(y))\", e < 1e-3, e, \"< 1e-3\");\n", + " end\n", + " f = fullfile(tmp, [\"nested_\" agg \".json\"]);\n", + " kk.save(f); k2 = load_kriging(f);\n", + " s = lkb_close(s, [label \": load(save(k)) predicts the same mean\"], k2.predict(Xt), kk.predict(Xt), 1e-10);\n", + " catch err\n", + " s = lkb_check(s, [label \": fit + predict\"], false, \"error\", \"no error\", err.message);\n", + " end\n", + " end\n", + "else\n", + " s = skip(s, \"NestedKriging\", \"not in this mLibKriging\");\n", + "end" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c004", + "metadata": {}, + "outputs": [], + "source": [ + "q = strcmp(s.metrics(:, 5), \"q2\");\n", + "disp(s.metrics(q, [3, 4, 6]))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c005", + "metadata": {}, + "outputs": [], + "source": [ + "lkb_finalize(s);" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Octave", + "language": "octave", + "name": "octave" + }, + "language_info": { + "name": "octave" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/python_Kriging_cross-binding-reference.ipynb b/cross-binding-reference_Kriging_python.ipynb similarity index 98% rename from python_Kriging_cross-binding-reference.ipynb rename to cross-binding-reference_Kriging_python.ipynb index 3f8ae7b..203884e 100644 --- a/python_Kriging_cross-binding-reference.ipynb +++ b/cross-binding-reference_Kriging_python.ipynb @@ -42,7 +42,7 @@ "from lkbench import compat\n", "from lkbench.checks import DATA_DIR as DATA, RESULTS_DIR\n", "\n", - "suite = lkbench.Suite(\"python_Kriging_cross-binding-reference\")\n", + "suite = lkbench.Suite(\"cross-binding-reference_Kriging_python\")\n", "print(\"pylibkriging\", compat.version())\n", "DATASETS = [\"branin\", \"hartmann3\", \"borehole\"]\n", "rows = []\n", diff --git a/R_Kriging_cross-binding.ipynb b/cross-binding_Kriging_R.ipynb similarity index 96% rename from R_Kriging_cross-binding.ipynb rename to cross-binding_Kriging_R.ipynb index 93c1073..3fde9ef 100644 --- a/R_Kriging_cross-binding.ipynb +++ b/cross-binding_Kriging_R.ipynb @@ -7,7 +7,7 @@ "source": [ "# R · Kriging — cross-binding consistency: the R side\n", "\n", - "Same protocol as `python_Kriging_cross-binding-reference.ipynb` (see there), on the\n", + "Same protocol as `cross-binding-reference_Kriging_python.ipynb` (see there), on the\n", "shared datasets of `data/`; writes `results/xbind_R.csv`, compared with the\n", "Python values by `tools/report.py`." ] @@ -21,7 +21,7 @@ "source": [ "suppressPackageStartupMessages(library(rlibkriging))\n", "source(\"tools/r/checks.R\")\n", - "s <- new_suite(\"R_Kriging_cross-binding\")\n", + "s <- new_suite(\"cross-binding_Kriging_R\")\n", "cat(\"rlibkriging\", as.character(packageVersion(\"rlibkriging\")), \"\\n\")\n", "rows <- list()\n", "put <- function(dataset, quantity, values) {\n", diff --git a/julia_Kriging_cross-binding.ipynb b/cross-binding_Kriging_julia.ipynb similarity index 96% rename from julia_Kriging_cross-binding.ipynb rename to cross-binding_Kriging_julia.ipynb index 68997e7..17e83c0 100644 --- a/julia_Kriging_cross-binding.ipynb +++ b/cross-binding_Kriging_julia.ipynb @@ -7,7 +7,7 @@ "source": [ "# Julia · Kriging — cross-binding consistency: the Julia side\n", "\n", - "Same protocol as `python_Kriging_cross-binding-reference.ipynb` (see there), on\n", + "Same protocol as `cross-binding-reference_Kriging_python.ipynb` (see there), on\n", "the shared datasets of `data/`; writes `results/xbind_julia.csv`, compared with\n", "the Python values by `tools/report.py`.\n", "\n", @@ -25,7 +25,7 @@ "source": [ "import JLibKriging as LK\n", "include(\"tools/julia/checks.jl\")\n", - "s = Suite(\"julia_Kriging_cross-binding\")\n", + "s = Suite(\"cross-binding_Kriging_julia\")\n", "rows = Vector{Vector{String}}()\n", "record!(dataset, quantity, values) =\n", " for (i, v) in enumerate(vec(Float64.(collect(values))))\n", diff --git a/octave_Kriging_cross-binding.ipynb b/cross-binding_Kriging_octave.ipynb similarity index 96% rename from octave_Kriging_cross-binding.ipynb rename to cross-binding_Kriging_octave.ipynb index 6369816..64fc325 100644 --- a/octave_Kriging_cross-binding.ipynb +++ b/cross-binding_Kriging_octave.ipynb @@ -7,7 +7,7 @@ "source": [ "# Octave · Kriging — cross-binding consistency: the Octave side\n", "\n", - "Same protocol as `python_Kriging_cross-binding-reference.ipynb` (see there), on the\n", + "Same protocol as `cross-binding-reference_Kriging_python.ipynb` (see there), on the\n", "shared datasets of `data/`; writes `results/xbind_octave.csv`, compared with\n", "the Python values by `tools/report.py`. mLibKriging must be on the path\n", "(`OCTAVE_PATH` or `addpath`)." @@ -21,7 +21,7 @@ "outputs": [], "source": [ "addpath(fullfile(pwd, \"tools\", \"octave\"));\n", - "s = lkb_suite(\"octave_Kriging_cross-binding\");\n", + "s = lkb_suite(\"cross-binding_Kriging_octave\");\n", "out = {};\n", "function out = put(out, dataset, quantity, values)\n", " v = double(values(:));\n", diff --git a/python_MLPKriging_deep-kernel-projected-branin.ipynb b/deep-kernel-projected-branin_MLPKriging_python.ipynb similarity index 79% rename from python_MLPKriging_deep-kernel-projected-branin.ipynb rename to deep-kernel-projected-branin_MLPKriging_python.ipynb index 6705b30..2039666 100644 --- a/python_MLPKriging_deep-kernel-projected-branin.ipynb +++ b/deep-kernel-projected-branin_MLPKriging_python.ipynb @@ -33,7 +33,7 @@ "import lkbench\n", "from lkbench import TESTS, design, metrics, compat\n", "\n", - "suite = lkbench.Suite(\"python_MLPKriging_deep-kernel-projected-branin\")\n", + "suite = lkbench.Suite(\"deep-kernel-projected-branin_MLPKriging_python\")\n", "print(\"pylibkriging\", compat.version())\n", "\n", "\n", @@ -87,10 +87,36 @@ " suite.skip(\"MLPKriging\", \"not available in this pylibkriging\")" ] }, + { + "cell_type": "markdown", + "id": "c003", + "metadata": {}, + "source": [ + "## save / load, pickle, copy\n", + "\n", + "On Hartmann 3 (n = 40): a model saved and loaded back (class-specific and\n", + "generic `pylibkriging.load`), pickled, or deep-copied predicts exactly as the\n", + "original." + ] + }, { "cell_type": "code", "execution_count": null, - "id": "c003", + "id": "c004", + "metadata": {}, + "outputs": [], + "source": [ + "if compat.has(\"MLPKriging\"):\n", + " X, y, Xt, yt = design.dataset(TESTS[\"hartmann3\"], 40, n_test=100, seed=61)\n", + " with suite.runs(\"MLPKriging: fit (persistence)\"):\n", + " lkbench.persistence.check(suite, \"MLPKriging\", lk.MLPKriging(y, X, hidden_dims=[8], d_out=2, activation=\"tanh\",\n", + " kernel=\"matern5_2\"), Xt, pred)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c005", "metadata": {}, "outputs": [], "source": [ diff --git a/deep-kernel_MLPKriging_R.ipynb b/deep-kernel_MLPKriging_R.ipynb new file mode 100644 index 0000000..8aa9d10 --- /dev/null +++ b/deep-kernel_MLPKriging_R.ipynb @@ -0,0 +1,124 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "c000", + "metadata": {}, + "source": [ + "# R · `MLPKriging` — a joint feature map ℝ² → ℝ² before the kernel\n", + "\n", + "Branin on [0,1]² (n = 30 LHS points unless stated, Q² on 500 uniform test\n", + "points), through rlibkriging. Checks (error-level unless stated):\n", + "\n", + "* 10 restarts (`BFGS10`): finite predictions, Q² > 0.9 (warn-level)\n", + "* `save` then `load`: the same predicted mean\n", + "* the default optimizer as good as `BFGS10` (warn-level; see libKriging/libKriging#382)\n", + "\n", + "A class missing from the installed rlibkriging (release channel) is recorded as SKIP." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c001", + "metadata": {}, + "outputs": [], + "source": [ + "suppressPackageStartupMessages(library(rlibkriging))\n", + "source(\"tools/r/checks.R\")\n", + "s <- new_suite(\"deep-kernel_MLPKriging_R\")\n", + "cat(\"rlibkriging\", as.character(packageVersion(\"rlibkriging\")), \"\\n\")\n", + "\n", + "branin <- function(U) { # on [0,1]^2\n", + " x1 <- 15 * U[, 1] - 5; x2 <- 15 * U[, 2]\n", + " (x2 - 5.1 / (4 * pi^2) * x1^2 + 5 / pi * x1 - 6)^2 + 10 * (1 - 1 / (8 * pi)) * cos(x1) + 10\n", + "}\n", + "set.seed(1)\n", + "lhs <- function(n, d) sapply(1:d, function(j) (sample(n) - runif(n)) / n)\n", + "X <- lhs(30, 2); y <- branin(X)\n", + "Xt <- matrix(runif(1000), ncol = 2); yt <- branin(Xt)\n", + "tmp <- tempdir()\n", + "has_class <- function(name) exists(name, mode = \"function\")\n", + "skip <- function(s, check, reason) {\n", + " s$checks[[length(s$checks) + 1]] <- data.frame(suite = s$name, lang = \"R\", check = check, severity = \"info\",\n", + " status = \"SKIP\", value = \"\", expected = \"\", detail = reason, stringsAsFactors = FALSE)\n", + " message(\"[SKIP] \", check, \": \", reason)\n", + "}\n", + "roundtrip_same_mean <- function(s, label, k, Xn) {\n", + " f <- file.path(tmp, paste0(gsub(\"[^A-Za-z0-9]\", \"_\", label), \".json\"))\n", + " check_runs(s, paste0(label, \": save + load\"), {\n", + " save(k, f)\n", + " k2 <- rlibkriging::load(f)\n", + " check_close(s, paste0(label, \": load(save(k)) predicts the same mean\"),\n", + " predict(k2, Xn, return_stdev = FALSE)$mean, predict(k, Xn, return_stdev = FALSE)$mean, rtol = 1e-10)\n", + " })\n", + "}" + ] + }, + { + "cell_type": "markdown", + "id": "c002", + "metadata": {}, + "source": [ + "## `MLPKriging`: a joint feature map ℝ² → ℝ² before the kernel" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c003", + "metadata": {}, + "outputs": [], + "source": [ + "if (has_class(\"MLPKriging\")) {\n", + " check_runs(s, \"MLPKriging: fit + predict\", {\n", + " km <- MLPKriging(y, X, hidden_dims = c(8), d_out = 2, activation = \"tanh\", kernel = \"matern5_2\", optim = \"BFGS10\")\n", + " pm <- predict(km, Xt, return_stdev = TRUE)\n", + " check_less(s, \"MLPKriging: non-finite predictions\", sum(!is.finite(c(pm$mean, pm$stdev))), 1)\n", + " add_metric(s, \"branin\", \"MLPKriging tanh [8] -> 2, BFGS10\", q2 = q2(yt, pm$mean))\n", + " check_greater(s, \"MLPKriging: Q2\", q2(yt, pm$mean), 0.9, severity = \"warn\")\n", + " roundtrip_same_mean(s, \"MLPKriging\", km, Xt)\n", + " # the default optimizer (\"BFGS+Adam\")\n", + " kd <- MLPKriging(y, X, hidden_dims = c(8), d_out = 2, activation = \"tanh\", kernel = \"matern5_2\")\n", + " qd <- q2(yt, predict(kd, Xt, return_stdev = FALSE)$mean)\n", + " add_metric(s, \"branin\", \"MLPKriging tanh [8] -> 2, default optim\", q2 = qd)\n", + " check_greater(s, \"MLPKriging: Q2(default optim) - Q2(BFGS10)\", qd - q2(yt, pm$mean), -0.05, severity = \"warn\")\n", + " })\n", + "} else skip(s, \"MLPKriging\", \"not in this rlibkriging\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c004", + "metadata": {}, + "outputs": [], + "source": [ + "me <- do.call(rbind, s$metrics)\n", + "print(me[me$metric == \"q2\", c(\"case\", \"package\", \"value\")], row.names = FALSE)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c005", + "metadata": {}, + "outputs": [], + "source": [ + "finalize(s)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "R", + "language": "R", + "name": "ir" + }, + "language_info": { + "name": "R" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/deep-kernel_MLPKriging_julia.ipynb b/deep-kernel_MLPKriging_julia.ipynb new file mode 100644 index 0000000..cd8cbef --- /dev/null +++ b/deep-kernel_MLPKriging_julia.ipynb @@ -0,0 +1,126 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "c000", + "metadata": {}, + "source": [ + "# Julia · `MLPKriging` — a joint feature map ℝ² → ℝ² before the kernel\n", + "\n", + "Branin on [0,1]² (n = 30 LHS points unless stated, Q² on 500 uniform test\n", + "points), through JLibKriging. Checks (error-level unless stated):\n", + "\n", + "* 10 restarts (`BFGS10`): finite predictions, Q² > 0.9 (warn-level)\n", + "* `save` then `load`: the same predicted mean\n", + "* the default optimizer as good as `BFGS10` (warn-level; see libKriging/libKriging#382)\n", + "\n", + "The same checks as `deep-kernel_MLPKriging_R.ipynb`, with the Julia binding." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c001", + "metadata": {}, + "outputs": [], + "source": [ + "import JLibKriging as LK\n", + "using Random\n", + "include(\"tools/julia/checks.jl\")\n", + "s = Suite(\"deep-kernel_MLPKriging_julia\")\n", + "\n", + "branin(U) = begin\n", + " x1 = 15 .* U[:, 1] .- 5; x2 = 15 .* U[:, 2]\n", + " (x2 .- 5.1 / (4π^2) .* x1 .^ 2 .+ 5 / π .* x1 .- 6) .^ 2 .+ 10 * (1 - 1 / (8π)) .* cos.(x1) .+ 10\n", + "end\n", + "rng = MersenneTwister(1)\n", + "lhs(n, d) = hcat([(randperm(rng, n) .- rand(rng, n)) ./ n for _ in 1:d]...)\n", + "X = lhs(30, 2); y = branin(X)\n", + "Xt = rand(rng, 500, 2); yt = branin(Xt)\n", + "sd(v) = sqrt(sum((v .- sum(v) / length(v)) .^ 2) / (length(v) - 1))\n", + "finite_count(v...) = count(!isfinite, vcat(v...))\n", + "\n", + "function check_runs!(f, s, check)\n", + " try\n", + " f()\n", + " catch e\n", + " add_check!(s, check, false; value=string(typeof(e)), expected=\"no error\", detail=first(sprint(showerror, e), 200))\n", + " end\n", + "end\n", + "\n", + "function roundtrip!(s, label, k, Xn)\n", + " check_runs!(s, \"$label: save + load\") do\n", + " f = joinpath(mktempdir(), \"k.json\")\n", + " LK.save(k, f)\n", + " k2 = LK.load(f)\n", + " check_close!(s, \"$label: load(save(k)) predicts the same mean\",\n", + " LK.predict(k2, Xn; return_stdev=false).mean, LK.predict(k, Xn; return_stdev=false).mean; rtol=1e-10)\n", + " end\n", + "end" + ] + }, + { + "cell_type": "markdown", + "id": "c002", + "metadata": {}, + "source": [ + "## `MLPKriging`: a joint feature map ℝ² → ℝ² before the kernel" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c003", + "metadata": {}, + "outputs": [], + "source": [ + "check_runs!(s, \"MLPKriging: fit + predict\") do\n", + " km = LK.MLPKriging(y, X, [8], 2; activation=\"tanh\", kernel=\"matern5_2\", optim=\"BFGS10\")\n", + " pm = LK.predict(km, Xt; return_stdev=true)\n", + " add_check!(s, \"MLPKriging: finite predictions\", finite_count(pm.mean, pm.stdev) == 0;\n", + " value=finite_count(pm.mean, pm.stdev), expected=\"0 non-finite\")\n", + " add_metric!(s, \"branin\", \"MLPKriging tanh [8] -> 2, BFGS10\"; q2=q2(yt, pm.mean))\n", + " add_check!(s, \"MLPKriging: Q2\", q2(yt, pm.mean) > 0.9; value=q2(yt, pm.mean), expected=\"> 0.9\", severity=\"warn\")\n", + " roundtrip!(s, \"MLPKriging\", km, Xt)\n", + " kd = LK.MLPKriging(y, X, [8], 2; activation=\"tanh\", kernel=\"matern5_2\") # default optimizer\n", + " qd = q2(yt, LK.predict(kd, Xt; return_stdev=false).mean)\n", + " add_metric!(s, \"branin\", \"MLPKriging tanh [8] -> 2, default optim\"; q2=qd)\n", + " add_check!(s, \"MLPKriging: Q2(default optim) - Q2(BFGS10)\", qd - q2(yt, pm.mean) > -0.05;\n", + " value=qd - q2(yt, pm.mean), expected=\"> -0.05\", severity=\"warn\")\n", + "end" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c004", + "metadata": {}, + "outputs": [], + "source": [ + "foreach(r -> r[5] == \"q2\" && println(rpad(r[3], 24), rpad(r[4], 28), r[6]), s.metrics)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c005", + "metadata": {}, + "outputs": [], + "source": [ + "finalize!(s)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Julia 1.11", + "language": "julia", + "name": "julia-1.11" + }, + "language_info": { + "name": "julia" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/deep-kernel_MLPKriging_octave.ipynb b/deep-kernel_MLPKriging_octave.ipynb new file mode 100644 index 0000000..fcf944b --- /dev/null +++ b/deep-kernel_MLPKriging_octave.ipynb @@ -0,0 +1,132 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "c000", + "metadata": {}, + "source": [ + "# Octave · `MLPKriging` — a joint feature map ℝ² → ℝ² before the kernel\n", + "\n", + "Branin on [0,1]² (n = 30 LHS points unless stated, Q² on 500 uniform test\n", + "points), through mLibKriging. Checks (error-level unless stated):\n", + "\n", + "* 10 restarts (`BFGS10`): finite predictions, Q² > 0.9 (warn-level)\n", + "* `save` then `load`: the same predicted mean\n", + "* the default optimizer as good as `BFGS10` (warn-level; see libKriging/libKriging#382)\n", + "\n", + "The same checks as `deep-kernel_MLPKriging_R.ipynb`, with the Octave binding.\n", + "\n", + "A class missing from the installed mLibKriging (release channel) is recorded as SKIP." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c001", + "metadata": {}, + "outputs": [], + "source": [ + "addpath(fullfile(pwd, \"tools\", \"octave\"));\n", + "s = lkb_suite(\"deep-kernel_MLPKriging_octave\");\n", + "q2 = @(y, m) 1 - sum((y - m).^2) / sum((y - mean(y)).^2);\n", + "branin = @(U) ((15 * U(:, 2)) - 5.1 / (4 * pi^2) * (15 * U(:, 1) - 5).^2 + 5 / pi * (15 * U(:, 1) - 5) - 6).^2 ...\n", + " + 10 * (1 - 1 / (8 * pi)) * cos(15 * U(:, 1) - 5) + 10;\n", + "rand(\"seed\", 1); randn(\"seed\", 1);\n", + "function U = lhs(n, d)\n", + " U = zeros(n, d);\n", + " for j = 1:d, U(:, j) = (randperm(n)' - rand(n, 1)) / n; end\n", + "end\n", + "X = lhs(30, 2); y = branin(X);\n", + "Xt = rand(500, 2); yt = branin(Xt);\n", + "tmp = tempname(); mkdir(tmp);\n", + "has_class = @(name) exist(name) == 2; % a class file on the path\n", + "\n", + "function s = roundtrip(s, label, k, Xn, loader)\n", + " f = fullfile(tempdir(), [regexprep(label, \"[^A-Za-z0-9]\", \"_\") \".json\"]);\n", + " try\n", + " k.save(f);\n", + " k2 = loader(f);\n", + " m1 = k.predict(Xn, false, false, false);\n", + " m2 = k2.predict(Xn, false, false, false);\n", + " s = lkb_close(s, [label \": load(save(k)) predicts the same mean\"], m2, m1, 1e-10);\n", + " catch err\n", + " s = lkb_check(s, [label \": save + load\"], false, \"error\", \"no error\", err.message);\n", + " end\n", + "end\n", + "\n", + "function s = skip(s, check, reason)\n", + " s.checks(end + 1, :) = {s.name, \"octave\", check, \"info\", \"SKIP\", \"\", \"\", reason};\n", + " printf(\"[SKIP] %s: %s\\n\", check, reason);\n", + "end" + ] + }, + { + "cell_type": "markdown", + "id": "c002", + "metadata": {}, + "source": [ + "## `MLPKriging`: a joint feature map ℝ² → ℝ² before the kernel" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c003", + "metadata": {}, + "outputs": [], + "source": [ + "if has_class(\"MLPKriging\")\n", + " try\n", + " km = MLPKriging(y, X, [8], 2, \"tanh\", \"matern5_2\", \"constant\", false, \"BFGS10\");\n", + " [mm, sm] = km.predict(Xt, true, false, false);\n", + " s = lkb_check(s, \"MLPKriging: finite predictions\", all(isfinite([mm; sm])), sum(~isfinite([mm; sm])), \"0 non-finite\");\n", + " s = lkb_metric(s, \"branin\", \"MLPKriging tanh [8] -> 2, BFGS10\", \"q2\", q2(yt, mm));\n", + " s = lkb_check(s, \"MLPKriging: Q2\", q2(yt, mm) > 0.9, q2(yt, mm), \"> 0.9\", \"\", \"warn\");\n", + " s = roundtrip(s, \"MLPKriging\", km, Xt, @load_kriging);\n", + " kd = MLPKriging(y, X, [8], 2, \"tanh\", \"matern5_2\"); % default optimizer\n", + " qd = q2(yt, kd.predict(Xt, false, false, false));\n", + " s = lkb_metric(s, \"branin\", \"MLPKriging tanh [8] -> 2, default optim\", \"q2\", qd);\n", + " s = lkb_check(s, \"MLPKriging: Q2(default optim) - Q2(BFGS10)\", qd - q2(yt, mm) > -0.05, qd - q2(yt, mm), \"> -0.05\", \"\", \"warn\");\n", + " catch err\n", + " s = lkb_check(s, \"MLPKriging: fit + predict\", false, \"error\", \"no error\", err.message);\n", + " end\n", + "else\n", + " s = skip(s, \"MLPKriging\", \"not in this mLibKriging\");\n", + "end" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c004", + "metadata": {}, + "outputs": [], + "source": [ + "q = strcmp(s.metrics(:, 5), \"q2\");\n", + "disp(s.metrics(q, [3, 4, 6]))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c005", + "metadata": {}, + "outputs": [], + "source": [ + "lkb_finalize(s);" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Octave", + "language": "octave", + "name": "octave" + }, + "language_info": { + "name": "octave" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/python_Kriging_high-dimension-normalize-screening.ipynb b/high-dimension-normalize-screening_Kriging_python.ipynb similarity index 97% rename from python_Kriging_high-dimension-normalize-screening.ipynb rename to high-dimension-normalize-screening_Kriging_python.ipynb index 85c77ee..74da769 100644 --- a/python_Kriging_high-dimension-normalize-screening.ipynb +++ b/high-dimension-normalize-screening_Kriging_python.ipynb @@ -26,7 +26,7 @@ " active ones (screening ability, warn-level).\n", "\n", "`MLPKriging` on a high-dimensional function with a low-dimensional structure:\n", - "`python_MLPKriging_deep-kernel-projected-branin.ipynb`." + "`deep-kernel-projected-branin_MLPKriging_python.ipynb`." ] }, { @@ -45,7 +45,7 @@ "import lkbench\n", "from lkbench import TESTS, design, metrics, compat\n", "\n", - "suite = lkbench.Suite(\"python_Kriging_high-dimension-normalize-screening\")\n", + "suite = lkbench.Suite(\"high-dimension-normalize-screening_Kriging_python\")\n", "print(\"pylibkriging\", compat.version())\n", "\n", "\n", diff --git a/interpolation-simulate-update-save_Kriging_R.ipynb b/interpolation-simulate-update-save_Kriging_R.ipynb new file mode 100644 index 0000000..90eb449 --- /dev/null +++ b/interpolation-simulate-update-save_Kriging_R.ipynb @@ -0,0 +1,131 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "c000", + "metadata": {}, + "source": [ + "# R · `Kriging`, noise-free — interpolation, conditional simulation, update, save/load\n", + "\n", + "Branin on [0,1]² (n = 30 LHS points unless stated, Q² on 500 uniform test\n", + "points), through rlibkriging. Checks (error-level unless stated):\n", + "\n", + "* interpolation (max |m(X) − y| / sd(y) < 1e-6) and Q² > 0.95 on 500 test points\n", + "* conditional simulations through the data (relative to σ)\n", + "* stdev > 0 off the design\n", + "* `update` without refit = a fit on all the points at the same θ, σ²\n", + "* `save` then `load`: the same predicted mean" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c001", + "metadata": {}, + "outputs": [], + "source": [ + "suppressPackageStartupMessages(library(rlibkriging))\n", + "source(\"tools/r/checks.R\")\n", + "s <- new_suite(\"interpolation-simulate-update-save_Kriging_R\")\n", + "cat(\"rlibkriging\", as.character(packageVersion(\"rlibkriging\")), \"\\n\")\n", + "\n", + "branin <- function(U) { # on [0,1]^2\n", + " x1 <- 15 * U[, 1] - 5; x2 <- 15 * U[, 2]\n", + " (x2 - 5.1 / (4 * pi^2) * x1^2 + 5 / pi * x1 - 6)^2 + 10 * (1 - 1 / (8 * pi)) * cos(x1) + 10\n", + "}\n", + "set.seed(1)\n", + "lhs <- function(n, d) sapply(1:d, function(j) (sample(n) - runif(n)) / n)\n", + "X <- lhs(30, 2); y <- branin(X)\n", + "Xt <- matrix(runif(1000), ncol = 2); yt <- branin(Xt)\n", + "tmp <- tempdir()\n", + "has_class <- function(name) exists(name, mode = \"function\")\n", + "skip <- function(s, check, reason) {\n", + " s$checks[[length(s$checks) + 1]] <- data.frame(suite = s$name, lang = \"R\", check = check, severity = \"info\",\n", + " status = \"SKIP\", value = \"\", expected = \"\", detail = reason, stringsAsFactors = FALSE)\n", + " message(\"[SKIP] \", check, \": \", reason)\n", + "}\n", + "roundtrip_same_mean <- function(s, label, k, Xn) {\n", + " f <- file.path(tmp, paste0(gsub(\"[^A-Za-z0-9]\", \"_\", label), \".json\"))\n", + " check_runs(s, paste0(label, \": save + load\"), {\n", + " save(k, f)\n", + " k2 <- rlibkriging::load(f)\n", + " check_close(s, paste0(label, \": load(save(k)) predicts the same mean\"),\n", + " predict(k2, Xn, return_stdev = FALSE)$mean, predict(k, Xn, return_stdev = FALSE)$mean, rtol = 1e-10)\n", + " })\n", + "}" + ] + }, + { + "cell_type": "markdown", + "id": "c002", + "metadata": {}, + "source": [ + "## `Kriging`, noise-free" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c003", + "metadata": {}, + "outputs": [], + "source": [ + "k <- Kriging(y, X, \"matern5_2\")\n", + "print(k)\n", + "p <- predict(k, Xt, return_stdev = TRUE)\n", + "add_metric(s, \"branin\", \"Kriging\", q2 = q2(yt, p$mean), rmse = rmse(yt, p$mean))\n", + "check_greater(s, \"Kriging: Q2\", q2(yt, p$mean), 0.95)\n", + "check_less(s, \"Kriging: interpolates (max |m(X) - y| / sd(y))\",\n", + " max(abs(predict(k, X, return_stdev = FALSE)$mean - y)) / sd(y), 1e-6)\n", + "sims <- simulate(k, nsim = 5, seed = 2, x = X)\n", + "check_less(s, \"Kriging: conditional simulations go through the data (max |sim(X) - y| / sigma)\",\n", + " max(abs(sims - y)) / sqrt(k$sigma2()), 1e-4)\n", + "check_greater(s, \"Kriging: stdev > 0 off the design\", min(p$stdev), 0)\n", + "# update with 5 new points (no refit) = a fit on the 35 points at the same theta\n", + "Xu <- lhs(5, 2); yu <- branin(Xu)\n", + "ku <- copy(k)\n", + "update(ku, yu, Xu, refit = FALSE)\n", + "k35 <- Kriging(c(y, yu), rbind(X, Xu), \"matern5_2\", optim = \"none\",\n", + " parameters = list(theta = matrix(k$theta(), nrow = 1), is_theta_estim = FALSE,\n", + " sigma2 = k$sigma2(), is_sigma2_estim = FALSE))\n", + "check_close(s, \"Kriging: update(refit = FALSE) == fit on all points (same theta, sigma2)\",\n", + " predict(ku, Xt, return_stdev = FALSE)$mean, predict(k35, Xt, return_stdev = FALSE)$mean,\n", + " rtol = 1e-6, atol = 1e-8 * sd(y))\n", + "roundtrip_same_mean(s, \"Kriging\", k, Xt)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c004", + "metadata": {}, + "outputs": [], + "source": [ + "me <- do.call(rbind, s$metrics)\n", + "print(me[me$metric == \"q2\", c(\"case\", \"package\", \"value\")], row.names = FALSE)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c005", + "metadata": {}, + "outputs": [], + "source": [ + "finalize(s)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "R", + "language": "R", + "name": "ir" + }, + "language_info": { + "name": "R" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/interpolation-simulate-update-save_Kriging_julia.ipynb b/interpolation-simulate-update-save_Kriging_julia.ipynb new file mode 100644 index 0000000..3968d6f --- /dev/null +++ b/interpolation-simulate-update-save_Kriging_julia.ipynb @@ -0,0 +1,134 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "c000", + "metadata": {}, + "source": [ + "# Julia · `Kriging`, noise-free — interpolation, conditional simulation, update, save/load\n", + "\n", + "Branin on [0,1]² (n = 30 LHS points unless stated, Q² on 500 uniform test\n", + "points), through JLibKriging. Checks (error-level unless stated):\n", + "\n", + "* interpolation (max |m(X) − y| / sd(y) < 1e-6) and Q² > 0.95 on 500 test points\n", + "* conditional simulations through the data (relative to σ)\n", + "* stdev > 0 off the design\n", + "* `update` without refit = a fit on all the points at the same θ, σ²\n", + "* `save` then `load`: the same predicted mean\n", + "\n", + "The same checks as `interpolation-simulate-update-save_Kriging_R.ipynb`, with the Julia binding." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c001", + "metadata": {}, + "outputs": [], + "source": [ + "import JLibKriging as LK\n", + "using Random\n", + "include(\"tools/julia/checks.jl\")\n", + "s = Suite(\"interpolation-simulate-update-save_Kriging_julia\")\n", + "\n", + "branin(U) = begin\n", + " x1 = 15 .* U[:, 1] .- 5; x2 = 15 .* U[:, 2]\n", + " (x2 .- 5.1 / (4π^2) .* x1 .^ 2 .+ 5 / π .* x1 .- 6) .^ 2 .+ 10 * (1 - 1 / (8π)) .* cos.(x1) .+ 10\n", + "end\n", + "rng = MersenneTwister(1)\n", + "lhs(n, d) = hcat([(randperm(rng, n) .- rand(rng, n)) ./ n for _ in 1:d]...)\n", + "X = lhs(30, 2); y = branin(X)\n", + "Xt = rand(rng, 500, 2); yt = branin(Xt)\n", + "sd(v) = sqrt(sum((v .- sum(v) / length(v)) .^ 2) / (length(v) - 1))\n", + "finite_count(v...) = count(!isfinite, vcat(v...))\n", + "\n", + "function check_runs!(f, s, check)\n", + " try\n", + " f()\n", + " catch e\n", + " add_check!(s, check, false; value=string(typeof(e)), expected=\"no error\", detail=first(sprint(showerror, e), 200))\n", + " end\n", + "end\n", + "\n", + "function roundtrip!(s, label, k, Xn)\n", + " check_runs!(s, \"$label: save + load\") do\n", + " f = joinpath(mktempdir(), \"k.json\")\n", + " LK.save(k, f)\n", + " k2 = LK.load(f)\n", + " check_close!(s, \"$label: load(save(k)) predicts the same mean\",\n", + " LK.predict(k2, Xn; return_stdev=false).mean, LK.predict(k, Xn; return_stdev=false).mean; rtol=1e-10)\n", + " end\n", + "end" + ] + }, + { + "cell_type": "markdown", + "id": "c002", + "metadata": {}, + "source": [ + "## `Kriging`, noise-free" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c003", + "metadata": {}, + "outputs": [], + "source": [ + "k = LK.Kriging(y, X, \"matern5_2\")\n", + "p = LK.predict(k, Xt; return_stdev=true)\n", + "add_metric!(s, \"branin\", \"Kriging\"; q2=q2(yt, p.mean))\n", + "add_check!(s, \"Kriging: Q2\", q2(yt, p.mean) > 0.95; value=q2(yt, p.mean), expected=\"> 0.95\")\n", + "e = maximum(abs.(LK.predict(k, X; return_stdev=false).mean .- y)) / sd(y)\n", + "add_check!(s, \"Kriging: interpolates (max |m(X) - y| / sd(y))\", e < 1e-6; value=e, expected=\"< 1e-6\")\n", + "S = LK.simulate(k, 5, 2, X)\n", + "e = maximum(abs.(S .- y)) / sqrt(LK.sigma2(k))\n", + "add_check!(s, \"Kriging: conditional simulations go through the data (max |sim(X) - y| / sigma)\", e < 1e-4;\n", + " value=e, expected=\"< 1e-4\")\n", + "add_check!(s, \"Kriging: stdev > 0 off the design\", minimum(p.stdev) > 0; value=minimum(p.stdev), expected=\"> 0\")\n", + "Xu = lhs(5, 2); yu = branin(Xu)\n", + "ku = copy(k)\n", + "LK.update!(ku, yu, Xu; refit=false)\n", + "k35 = LK.Kriging(vcat(y, yu), vcat(X, Xu), \"matern5_2\"; optim=\"none\",\n", + " theta=vec(LK.theta(k)), is_theta_estim=false, sigma2=LK.sigma2(k), is_sigma2_estim=false)\n", + "check_close!(s, \"Kriging: update!(refit=false) == fit on all points (same theta, sigma2)\",\n", + " LK.predict(ku, Xt; return_stdev=false).mean, LK.predict(k35, Xt; return_stdev=false).mean;\n", + " rtol=1e-6, atol=1e-8 * sd(y))\n", + "roundtrip!(s, \"Kriging\", k, Xt)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c004", + "metadata": {}, + "outputs": [], + "source": [ + "foreach(r -> r[5] == \"q2\" && println(rpad(r[3], 24), rpad(r[4], 28), r[6]), s.metrics)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c005", + "metadata": {}, + "outputs": [], + "source": [ + "finalize!(s)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Julia 1.11", + "language": "julia", + "name": "julia-1.11" + }, + "language_info": { + "name": "julia" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/interpolation-simulate-update-save_Kriging_octave.ipynb b/interpolation-simulate-update-save_Kriging_octave.ipynb new file mode 100644 index 0000000..e82db02 --- /dev/null +++ b/interpolation-simulate-update-save_Kriging_octave.ipynb @@ -0,0 +1,134 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "c000", + "metadata": {}, + "source": [ + "# Octave · `Kriging`, noise-free — interpolation, conditional simulation, update, save/load\n", + "\n", + "Branin on [0,1]² (n = 30 LHS points unless stated, Q² on 500 uniform test\n", + "points), through mLibKriging. Checks (error-level unless stated):\n", + "\n", + "* interpolation (max |m(X) − y| / sd(y) < 1e-6) and Q² > 0.95 on 500 test points\n", + "* conditional simulations through the data (relative to σ)\n", + "* stdev > 0 off the design\n", + "* `update` without refit = a fit on all the points at the same θ, σ²\n", + "* `save` then `load`: the same predicted mean\n", + "\n", + "The same checks as `interpolation-simulate-update-save_Kriging_R.ipynb`, with the Octave binding." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c001", + "metadata": {}, + "outputs": [], + "source": [ + "addpath(fullfile(pwd, \"tools\", \"octave\"));\n", + "s = lkb_suite(\"interpolation-simulate-update-save_Kriging_octave\");\n", + "q2 = @(y, m) 1 - sum((y - m).^2) / sum((y - mean(y)).^2);\n", + "branin = @(U) ((15 * U(:, 2)) - 5.1 / (4 * pi^2) * (15 * U(:, 1) - 5).^2 + 5 / pi * (15 * U(:, 1) - 5) - 6).^2 ...\n", + " + 10 * (1 - 1 / (8 * pi)) * cos(15 * U(:, 1) - 5) + 10;\n", + "rand(\"seed\", 1); randn(\"seed\", 1);\n", + "function U = lhs(n, d)\n", + " U = zeros(n, d);\n", + " for j = 1:d, U(:, j) = (randperm(n)' - rand(n, 1)) / n; end\n", + "end\n", + "X = lhs(30, 2); y = branin(X);\n", + "Xt = rand(500, 2); yt = branin(Xt);\n", + "tmp = tempname(); mkdir(tmp);\n", + "has_class = @(name) exist(name) == 2; % a class file on the path\n", + "\n", + "function s = roundtrip(s, label, k, Xn, loader)\n", + " f = fullfile(tempdir(), [regexprep(label, \"[^A-Za-z0-9]\", \"_\") \".json\"]);\n", + " try\n", + " k.save(f);\n", + " k2 = loader(f);\n", + " m1 = k.predict(Xn, false, false, false);\n", + " m2 = k2.predict(Xn, false, false, false);\n", + " s = lkb_close(s, [label \": load(save(k)) predicts the same mean\"], m2, m1, 1e-10);\n", + " catch err\n", + " s = lkb_check(s, [label \": save + load\"], false, \"error\", \"no error\", err.message);\n", + " end\n", + "end\n", + "\n", + "function s = skip(s, check, reason)\n", + " s.checks(end + 1, :) = {s.name, \"octave\", check, \"info\", \"SKIP\", \"\", \"\", reason};\n", + " printf(\"[SKIP] %s: %s\\n\", check, reason);\n", + "end" + ] + }, + { + "cell_type": "markdown", + "id": "c002", + "metadata": {}, + "source": [ + "## `Kriging`, noise-free" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c003", + "metadata": {}, + "outputs": [], + "source": [ + "k = Kriging(y, X, \"matern5_2\");\n", + "[m, sd] = k.predict(Xt, true, false, false);\n", + "s = lkb_metric(s, \"branin\", \"Kriging\", \"q2\", q2(yt, m));\n", + "s = lkb_check(s, \"Kriging: Q2\", q2(yt, m) > 0.95, q2(yt, m), \"> 0.95\");\n", + "mX = k.predict(X, false, false, false);\n", + "e = max(abs(mX - y)) / std(y);\n", + "s = lkb_check(s, \"Kriging: interpolates (max |m(X) - y| / sd(y))\", e < 1e-6, e, \"< 1e-6\");\n", + "S = k.simulate(int32(5), int32(2), X, false);\n", + "e = max(max(abs(S - y))) / sqrt(k.sigma2());\n", + "s = lkb_check(s, \"Kriging: conditional simulations go through the data (max |sim(X) - y| / sigma)\", e < 1e-4, e, \"< 1e-4\");\n", + "s = lkb_check(s, \"Kriging: stdev > 0 off the design\", min(sd) > 0, min(sd), \"> 0\");\n", + "Xu = lhs(5, 2); yu = branin(Xu);\n", + "ku = k.copy();\n", + "ku.update(yu, Xu, false);\n", + "th = k.theta();\n", + "k35 = Kriging([y; yu], [X; Xu], \"matern5_2\", \"constant\", false, \"none\", \"LL\", ...\n", + " Params(\"theta\", th(:)', \"is_theta_estim\", false, \"sigma2\", k.sigma2(), \"is_sigma2_estim\", false));\n", + "s = lkb_close(s, \"Kriging: update(refit = false) == fit on all points (same theta, sigma2)\", ...\n", + " ku.predict(Xt, false, false, false), k35.predict(Xt, false, false, false), 1e-6, 1e-8 * std(y));\n", + "s = roundtrip(s, \"Kriging\", k, Xt, @load_kriging);" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c004", + "metadata": {}, + "outputs": [], + "source": [ + "q = strcmp(s.metrics(:, 5), \"q2\");\n", + "disp(s.metrics(q, [3, 4, 6]))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c005", + "metadata": {}, + "outputs": [], + "source": [ + "lkb_finalize(s);" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Octave", + "language": "octave", + "name": "octave" + }, + "language_info": { + "name": "octave" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/julia_all-classes_api-tour.ipynb b/julia_all-classes_api-tour.ipynb deleted file mode 100644 index 84b51c4..0000000 --- a/julia_all-classes_api-tour.ipynb +++ /dev/null @@ -1,313 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "c000", - "metadata": {}, - "source": [ - "# Julia · all classes — every libKriging model class and fit method through JLibKriging\n", - "\n", - "Same tour as `R_all-classes_api-tour.ipynb` (see there for the checks): Branin\n", - "on [0,1]², every class and fit method of the Julia binding, each with the\n", - "checks that hold for it whatever the data.\n", - "\n", - "The module is `JLibKriging` (the packaged binding, [JLibKriging.jl](https://github.com/libKriging/JLibKriging.jl)),\n", - "imported under a short name so that its exported accessors (`X`, `y`, `theta`, ...)\n", - "do not clash with the variables of the notebook." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c001", - "metadata": {}, - "outputs": [], - "source": [ - "import JLibKriging as LK\n", - "using Random\n", - "include(\"tools/julia/checks.jl\")\n", - "s = Suite(\"julia_all-classes_api-tour\")\n", - "\n", - "branin(U) = begin\n", - " x1 = 15 .* U[:, 1] .- 5; x2 = 15 .* U[:, 2]\n", - " (x2 .- 5.1 / (4π^2) .* x1 .^ 2 .+ 5 / π .* x1 .- 6) .^ 2 .+ 10 * (1 - 1 / (8π)) .* cos.(x1) .+ 10\n", - "end\n", - "rng = MersenneTwister(1)\n", - "lhs(n, d) = hcat([(randperm(rng, n) .- rand(rng, n)) ./ n for _ in 1:d]...)\n", - "X = lhs(30, 2); y = branin(X)\n", - "Xt = rand(rng, 500, 2); yt = branin(Xt)\n", - "sd(v) = sqrt(sum((v .- sum(v) / length(v)) .^ 2) / (length(v) - 1))\n", - "finite_count(v...) = count(!isfinite, vcat(v...))\n", - "\n", - "function check_runs!(f, s, check)\n", - " try\n", - " f()\n", - " catch e\n", - " add_check!(s, check, false; value=string(typeof(e)), expected=\"no error\", detail=first(sprint(showerror, e), 200))\n", - " end\n", - "end\n", - "\n", - "function roundtrip!(s, label, k, Xn)\n", - " check_runs!(s, \"$label: save + load\") do\n", - " f = joinpath(mktempdir(), \"k.json\")\n", - " LK.save(k, f)\n", - " k2 = LK.load(f)\n", - " check_close!(s, \"$label: load(save(k)) predicts the same mean\",\n", - " LK.predict(k2, Xn; return_stdev=false).mean, LK.predict(k, Xn; return_stdev=false).mean; rtol=1e-10)\n", - " end\n", - "end" - ] - }, - { - "cell_type": "markdown", - "id": "c002", - "metadata": {}, - "source": [ - "## `Kriging`, noise-free" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c003", - "metadata": {}, - "outputs": [], - "source": [ - "k = LK.Kriging(y, X, \"matern5_2\")\n", - "p = LK.predict(k, Xt; return_stdev=true)\n", - "add_metric!(s, \"branin\", \"Kriging\"; q2=q2(yt, p.mean))\n", - "add_check!(s, \"Kriging: Q2\", q2(yt, p.mean) > 0.95; value=q2(yt, p.mean), expected=\"> 0.95\")\n", - "e = maximum(abs.(LK.predict(k, X; return_stdev=false).mean .- y)) / sd(y)\n", - "add_check!(s, \"Kriging: interpolates (max |m(X) - y| / sd(y))\", e < 1e-6; value=e, expected=\"< 1e-6\")\n", - "S = LK.simulate(k, 5, 2, X)\n", - "e = maximum(abs.(S .- y)) / sqrt(LK.sigma2(k))\n", - "add_check!(s, \"Kriging: conditional simulations go through the data (max |sim(X) - y| / sigma)\", e < 1e-4;\n", - " value=e, expected=\"< 1e-4\")\n", - "add_check!(s, \"Kriging: stdev > 0 off the design\", minimum(p.stdev) > 0; value=minimum(p.stdev), expected=\"> 0\")\n", - "Xu = lhs(5, 2); yu = branin(Xu)\n", - "ku = copy(k)\n", - "LK.update!(ku, yu, Xu; refit=false)\n", - "k35 = LK.Kriging(vcat(y, yu), vcat(X, Xu), \"matern5_2\"; optim=\"none\",\n", - " theta=vec(LK.theta(k)), is_theta_estim=false, sigma2=LK.sigma2(k), is_sigma2_estim=false)\n", - "check_close!(s, \"Kriging: update!(refit=false) == fit on all points (same theta, sigma2)\",\n", - " LK.predict(ku, Xt; return_stdev=false).mean, LK.predict(k35, Xt; return_stdev=false).mean;\n", - " rtol=1e-6, atol=1e-8 * sd(y))\n", - "roundtrip!(s, \"Kriging\", k, Xt)" - ] - }, - { - "cell_type": "markdown", - "id": "c004", - "metadata": {}, - "source": [ - "## `Kriging` with noise: estimated nugget, known variances\n", - "\n", - "Noise of standard deviation 5% of sd(y). The nugget model is fitted with 10\n", - "restarts (`optim=\"BFGS10\"`); the default single start is compared with it\n", - "(warn-level: a lower likelihood means a local optimum). As DiceKriging's\n", - "nugget, the predictor returns y exactly *at* the design points (the nugget is\n", - "a micro-scale variation): smoothing is checked next to them." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c005", - "metadata": {}, - "outputs": [], - "source": [ - "sd_noise = 0.05 * sd(y)\n", - "yn = y .+ sd_noise .* randn(rng, length(y))\n", - "kn = LK.Kriging(yn, X, \"matern5_2\"; noise=\"nugget\", optim=\"BFGS10\")\n", - "mn = LK.predict(kn, Xt; return_stdev=false).mean\n", - "add_metric!(s, \"branin noisy\", \"Kriging nugget BFGS10\"; q2=q2(yt, mn))\n", - "add_check!(s, \"Kriging(nugget): estimated nugget > 0\", LK.nugget(kn) > 0; value=LK.nugget(kn), expected=\"> 0\")\n", - "add_check!(s, \"Kriging(nugget): Q2\", q2(yt, mn) > 0.95; value=q2(yt, mn), expected=\"> 0.95\")\n", - "r = sqrt(sum((LK.predict(kn, min.(X .+ 1e-7, 1.0); return_stdev=false).mean .- yn) .^ 2) / length(yn)) / sd_noise\n", - "add_check!(s, \"Kriging(nugget): smooths the noisy data next to the design points (rms residual / noise sd)\", r > 0.3;\n", - " value=r, expected=\"> 0.3\")\n", - "kn1 = LK.Kriging(yn, X, \"matern5_2\"; noise=\"nugget\")\n", - "add_metric!(s, \"branin noisy\", \"Kriging nugget BFGS\"; q2=q2(yt, LK.predict(kn1, Xt; return_stdev=false).mean))\n", - "dll = LK.log_likelihood(kn1) - LK.log_likelihood(kn)\n", - "add_check!(s, \"Kriging(nugget): LL(single start) - LL(10 restarts)\", dll > -1; value=dll, expected=\"> -1\", severity=\"warn\")\n", - "roundtrip!(s, \"Kriging(nugget)\", kn, Xt)\n", - "v = fill(sd_noise^2, length(yn))\n", - "kh = LK.Kriging(yn, X, \"matern5_2\"; noise=v)\n", - "check_close!(s, \"Kriging(noise = variances): noise() returns them\", LK.noise(kh), v; rtol=1e-12)\n", - "ph = LK.predict(kh, Xt; return_stdev=true)\n", - "add_check!(s, \"Kriging(noise = variances): finite predictions\", finite_count(ph.mean, ph.stdev) == 0;\n", - " value=finite_count(ph.mean, ph.stdev), expected=\"0 non-finite\")\n", - "add_metric!(s, \"branin noisy\", \"Kriging known noise\"; q2=q2(yt, ph.mean))" - ] - }, - { - "cell_type": "markdown", - "id": "c006", - "metadata": {}, - "source": [ - "## `Kriging` with an approximate fit: Vecchia, Nyström (n = 400)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c007", - "metadata": {}, - "outputs": [], - "source": [ - "Xl = lhs(400, 2); yl = branin(Xl)\n", - "for m in (\"vecchia(20)\", \"nystrom(60)\")\n", - " short = split(m, \"(\")[1]\n", - " label = \"Kriging(method = '$m')\"\n", - " check_runs!(s, \"$label: fit + predict\") do\n", - " ka = LK.Kriging(yl, Xl, \"matern5_2\"; method=m)\n", - " add_check!(s, \"$label: method()\", LK.method(ka) == m; value=LK.method(ka), expected=m)\n", - " pa = LK.predict(ka, Xt; return_stdev=true, method=String(short))\n", - " add_check!(s, \"$label: finite predictions\", finite_count(pa.mean, pa.stdev) == 0;\n", - " value=finite_count(pa.mean, pa.stdev), expected=\"0 non-finite\")\n", - " add_metric!(s, \"branin n=400\", \"Kriging $m\"; q2=q2(yt, pa.mean))\n", - " add_check!(s, \"$label: Q2\", q2(yt, pa.mean) > 0.95; value=q2(yt, pa.mean), expected=\"> 0.95\", severity=\"warn\")\n", - " end\n", - "end" - ] - }, - { - "cell_type": "markdown", - "id": "c008", - "metadata": {}, - "source": [ - "## `WarpKriging`: continuous warpings, then a categorical input\n", - "\n", - "The categorical case: x₂ restricted to 5 levels (coded 0, …, 4), embedded in ℝ²." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c009", - "metadata": {}, - "outputs": [], - "source": [ - "check_runs!(s, \"WarpKriging(kumaraswamy, kumaraswamy): fit + predict\") do\n", - " kw = LK.WarpKriging(y, X, [\"kumaraswamy\", \"kumaraswamy\"], \"matern5_2\")\n", - " pw = LK.predict(kw, Xt; return_stdev=true)\n", - " add_check!(s, \"WarpKriging(kumaraswamy): finite predictions\", finite_count(pw.mean, pw.stdev) == 0;\n", - " value=finite_count(pw.mean, pw.stdev), expected=\"0 non-finite\")\n", - " add_metric!(s, \"branin\", \"WarpKriging kumaraswamy\"; q2=q2(yt, pw.mean))\n", - " add_check!(s, \"WarpKriging(kumaraswamy): Q2\", q2(yt, pw.mean) > 0.9; value=q2(yt, pw.mean), expected=\"> 0.9\",\n", - " severity=\"warn\")\n", - " roundtrip!(s, \"WarpKriging(kumaraswamy)\", kw, Xt)\n", - "end\n", - "lev(U) = hcat(U[:, 1], round.(U[:, 2] .* 4))\n", - "unlev(L) = hcat(L[:, 1], L[:, 2] ./ 4)\n", - "Xc = lev(lhs(40, 2)); yc = branin(unlev(Xc))\n", - "Xtc = lev(Xt); ytc = branin(unlev(Xtc))\n", - "check_runs!(s, \"WarpKriging(none, categorical(5,2)): fit + predict\") do\n", - " kc = LK.WarpKriging(yc, Xc, [\"none\", \"categorical(5,2)\"], \"matern5_2\")\n", - " pc = LK.predict(kc, Xtc; return_stdev=true)\n", - " add_check!(s, \"WarpKriging(categorical): finite predictions\", finite_count(pc.mean, pc.stdev) == 0;\n", - " value=finite_count(pc.mean, pc.stdev), expected=\"0 non-finite\")\n", - " add_metric!(s, \"branin, x2 categorical\", \"WarpKriging categorical\"; q2=q2(ytc, pc.mean))\n", - " add_check!(s, \"WarpKriging(categorical): Q2\", q2(ytc, pc.mean) > 0.8; value=q2(ytc, pc.mean), expected=\"> 0.8\",\n", - " severity=\"warn\")\n", - " roundtrip!(s, \"WarpKriging(categorical)\", kc, Xtc)\n", - "end" - ] - }, - { - "cell_type": "markdown", - "id": "c010", - "metadata": {}, - "source": [ - "## `MLPKriging`: a joint feature map ℝ² → ℝ² before the kernel" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c011", - "metadata": {}, - "outputs": [], - "source": [ - "check_runs!(s, \"MLPKriging: fit + predict\") do\n", - " km = LK.MLPKriging(y, X, [8], 2; activation=\"tanh\", kernel=\"matern5_2\", optim=\"BFGS10\")\n", - " pm = LK.predict(km, Xt; return_stdev=true)\n", - " add_check!(s, \"MLPKriging: finite predictions\", finite_count(pm.mean, pm.stdev) == 0;\n", - " value=finite_count(pm.mean, pm.stdev), expected=\"0 non-finite\")\n", - " add_metric!(s, \"branin\", \"MLPKriging tanh [8] -> 2, BFGS10\"; q2=q2(yt, pm.mean))\n", - " add_check!(s, \"MLPKriging: Q2\", q2(yt, pm.mean) > 0.9; value=q2(yt, pm.mean), expected=\"> 0.9\", severity=\"warn\")\n", - " roundtrip!(s, \"MLPKriging\", km, Xt)\n", - " kd = LK.MLPKriging(y, X, [8], 2; activation=\"tanh\", kernel=\"matern5_2\") # default optimizer\n", - " qd = q2(yt, LK.predict(kd, Xt; return_stdev=false).mean)\n", - " add_metric!(s, \"branin\", \"MLPKriging tanh [8] -> 2, default optim\"; q2=qd)\n", - " add_check!(s, \"MLPKriging: Q2(default optim) - Q2(BFGS10)\", qd - q2(yt, pm.mean) > -0.05;\n", - " value=qd - q2(yt, pm.mean), expected=\"> -0.05\", severity=\"warn\")\n", - "end" - ] - }, - { - "cell_type": "markdown", - "id": "c012", - "metadata": {}, - "source": [ - "## `NestedKriging`: 4 groups of 100 points, `NK` and `PoE` aggregations" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c013", - "metadata": {}, - "outputs": [], - "source": [ - "for agg in (\"NK\", \"PoE\")\n", - " label = \"NestedKriging($agg)\"\n", - " check_runs!(s, \"$label: fit + predict\") do\n", - " kk = LK.NestedKriging(yl, Xl, \"matern5_2\", 4; aggregation=agg, seed=3)\n", - " pk = LK.predict(kk, Xt; return_stdev=true)\n", - " add_check!(s, \"$label: stdev >= 0\", all(pk.stdev .>= 0); value=minimum(pk.stdev), expected=\">= 0\")\n", - " add_metric!(s, \"branin n=400\", \"NestedKriging $agg\"; q2=q2(yt, pk.mean))\n", - " add_check!(s, \"$label: Q2\", q2(yt, pk.mean) > 0.95; value=q2(yt, pk.mean), expected=\"> 0.95\", severity=\"warn\")\n", - " if agg == \"NK\"\n", - " e = maximum(abs.(LK.predict(kk, Xl; return_stdev=false).mean .- yl)) / sd(yl)\n", - " add_check!(s, \"NestedKriging(NK): interpolates (max |m(X) - y| / sd(y))\", e < 1e-3; value=e, expected=\"< 1e-3\")\n", - " end\n", - " roundtrip!(s, label, kk, Xt)\n", - " end\n", - "end" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c014", - "metadata": {}, - "outputs": [], - "source": [ - "foreach(r -> r[5] == \"q2\" && println(rpad(r[3], 24), rpad(r[4], 28), r[6]), s.metrics)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c015", - "metadata": {}, - "outputs": [], - "source": [ - "finalize!(s)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Julia 1.11", - "language": "julia", - "name": "julia-1.11" - }, - "language_info": { - "name": "julia" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/python_Kriging_kernels-trends-interpolation.ipynb b/kernels-trends-interpolation_Kriging_python.ipynb similarity index 98% rename from python_Kriging_kernels-trends-interpolation.ipynb rename to kernels-trends-interpolation_Kriging_python.ipynb index 27f9a85..a04e7d4 100644 --- a/python_Kriging_kernels-trends-interpolation.ipynb +++ b/kernels-trends-interpolation_Kriging_python.ipynb @@ -32,7 +32,7 @@ " `matern5_2` / `constant` with a multistart optimizer (`BFGS10`): a\n", " regression shows up here first. The same floor for the default single-start\n", " `BFGS` is warn-level: a single start can end in a poor local optimum of the\n", - " likelihood (see `python_Kriging_objectives-LL-LOO-LMP-gradients.ipynb`),\n", + " likelihood (see `objectives-LL-LOO-LMP-gradients_Kriging_python.ipynb`),\n", " which is worth seeing in the report but is not a defect of the model;\n", "* **sanity of the fitted hyperparameters**: finite, positive θ and σ²;\n", "* **consistency**: `predict` on a batch equals `predict` point by point, the\n", @@ -57,7 +57,7 @@ "import lkbench\n", "from lkbench import TESTS, design, metrics, compat\n", "\n", - "suite = lkbench.Suite(\"python_Kriging_kernels-trends-interpolation\")\n", + "suite = lkbench.Suite(\"kernels-trends-interpolation_Kriging_python\")\n", "print(\"pylibkriging\", compat.version())\n", "\n", "KERNELS = [\"gauss\", \"exp\", \"matern3_2\", \"matern5_2\"]\n", diff --git a/large-n_Kriging-vecchia-nystrom_R.ipynb b/large-n_Kriging-vecchia-nystrom_R.ipynb new file mode 100644 index 0000000..4240136 --- /dev/null +++ b/large-n_Kriging-vecchia-nystrom_R.ipynb @@ -0,0 +1,120 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "c000", + "metadata": {}, + "source": [ + "# R · `Kriging` with an approximate fit — Vecchia, Nyström (n = 400)\n", + "\n", + "Branin on [0,1]² (n = 30 LHS points unless stated, Q² on 500 uniform test\n", + "points), through rlibkriging. Checks (error-level unless stated):\n", + "\n", + "* the fit method is reported by `method()`\n", + "* `predict` with the fit method: finite mean and stdev\n", + "* Q² > 0.95 (warn-level)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c001", + "metadata": {}, + "outputs": [], + "source": [ + "suppressPackageStartupMessages(library(rlibkriging))\n", + "source(\"tools/r/checks.R\")\n", + "s <- new_suite(\"large-n_Kriging-vecchia-nystrom_R\")\n", + "cat(\"rlibkriging\", as.character(packageVersion(\"rlibkriging\")), \"\\n\")\n", + "\n", + "branin <- function(U) { # on [0,1]^2\n", + " x1 <- 15 * U[, 1] - 5; x2 <- 15 * U[, 2]\n", + " (x2 - 5.1 / (4 * pi^2) * x1^2 + 5 / pi * x1 - 6)^2 + 10 * (1 - 1 / (8 * pi)) * cos(x1) + 10\n", + "}\n", + "set.seed(1)\n", + "lhs <- function(n, d) sapply(1:d, function(j) (sample(n) - runif(n)) / n)\n", + "X <- lhs(30, 2); y <- branin(X)\n", + "Xt <- matrix(runif(1000), ncol = 2); yt <- branin(Xt)\n", + "tmp <- tempdir()\n", + "has_class <- function(name) exists(name, mode = \"function\")\n", + "skip <- function(s, check, reason) {\n", + " s$checks[[length(s$checks) + 1]] <- data.frame(suite = s$name, lang = \"R\", check = check, severity = \"info\",\n", + " status = \"SKIP\", value = \"\", expected = \"\", detail = reason, stringsAsFactors = FALSE)\n", + " message(\"[SKIP] \", check, \": \", reason)\n", + "}\n", + "roundtrip_same_mean <- function(s, label, k, Xn) {\n", + " f <- file.path(tmp, paste0(gsub(\"[^A-Za-z0-9]\", \"_\", label), \".json\"))\n", + " check_runs(s, paste0(label, \": save + load\"), {\n", + " save(k, f)\n", + " k2 <- rlibkriging::load(f)\n", + " check_close(s, paste0(label, \": load(save(k)) predicts the same mean\"),\n", + " predict(k2, Xn, return_stdev = FALSE)$mean, predict(k, Xn, return_stdev = FALSE)$mean, rtol = 1e-10)\n", + " })\n", + "}" + ] + }, + { + "cell_type": "markdown", + "id": "c002", + "metadata": {}, + "source": [ + "## `Kriging` with an approximate fit: Vecchia, Nyström (n = 400)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c003", + "metadata": {}, + "outputs": [], + "source": [ + "Xl <- lhs(400, 2); yl <- branin(Xl)\n", + "for (m in c(\"vecchia(20)\", \"nystrom(60)\")) {\n", + " short <- sub(\"\\\\(.*\", \"\", m)\n", + " check_runs(s, paste0(\"Kriging(method = '\", m, \"'): fit + predict\"), {\n", + " ka <- Kriging(yl, Xl, \"matern5_2\", method = m)\n", + " add_check(s, paste0(\"Kriging(method = '\", m, \"'): method()\"), identical(ka$method(), m), ka$method(), m)\n", + " pa <- predict(ka, Xt, return_stdev = TRUE, method = short)\n", + " check_less(s, paste0(\"Kriging(method = '\", m, \"'): non-finite predictions\"),\n", + " sum(!is.finite(c(pa$mean, pa$stdev))), 1)\n", + " add_metric(s, \"branin n=400\", paste(\"Kriging\", m), q2 = q2(yt, pa$mean))\n", + " check_greater(s, paste0(\"Kriging(method = '\", m, \"'): Q2\"), q2(yt, pa$mean), 0.95, severity = \"warn\")\n", + " })\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c004", + "metadata": {}, + "outputs": [], + "source": [ + "me <- do.call(rbind, s$metrics)\n", + "print(me[me$metric == \"q2\", c(\"case\", \"package\", \"value\")], row.names = FALSE)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c005", + "metadata": {}, + "outputs": [], + "source": [ + "finalize(s)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "R", + "language": "R", + "name": "ir" + }, + "language_info": { + "name": "R" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/large-n_Kriging-vecchia-nystrom_julia.ipynb b/large-n_Kriging-vecchia-nystrom_julia.ipynb new file mode 100644 index 0000000..5240454 --- /dev/null +++ b/large-n_Kriging-vecchia-nystrom_julia.ipynb @@ -0,0 +1,126 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "c000", + "metadata": {}, + "source": [ + "# Julia · `Kriging` with an approximate fit — Vecchia, Nyström (n = 400)\n", + "\n", + "Branin on [0,1]² (n = 30 LHS points unless stated, Q² on 500 uniform test\n", + "points), through JLibKriging. Checks (error-level unless stated):\n", + "\n", + "* the fit method is reported by `method()`\n", + "* `predict` with the fit method: finite mean and stdev\n", + "* Q² > 0.95 (warn-level)\n", + "\n", + "The same checks as `large-n_Kriging-vecchia-nystrom_R.ipynb`, with the Julia binding." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c001", + "metadata": {}, + "outputs": [], + "source": [ + "import JLibKriging as LK\n", + "using Random\n", + "include(\"tools/julia/checks.jl\")\n", + "s = Suite(\"large-n_Kriging-vecchia-nystrom_julia\")\n", + "\n", + "branin(U) = begin\n", + " x1 = 15 .* U[:, 1] .- 5; x2 = 15 .* U[:, 2]\n", + " (x2 .- 5.1 / (4π^2) .* x1 .^ 2 .+ 5 / π .* x1 .- 6) .^ 2 .+ 10 * (1 - 1 / (8π)) .* cos.(x1) .+ 10\n", + "end\n", + "rng = MersenneTwister(1)\n", + "lhs(n, d) = hcat([(randperm(rng, n) .- rand(rng, n)) ./ n for _ in 1:d]...)\n", + "X = lhs(30, 2); y = branin(X)\n", + "Xt = rand(rng, 500, 2); yt = branin(Xt)\n", + "sd(v) = sqrt(sum((v .- sum(v) / length(v)) .^ 2) / (length(v) - 1))\n", + "finite_count(v...) = count(!isfinite, vcat(v...))\n", + "\n", + "function check_runs!(f, s, check)\n", + " try\n", + " f()\n", + " catch e\n", + " add_check!(s, check, false; value=string(typeof(e)), expected=\"no error\", detail=first(sprint(showerror, e), 200))\n", + " end\n", + "end\n", + "\n", + "function roundtrip!(s, label, k, Xn)\n", + " check_runs!(s, \"$label: save + load\") do\n", + " f = joinpath(mktempdir(), \"k.json\")\n", + " LK.save(k, f)\n", + " k2 = LK.load(f)\n", + " check_close!(s, \"$label: load(save(k)) predicts the same mean\",\n", + " LK.predict(k2, Xn; return_stdev=false).mean, LK.predict(k, Xn; return_stdev=false).mean; rtol=1e-10)\n", + " end\n", + "end" + ] + }, + { + "cell_type": "markdown", + "id": "c002", + "metadata": {}, + "source": [ + "## `Kriging` with an approximate fit: Vecchia, Nyström (n = 400)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c003", + "metadata": {}, + "outputs": [], + "source": [ + "Xl = lhs(400, 2); yl = branin(Xl)\n", + "for m in (\"vecchia(20)\", \"nystrom(60)\")\n", + " short = split(m, \"(\")[1]\n", + " label = \"Kriging(method = '$m')\"\n", + " check_runs!(s, \"$label: fit + predict\") do\n", + " ka = LK.Kriging(yl, Xl, \"matern5_2\"; method=m)\n", + " add_check!(s, \"$label: method()\", LK.method(ka) == m; value=LK.method(ka), expected=m)\n", + " pa = LK.predict(ka, Xt; return_stdev=true, method=String(short))\n", + " add_check!(s, \"$label: finite predictions\", finite_count(pa.mean, pa.stdev) == 0;\n", + " value=finite_count(pa.mean, pa.stdev), expected=\"0 non-finite\")\n", + " add_metric!(s, \"branin n=400\", \"Kriging $m\"; q2=q2(yt, pa.mean))\n", + " add_check!(s, \"$label: Q2\", q2(yt, pa.mean) > 0.95; value=q2(yt, pa.mean), expected=\"> 0.95\", severity=\"warn\")\n", + " end\n", + "end" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c004", + "metadata": {}, + "outputs": [], + "source": [ + "foreach(r -> r[5] == \"q2\" && println(rpad(r[3], 24), rpad(r[4], 28), r[6]), s.metrics)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c005", + "metadata": {}, + "outputs": [], + "source": [ + "finalize!(s)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Julia 1.11", + "language": "julia", + "name": "julia-1.11" + }, + "language_info": { + "name": "julia" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/large-n_Kriging-vecchia-nystrom_octave.ipynb b/large-n_Kriging-vecchia-nystrom_octave.ipynb new file mode 100644 index 0000000..3e5a3a3 --- /dev/null +++ b/large-n_Kriging-vecchia-nystrom_octave.ipynb @@ -0,0 +1,128 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "c000", + "metadata": {}, + "source": [ + "# Octave · `Kriging` with an approximate fit — Vecchia, Nyström (n = 400)\n", + "\n", + "Branin on [0,1]² (n = 30 LHS points unless stated, Q² on 500 uniform test\n", + "points), through mLibKriging. Checks (error-level unless stated):\n", + "\n", + "* the fit method is reported by `method()`\n", + "* `predict` with the fit method: finite mean and stdev\n", + "* Q² > 0.95 (warn-level)\n", + "\n", + "The same checks as `large-n_Kriging-vecchia-nystrom_R.ipynb`, with the Octave binding." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c001", + "metadata": {}, + "outputs": [], + "source": [ + "addpath(fullfile(pwd, \"tools\", \"octave\"));\n", + "s = lkb_suite(\"large-n_Kriging-vecchia-nystrom_octave\");\n", + "q2 = @(y, m) 1 - sum((y - m).^2) / sum((y - mean(y)).^2);\n", + "branin = @(U) ((15 * U(:, 2)) - 5.1 / (4 * pi^2) * (15 * U(:, 1) - 5).^2 + 5 / pi * (15 * U(:, 1) - 5) - 6).^2 ...\n", + " + 10 * (1 - 1 / (8 * pi)) * cos(15 * U(:, 1) - 5) + 10;\n", + "rand(\"seed\", 1); randn(\"seed\", 1);\n", + "function U = lhs(n, d)\n", + " U = zeros(n, d);\n", + " for j = 1:d, U(:, j) = (randperm(n)' - rand(n, 1)) / n; end\n", + "end\n", + "X = lhs(30, 2); y = branin(X);\n", + "Xt = rand(500, 2); yt = branin(Xt);\n", + "tmp = tempname(); mkdir(tmp);\n", + "has_class = @(name) exist(name) == 2; % a class file on the path\n", + "\n", + "function s = roundtrip(s, label, k, Xn, loader)\n", + " f = fullfile(tempdir(), [regexprep(label, \"[^A-Za-z0-9]\", \"_\") \".json\"]);\n", + " try\n", + " k.save(f);\n", + " k2 = loader(f);\n", + " m1 = k.predict(Xn, false, false, false);\n", + " m2 = k2.predict(Xn, false, false, false);\n", + " s = lkb_close(s, [label \": load(save(k)) predicts the same mean\"], m2, m1, 1e-10);\n", + " catch err\n", + " s = lkb_check(s, [label \": save + load\"], false, \"error\", \"no error\", err.message);\n", + " end\n", + "end\n", + "\n", + "function s = skip(s, check, reason)\n", + " s.checks(end + 1, :) = {s.name, \"octave\", check, \"info\", \"SKIP\", \"\", \"\", reason};\n", + " printf(\"[SKIP] %s: %s\\n\", check, reason);\n", + "end" + ] + }, + { + "cell_type": "markdown", + "id": "c002", + "metadata": {}, + "source": [ + "## `Kriging` with an approximate fit: Vecchia, Nyström (n = 400)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c003", + "metadata": {}, + "outputs": [], + "source": [ + "Xl = lhs(400, 2); yl = branin(Xl);\n", + "for mth = {\"vecchia(20)\", \"nystrom(60)\"}\n", + " mth = mth{1}; short = regexprep(mth, \"\\\\(.*\", \"\");\n", + " label = [\"Kriging(method = '\" mth \"')\"];\n", + " try\n", + " ka = Kriging(yl, Xl, \"matern5_2\", \"constant\", false, \"BFGS\", \"LL\", [], \"none\", [], mth);\n", + " got = ka.method();\n", + " s = lkb_check(s, [label \": method()\"], strcmp(got, mth), got, mth);\n", + " [ma, sa] = ka.predict(Xt, true, false, false, short);\n", + " s = lkb_check(s, [label \": finite predictions\"], all(isfinite([ma; sa])), sum(~isfinite([ma; sa])), \"0 non-finite\");\n", + " s = lkb_metric(s, \"branin n=400\", [\"Kriging \" mth], \"q2\", q2(yt, ma));\n", + " s = lkb_check(s, [label \": Q2\"], q2(yt, ma) > 0.95, q2(yt, ma), \"> 0.95\", \"\", \"warn\");\n", + " catch err\n", + " s = lkb_check(s, [label \": fit + predict\"], false, \"error\", \"no error\", err.message);\n", + " end\n", + "end" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c004", + "metadata": {}, + "outputs": [], + "source": [ + "q = strcmp(s.metrics(:, 5), \"q2\");\n", + "disp(s.metrics(q, [3, 4, 6]))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c005", + "metadata": {}, + "outputs": [], + "source": [ + "lkb_finalize(s);" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Octave", + "language": "octave", + "name": "octave" + }, + "language_info": { + "name": "octave" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/python_Kriging-vecchia-nystrom_large-n.ipynb b/large-n_Kriging-vecchia-nystrom_python.ipynb similarity index 98% rename from python_Kriging-vecchia-nystrom_large-n.ipynb rename to large-n_Kriging-vecchia-nystrom_python.ipynb index 8f3f051..e414b18 100644 --- a/python_Kriging-vecchia-nystrom_large-n.ipynb +++ b/large-n_Kriging-vecchia-nystrom_python.ipynb @@ -16,7 +16,7 @@ "* `Kriging.subsetOfData`: a representative subset before an exact fit.\n", "\n", "(`NestedKriging`, the partition-based alternative, on the same problems:\n", - "`python_NestedKriging_aggregations-large-n.ipynb`.)\n", + "`aggregations-large-n_NestedKriging_python.ipynb`.)\n", "\n", "Problems: Hartmann 3 (3D, local structure: Vecchia's ground) and the OTL\n", "circuit (6D, smooth: Nyström's ground), n = 1500 (exact fit still affordable,\n", @@ -46,7 +46,7 @@ "import lkbench\n", "from lkbench import TESTS, design, metrics, compat\n", "\n", - "suite = lkbench.Suite(\"python_Kriging-vecchia-nystrom_large-n\")\n", + "suite = lkbench.Suite(\"large-n_Kriging-vecchia-nystrom_python\")\n", "print(\"pylibkriging\", compat.version())\n", "HAS_METHOD = compat.has(\"method\")\n", "if not HAS_METHOD:\n", diff --git a/noisy-data_Kriging-nugget-noise_R.ipynb b/noisy-data_Kriging-nugget-noise_R.ipynb new file mode 100644 index 0000000..65c0196 --- /dev/null +++ b/noisy-data_Kriging-nugget-noise_R.ipynb @@ -0,0 +1,137 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "c000", + "metadata": {}, + "source": [ + "# R · `Kriging` with noise — estimated nugget, known variances\n", + "\n", + "Branin on [0,1]² (n = 30 LHS points unless stated, Q² on 500 uniform test\n", + "points), through rlibkriging. Checks (error-level unless stated):\n", + "\n", + "* `noise = \"nugget\"` (10 restarts): nugget > 0, Q² > 0.95 on the noise-free function, smoothing next to the design points\n", + "* the default single start reaches the multistart likelihood (warn-level)\n", + "* `save` then `load`: the same predicted mean\n", + "* `noise = `: the stored variances, finite predictions" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c001", + "metadata": {}, + "outputs": [], + "source": [ + "suppressPackageStartupMessages(library(rlibkriging))\n", + "source(\"tools/r/checks.R\")\n", + "s <- new_suite(\"noisy-data_Kriging-nugget-noise_R\")\n", + "cat(\"rlibkriging\", as.character(packageVersion(\"rlibkriging\")), \"\\n\")\n", + "\n", + "branin <- function(U) { # on [0,1]^2\n", + " x1 <- 15 * U[, 1] - 5; x2 <- 15 * U[, 2]\n", + " (x2 - 5.1 / (4 * pi^2) * x1^2 + 5 / pi * x1 - 6)^2 + 10 * (1 - 1 / (8 * pi)) * cos(x1) + 10\n", + "}\n", + "set.seed(1)\n", + "lhs <- function(n, d) sapply(1:d, function(j) (sample(n) - runif(n)) / n)\n", + "X <- lhs(30, 2); y <- branin(X)\n", + "Xt <- matrix(runif(1000), ncol = 2); yt <- branin(Xt)\n", + "tmp <- tempdir()\n", + "has_class <- function(name) exists(name, mode = \"function\")\n", + "skip <- function(s, check, reason) {\n", + " s$checks[[length(s$checks) + 1]] <- data.frame(suite = s$name, lang = \"R\", check = check, severity = \"info\",\n", + " status = \"SKIP\", value = \"\", expected = \"\", detail = reason, stringsAsFactors = FALSE)\n", + " message(\"[SKIP] \", check, \": \", reason)\n", + "}\n", + "roundtrip_same_mean <- function(s, label, k, Xn) {\n", + " f <- file.path(tmp, paste0(gsub(\"[^A-Za-z0-9]\", \"_\", label), \".json\"))\n", + " check_runs(s, paste0(label, \": save + load\"), {\n", + " save(k, f)\n", + " k2 <- rlibkriging::load(f)\n", + " check_close(s, paste0(label, \": load(save(k)) predicts the same mean\"),\n", + " predict(k2, Xn, return_stdev = FALSE)$mean, predict(k, Xn, return_stdev = FALSE)$mean, rtol = 1e-10)\n", + " })\n", + "}" + ] + }, + { + "cell_type": "markdown", + "id": "c002", + "metadata": {}, + "source": [ + "## `Kriging` with noise: estimated nugget, known variances\n", + "\n", + "Noise of standard deviation 5% of sd(y). The nugget model is fitted with 10\n", + "restarts (`optim = \"BFGS10\"`); the default single start is compared with it\n", + "(warn-level: a lower likelihood means a local optimum). As DiceKriging's\n", + "nugget, the predictor returns y exactly *at* the design points (the nugget is\n", + "a micro-scale variation): smoothing is checked next to them." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c003", + "metadata": {}, + "outputs": [], + "source": [ + "sd_noise <- 0.05 * sd(y)\n", + "yn <- y + rnorm(length(y), sd = sd_noise)\n", + "kn <- Kriging(yn, X, \"matern5_2\", noise = \"nugget\", optim = \"BFGS10\")\n", + "pn <- predict(kn, Xt, return_stdev = TRUE)\n", + "add_metric(s, \"branin noisy\", \"Kriging nugget BFGS10\", q2 = q2(yt, pn$mean), nugget = kn$nugget(),\n", + " ll = logLikelihood(kn))\n", + "check_greater(s, \"Kriging(nugget): estimated nugget > 0\", kn$nugget(), 0)\n", + "check_greater(s, \"Kriging(nugget): Q2\", q2(yt, pn$mean), 0.95)\n", + "near <- pmin(X + 1e-7, 1)\n", + "check_greater(s, \"Kriging(nugget): smooths the noisy data next to the design points (rms residual / noise sd)\",\n", + " rmse(yn, predict(kn, near, return_stdev = FALSE)$mean) / sd_noise, 0.3)\n", + "kn1 <- Kriging(yn, X, \"matern5_2\", noise = \"nugget\")\n", + "add_metric(s, \"branin noisy\", \"Kriging nugget BFGS\", q2 = q2(yt, predict(kn1, Xt, return_stdev = FALSE)$mean),\n", + " nugget = kn1$nugget(), ll = logLikelihood(kn1))\n", + "check_greater(s, \"Kriging(nugget): LL(single start) - LL(10 restarts)\", logLikelihood(kn1) - logLikelihood(kn), -1,\n", + " severity = \"warn\")\n", + "roundtrip_same_mean(s, \"Kriging(nugget)\", kn, Xt)\n", + "kh <- Kriging(yn, X, \"matern5_2\", noise = rep(sd_noise^2, length(yn)))\n", + "check_close(s, \"Kriging(noise = variances): noise() returns them\", kh$noise(), rep(sd_noise^2, length(yn)),\n", + " rtol = 1e-12)\n", + "ph <- predict(kh, Xt, return_stdev = TRUE)\n", + "check_less(s, \"Kriging(noise = variances): non-finite predictions\", sum(!is.finite(c(ph$mean, ph$stdev))), 1)\n", + "add_metric(s, \"branin noisy\", \"Kriging known noise\", q2 = q2(yt, ph$mean))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c004", + "metadata": {}, + "outputs": [], + "source": [ + "me <- do.call(rbind, s$metrics)\n", + "print(me[me$metric == \"q2\", c(\"case\", \"package\", \"value\")], row.names = FALSE)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c005", + "metadata": {}, + "outputs": [], + "source": [ + "finalize(s)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "R", + "language": "R", + "name": "ir" + }, + "language_info": { + "name": "R" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/noisy-data_Kriging-nugget-noise_julia.ipynb b/noisy-data_Kriging-nugget-noise_julia.ipynb new file mode 100644 index 0000000..77c1ad7 --- /dev/null +++ b/noisy-data_Kriging-nugget-noise_julia.ipynb @@ -0,0 +1,141 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "c000", + "metadata": {}, + "source": [ + "# Julia · `Kriging` with noise — estimated nugget, known variances\n", + "\n", + "Branin on [0,1]² (n = 30 LHS points unless stated, Q² on 500 uniform test\n", + "points), through JLibKriging. Checks (error-level unless stated):\n", + "\n", + "* `noise = \"nugget\"` (10 restarts): nugget > 0, Q² > 0.95 on the noise-free function, smoothing next to the design points\n", + "* the default single start reaches the multistart likelihood (warn-level)\n", + "* `save` then `load`: the same predicted mean\n", + "* `noise = `: the stored variances, finite predictions\n", + "\n", + "The same checks as `noisy-data_Kriging-nugget-noise_R.ipynb`, with the Julia binding." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c001", + "metadata": {}, + "outputs": [], + "source": [ + "import JLibKriging as LK\n", + "using Random\n", + "include(\"tools/julia/checks.jl\")\n", + "s = Suite(\"noisy-data_Kriging-nugget-noise_julia\")\n", + "\n", + "branin(U) = begin\n", + " x1 = 15 .* U[:, 1] .- 5; x2 = 15 .* U[:, 2]\n", + " (x2 .- 5.1 / (4π^2) .* x1 .^ 2 .+ 5 / π .* x1 .- 6) .^ 2 .+ 10 * (1 - 1 / (8π)) .* cos.(x1) .+ 10\n", + "end\n", + "rng = MersenneTwister(1)\n", + "lhs(n, d) = hcat([(randperm(rng, n) .- rand(rng, n)) ./ n for _ in 1:d]...)\n", + "X = lhs(30, 2); y = branin(X)\n", + "Xt = rand(rng, 500, 2); yt = branin(Xt)\n", + "sd(v) = sqrt(sum((v .- sum(v) / length(v)) .^ 2) / (length(v) - 1))\n", + "finite_count(v...) = count(!isfinite, vcat(v...))\n", + "\n", + "function check_runs!(f, s, check)\n", + " try\n", + " f()\n", + " catch e\n", + " add_check!(s, check, false; value=string(typeof(e)), expected=\"no error\", detail=first(sprint(showerror, e), 200))\n", + " end\n", + "end\n", + "\n", + "function roundtrip!(s, label, k, Xn)\n", + " check_runs!(s, \"$label: save + load\") do\n", + " f = joinpath(mktempdir(), \"k.json\")\n", + " LK.save(k, f)\n", + " k2 = LK.load(f)\n", + " check_close!(s, \"$label: load(save(k)) predicts the same mean\",\n", + " LK.predict(k2, Xn; return_stdev=false).mean, LK.predict(k, Xn; return_stdev=false).mean; rtol=1e-10)\n", + " end\n", + "end" + ] + }, + { + "cell_type": "markdown", + "id": "c002", + "metadata": {}, + "source": [ + "## `Kriging` with noise: estimated nugget, known variances\n", + "\n", + "Noise of standard deviation 5% of sd(y). The nugget model is fitted with 10\n", + "restarts (`optim=\"BFGS10\"`); the default single start is compared with it\n", + "(warn-level: a lower likelihood means a local optimum). As DiceKriging's\n", + "nugget, the predictor returns y exactly *at* the design points (the nugget is\n", + "a micro-scale variation): smoothing is checked next to them." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c003", + "metadata": {}, + "outputs": [], + "source": [ + "sd_noise = 0.05 * sd(y)\n", + "yn = y .+ sd_noise .* randn(rng, length(y))\n", + "kn = LK.Kriging(yn, X, \"matern5_2\"; noise=\"nugget\", optim=\"BFGS10\")\n", + "mn = LK.predict(kn, Xt; return_stdev=false).mean\n", + "add_metric!(s, \"branin noisy\", \"Kriging nugget BFGS10\"; q2=q2(yt, mn))\n", + "add_check!(s, \"Kriging(nugget): estimated nugget > 0\", LK.nugget(kn) > 0; value=LK.nugget(kn), expected=\"> 0\")\n", + "add_check!(s, \"Kriging(nugget): Q2\", q2(yt, mn) > 0.95; value=q2(yt, mn), expected=\"> 0.95\")\n", + "r = sqrt(sum((LK.predict(kn, min.(X .+ 1e-7, 1.0); return_stdev=false).mean .- yn) .^ 2) / length(yn)) / sd_noise\n", + "add_check!(s, \"Kriging(nugget): smooths the noisy data next to the design points (rms residual / noise sd)\", r > 0.3;\n", + " value=r, expected=\"> 0.3\")\n", + "kn1 = LK.Kriging(yn, X, \"matern5_2\"; noise=\"nugget\")\n", + "add_metric!(s, \"branin noisy\", \"Kriging nugget BFGS\"; q2=q2(yt, LK.predict(kn1, Xt; return_stdev=false).mean))\n", + "dll = LK.log_likelihood(kn1) - LK.log_likelihood(kn)\n", + "add_check!(s, \"Kriging(nugget): LL(single start) - LL(10 restarts)\", dll > -1; value=dll, expected=\"> -1\", severity=\"warn\")\n", + "roundtrip!(s, \"Kriging(nugget)\", kn, Xt)\n", + "v = fill(sd_noise^2, length(yn))\n", + "kh = LK.Kriging(yn, X, \"matern5_2\"; noise=v)\n", + "check_close!(s, \"Kriging(noise = variances): noise() returns them\", LK.noise(kh), v; rtol=1e-12)\n", + "ph = LK.predict(kh, Xt; return_stdev=true)\n", + "add_check!(s, \"Kriging(noise = variances): finite predictions\", finite_count(ph.mean, ph.stdev) == 0;\n", + " value=finite_count(ph.mean, ph.stdev), expected=\"0 non-finite\")\n", + "add_metric!(s, \"branin noisy\", \"Kriging known noise\"; q2=q2(yt, ph.mean))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c004", + "metadata": {}, + "outputs": [], + "source": [ + "foreach(r -> r[5] == \"q2\" && println(rpad(r[3], 24), rpad(r[4], 28), r[6]), s.metrics)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c005", + "metadata": {}, + "outputs": [], + "source": [ + "finalize!(s)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Julia 1.11", + "language": "julia", + "name": "julia-1.11" + }, + "language_info": { + "name": "julia" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/noisy-data_Kriging-nugget-noise_octave.ipynb b/noisy-data_Kriging-nugget-noise_octave.ipynb new file mode 100644 index 0000000..0741b79 --- /dev/null +++ b/noisy-data_Kriging-nugget-noise_octave.ipynb @@ -0,0 +1,140 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "c000", + "metadata": {}, + "source": [ + "# Octave · `Kriging` with noise — estimated nugget, known variances\n", + "\n", + "Branin on [0,1]² (n = 30 LHS points unless stated, Q² on 500 uniform test\n", + "points), through mLibKriging. Checks (error-level unless stated):\n", + "\n", + "* `noise = \"nugget\"` (10 restarts): nugget > 0, Q² > 0.95 on the noise-free function, smoothing next to the design points\n", + "* the default single start reaches the multistart likelihood (warn-level)\n", + "* `save` then `load`: the same predicted mean\n", + "* `noise = `: the stored variances, finite predictions\n", + "\n", + "The same checks as `noisy-data_Kriging-nugget-noise_R.ipynb`, with the Octave binding." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c001", + "metadata": {}, + "outputs": [], + "source": [ + "addpath(fullfile(pwd, \"tools\", \"octave\"));\n", + "s = lkb_suite(\"noisy-data_Kriging-nugget-noise_octave\");\n", + "q2 = @(y, m) 1 - sum((y - m).^2) / sum((y - mean(y)).^2);\n", + "branin = @(U) ((15 * U(:, 2)) - 5.1 / (4 * pi^2) * (15 * U(:, 1) - 5).^2 + 5 / pi * (15 * U(:, 1) - 5) - 6).^2 ...\n", + " + 10 * (1 - 1 / (8 * pi)) * cos(15 * U(:, 1) - 5) + 10;\n", + "rand(\"seed\", 1); randn(\"seed\", 1);\n", + "function U = lhs(n, d)\n", + " U = zeros(n, d);\n", + " for j = 1:d, U(:, j) = (randperm(n)' - rand(n, 1)) / n; end\n", + "end\n", + "X = lhs(30, 2); y = branin(X);\n", + "Xt = rand(500, 2); yt = branin(Xt);\n", + "tmp = tempname(); mkdir(tmp);\n", + "has_class = @(name) exist(name) == 2; % a class file on the path\n", + "\n", + "function s = roundtrip(s, label, k, Xn, loader)\n", + " f = fullfile(tempdir(), [regexprep(label, \"[^A-Za-z0-9]\", \"_\") \".json\"]);\n", + " try\n", + " k.save(f);\n", + " k2 = loader(f);\n", + " m1 = k.predict(Xn, false, false, false);\n", + " m2 = k2.predict(Xn, false, false, false);\n", + " s = lkb_close(s, [label \": load(save(k)) predicts the same mean\"], m2, m1, 1e-10);\n", + " catch err\n", + " s = lkb_check(s, [label \": save + load\"], false, \"error\", \"no error\", err.message);\n", + " end\n", + "end\n", + "\n", + "function s = skip(s, check, reason)\n", + " s.checks(end + 1, :) = {s.name, \"octave\", check, \"info\", \"SKIP\", \"\", \"\", reason};\n", + " printf(\"[SKIP] %s: %s\\n\", check, reason);\n", + "end" + ] + }, + { + "cell_type": "markdown", + "id": "c002", + "metadata": {}, + "source": [ + "## `Kriging` with noise: estimated nugget, known variances\n", + "\n", + "Noise of standard deviation 5% of sd(y). The nugget model is fitted with 10\n", + "restarts (`\"BFGS10\"`); the default single start is compared with it\n", + "(warn-level: a lower likelihood means a local optimum). As DiceKriging's\n", + "nugget, the predictor returns y exactly *at* the design points (the nugget is\n", + "a micro-scale variation): smoothing is checked next to them." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c003", + "metadata": {}, + "outputs": [], + "source": [ + "sd_noise = 0.05 * std(y);\n", + "yn = y + sd_noise * randn(size(y));\n", + "kn = Kriging(yn, X, \"matern5_2\", \"constant\", false, \"BFGS10\", \"LL\", [], \"nugget\");\n", + "mn = kn.predict(Xt, false, false, false);\n", + "nug = kn.nugget(); ll10 = kn.logLikelihood();\n", + "s = lkb_metric(s, \"branin noisy\", \"Kriging nugget BFGS10\", \"q2\", q2(yt, mn));\n", + "s = lkb_check(s, \"Kriging(nugget): estimated nugget > 0\", nug > 0, nug, \"> 0\");\n", + "s = lkb_check(s, \"Kriging(nugget): Q2\", q2(yt, mn) > 0.95, q2(yt, mn), \"> 0.95\");\n", + "r = sqrt(mean((kn.predict(min(X + 1e-7, 1), false, false, false) - yn).^2)) / sd_noise;\n", + "s = lkb_check(s, \"Kriging(nugget): smooths the noisy data next to the design points (rms residual / noise sd)\", r > 0.3, r, \"> 0.3\");\n", + "kn1 = Kriging(yn, X, \"matern5_2\", \"constant\", false, \"BFGS\", \"LL\", [], \"nugget\");\n", + "ll1 = kn1.logLikelihood();\n", + "s = lkb_metric(s, \"branin noisy\", \"Kriging nugget BFGS\", \"q2\", q2(yt, kn1.predict(Xt, false, false, false)));\n", + "s = lkb_check(s, \"Kriging(nugget): LL(single start) - LL(10 restarts)\", ll1 - ll10 > -1, ll1 - ll10, \"> -1\", \"\", \"warn\");\n", + "s = roundtrip(s, \"Kriging(nugget)\", kn, Xt, @load_kriging);\n", + "v = sd_noise^2 * ones(size(yn));\n", + "kh = Kriging(yn, X, \"matern5_2\", \"constant\", false, \"BFGS\", \"LL\", [], \"heterogeneous\", v);\n", + "s = lkb_close(s, \"Kriging(noise = variances): noise() returns them\", kh.noise(), v, 1e-12);\n", + "[mh, sh] = kh.predict(Xt, true, false, false);\n", + "s = lkb_check(s, \"Kriging(noise = variances): finite predictions\", all(isfinite([mh; sh])), sum(~isfinite([mh; sh])), \"0 non-finite\");\n", + "s = lkb_metric(s, \"branin noisy\", \"Kriging known noise\", \"q2\", q2(yt, mh));" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c004", + "metadata": {}, + "outputs": [], + "source": [ + "q = strcmp(s.metrics(:, 5), \"q2\");\n", + "disp(s.metrics(q, [3, 4, 6]))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c005", + "metadata": {}, + "outputs": [], + "source": [ + "lkb_finalize(s);" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Octave", + "language": "octave", + "name": "octave" + }, + "language_info": { + "name": "octave" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/python_Kriging-nugget-noise_noisy-data.ipynb b/noisy-data_Kriging-nugget-noise_python.ipynb similarity index 99% rename from python_Kriging-nugget-noise_noisy-data.ipynb rename to noisy-data_Kriging-nugget-noise_python.ipynb index 1d0c11b..e06bc56 100644 --- a/python_Kriging-nugget-noise_noisy-data.ipynb +++ b/noisy-data_Kriging-nugget-noise_python.ipynb @@ -48,7 +48,7 @@ "import lkbench\n", "from lkbench import TESTS, design, metrics, compat\n", "\n", - "suite = lkbench.Suite(\"python_Kriging-nugget-noise_noisy-data\")\n", + "suite = lkbench.Suite(\"noisy-data_Kriging-nugget-noise_python\")\n", "print(\"pylibkriging\", compat.version())\n", "\n", "\n", diff --git a/python_Kriging_objectives-LL-LOO-LMP-gradients.ipynb b/objectives-LL-LOO-LMP-gradients_Kriging_python.ipynb similarity index 99% rename from python_Kriging_objectives-LL-LOO-LMP-gradients.ipynb rename to objectives-LL-LOO-LMP-gradients_Kriging_python.ipynb index 9ba2c39..6961b48 100644 --- a/python_Kriging_objectives-LL-LOO-LMP-gradients.ipynb +++ b/objectives-LL-LOO-LMP-gradients_Kriging_python.ipynb @@ -45,7 +45,7 @@ "import lkbench\n", "from lkbench import TESTS, design, metrics, compat\n", "\n", - "suite = lkbench.Suite(\"python_Kriging_objectives-LL-LOO-LMP-gradients\")\n", + "suite = lkbench.Suite(\"objectives-LL-LOO-LMP-gradients_Kriging_python\")\n", "print(\"pylibkriging\", compat.version())\n", "CASES = {\"branin\": 25, \"hartmann3\": 40, \"ishigami\": 60, \"friedman\": 50}\n", "KERNELS = [\"gauss\", \"exp\", \"matern3_2\", \"matern5_2\"]\n", diff --git a/octave_all-classes_api-tour.ipynb b/octave_all-classes_api-tour.ipynb deleted file mode 100644 index b0e0ccb..0000000 --- a/octave_all-classes_api-tour.ipynb +++ /dev/null @@ -1,328 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "c000", - "metadata": {}, - "source": [ - "# Octave · all classes — every libKriging model class and fit method through mLibKriging\n", - "\n", - "Same tour as `R_all-classes_api-tour.ipynb` (see there for the checks): Branin\n", - "on [0,1]², every class and fit method of the Octave binding (positional\n", - "arguments), each with the checks that hold for it whatever the data. A class\n", - "missing from the installed mLibKriging is recorded as SKIP. mLibKriging must\n", - "be on the path (`OCTAVE_PATH` or `addpath`), see `tools/install/octave.sh`." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c001", - "metadata": {}, - "outputs": [], - "source": [ - "addpath(fullfile(pwd, \"tools\", \"octave\"));\n", - "s = lkb_suite(\"octave_all-classes_api-tour\");\n", - "q2 = @(y, m) 1 - sum((y - m).^2) / sum((y - mean(y)).^2);\n", - "branin = @(U) ((15 * U(:, 2)) - 5.1 / (4 * pi^2) * (15 * U(:, 1) - 5).^2 + 5 / pi * (15 * U(:, 1) - 5) - 6).^2 ...\n", - " + 10 * (1 - 1 / (8 * pi)) * cos(15 * U(:, 1) - 5) + 10;\n", - "rand(\"seed\", 1); randn(\"seed\", 1);\n", - "function U = lhs(n, d)\n", - " U = zeros(n, d);\n", - " for j = 1:d, U(:, j) = (randperm(n)' - rand(n, 1)) / n; end\n", - "end\n", - "X = lhs(30, 2); y = branin(X);\n", - "Xt = rand(500, 2); yt = branin(Xt);\n", - "tmp = tempname(); mkdir(tmp);\n", - "has_class = @(name) exist(name) == 2; % a class file on the path\n", - "\n", - "function s = roundtrip(s, label, k, Xn, loader)\n", - " f = fullfile(tempdir(), [regexprep(label, \"[^A-Za-z0-9]\", \"_\") \".json\"]);\n", - " try\n", - " k.save(f);\n", - " k2 = loader(f);\n", - " m1 = k.predict(Xn, false, false, false);\n", - " m2 = k2.predict(Xn, false, false, false);\n", - " s = lkb_close(s, [label \": load(save(k)) predicts the same mean\"], m2, m1, 1e-10);\n", - " catch err\n", - " s = lkb_check(s, [label \": save + load\"], false, \"error\", \"no error\", err.message);\n", - " end\n", - "end\n", - "\n", - "function s = skip(s, check, reason)\n", - " s.checks(end + 1, :) = {s.name, \"octave\", check, \"info\", \"SKIP\", \"\", \"\", reason};\n", - " printf(\"[SKIP] %s: %s\\n\", check, reason);\n", - "end" - ] - }, - { - "cell_type": "markdown", - "id": "c002", - "metadata": {}, - "source": [ - "## `Kriging`, noise-free" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c003", - "metadata": {}, - "outputs": [], - "source": [ - "k = Kriging(y, X, \"matern5_2\");\n", - "[m, sd] = k.predict(Xt, true, false, false);\n", - "s = lkb_metric(s, \"branin\", \"Kriging\", \"q2\", q2(yt, m));\n", - "s = lkb_check(s, \"Kriging: Q2\", q2(yt, m) > 0.95, q2(yt, m), \"> 0.95\");\n", - "mX = k.predict(X, false, false, false);\n", - "e = max(abs(mX - y)) / std(y);\n", - "s = lkb_check(s, \"Kriging: interpolates (max |m(X) - y| / sd(y))\", e < 1e-6, e, \"< 1e-6\");\n", - "S = k.simulate(int32(5), int32(2), X, false);\n", - "e = max(max(abs(S - y))) / sqrt(k.sigma2());\n", - "s = lkb_check(s, \"Kriging: conditional simulations go through the data (max |sim(X) - y| / sigma)\", e < 1e-4, e, \"< 1e-4\");\n", - "s = lkb_check(s, \"Kriging: stdev > 0 off the design\", min(sd) > 0, min(sd), \"> 0\");\n", - "Xu = lhs(5, 2); yu = branin(Xu);\n", - "ku = k.copy();\n", - "ku.update(yu, Xu, false);\n", - "th = k.theta();\n", - "k35 = Kriging([y; yu], [X; Xu], \"matern5_2\", \"constant\", false, \"none\", \"LL\", ...\n", - " Params(\"theta\", th(:)', \"is_theta_estim\", false, \"sigma2\", k.sigma2(), \"is_sigma2_estim\", false));\n", - "s = lkb_close(s, \"Kriging: update(refit = false) == fit on all points (same theta, sigma2)\", ...\n", - " ku.predict(Xt, false, false, false), k35.predict(Xt, false, false, false), 1e-6, 1e-8 * std(y));\n", - "s = roundtrip(s, \"Kriging\", k, Xt, @load_kriging);" - ] - }, - { - "cell_type": "markdown", - "id": "c004", - "metadata": {}, - "source": [ - "## `Kriging` with noise: estimated nugget, known variances\n", - "\n", - "Noise of standard deviation 5% of sd(y). The nugget model is fitted with 10\n", - "restarts (`\"BFGS10\"`); the default single start is compared with it\n", - "(warn-level: a lower likelihood means a local optimum). As DiceKriging's\n", - "nugget, the predictor returns y exactly *at* the design points (the nugget is\n", - "a micro-scale variation): smoothing is checked next to them." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c005", - "metadata": {}, - "outputs": [], - "source": [ - "sd_noise = 0.05 * std(y);\n", - "yn = y + sd_noise * randn(size(y));\n", - "kn = Kriging(yn, X, \"matern5_2\", \"constant\", false, \"BFGS10\", \"LL\", [], \"nugget\");\n", - "mn = kn.predict(Xt, false, false, false);\n", - "nug = kn.nugget(); ll10 = kn.logLikelihood();\n", - "s = lkb_metric(s, \"branin noisy\", \"Kriging nugget BFGS10\", \"q2\", q2(yt, mn));\n", - "s = lkb_check(s, \"Kriging(nugget): estimated nugget > 0\", nug > 0, nug, \"> 0\");\n", - "s = lkb_check(s, \"Kriging(nugget): Q2\", q2(yt, mn) > 0.95, q2(yt, mn), \"> 0.95\");\n", - "r = sqrt(mean((kn.predict(min(X + 1e-7, 1), false, false, false) - yn).^2)) / sd_noise;\n", - "s = lkb_check(s, \"Kriging(nugget): smooths the noisy data next to the design points (rms residual / noise sd)\", r > 0.3, r, \"> 0.3\");\n", - "kn1 = Kriging(yn, X, \"matern5_2\", \"constant\", false, \"BFGS\", \"LL\", [], \"nugget\");\n", - "ll1 = kn1.logLikelihood();\n", - "s = lkb_metric(s, \"branin noisy\", \"Kriging nugget BFGS\", \"q2\", q2(yt, kn1.predict(Xt, false, false, false)));\n", - "s = lkb_check(s, \"Kriging(nugget): LL(single start) - LL(10 restarts)\", ll1 - ll10 > -1, ll1 - ll10, \"> -1\", \"\", \"warn\");\n", - "s = roundtrip(s, \"Kriging(nugget)\", kn, Xt, @load_kriging);\n", - "v = sd_noise^2 * ones(size(yn));\n", - "kh = Kriging(yn, X, \"matern5_2\", \"constant\", false, \"BFGS\", \"LL\", [], \"heterogeneous\", v);\n", - "s = lkb_close(s, \"Kriging(noise = variances): noise() returns them\", kh.noise(), v, 1e-12);\n", - "[mh, sh] = kh.predict(Xt, true, false, false);\n", - "s = lkb_check(s, \"Kriging(noise = variances): finite predictions\", all(isfinite([mh; sh])), sum(~isfinite([mh; sh])), \"0 non-finite\");\n", - "s = lkb_metric(s, \"branin noisy\", \"Kriging known noise\", \"q2\", q2(yt, mh));" - ] - }, - { - "cell_type": "markdown", - "id": "c006", - "metadata": {}, - "source": [ - "## `Kriging` with an approximate fit: Vecchia, Nyström (n = 400)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c007", - "metadata": {}, - "outputs": [], - "source": [ - "Xl = lhs(400, 2); yl = branin(Xl);\n", - "for mth = {\"vecchia(20)\", \"nystrom(60)\"}\n", - " mth = mth{1}; short = regexprep(mth, \"\\\\(.*\", \"\");\n", - " label = [\"Kriging(method = '\" mth \"')\"];\n", - " try\n", - " ka = Kriging(yl, Xl, \"matern5_2\", \"constant\", false, \"BFGS\", \"LL\", [], \"none\", [], mth);\n", - " got = ka.method();\n", - " s = lkb_check(s, [label \": method()\"], strcmp(got, mth), got, mth);\n", - " [ma, sa] = ka.predict(Xt, true, false, false, short);\n", - " s = lkb_check(s, [label \": finite predictions\"], all(isfinite([ma; sa])), sum(~isfinite([ma; sa])), \"0 non-finite\");\n", - " s = lkb_metric(s, \"branin n=400\", [\"Kriging \" mth], \"q2\", q2(yt, ma));\n", - " s = lkb_check(s, [label \": Q2\"], q2(yt, ma) > 0.95, q2(yt, ma), \"> 0.95\", \"\", \"warn\");\n", - " catch err\n", - " s = lkb_check(s, [label \": fit + predict\"], false, \"error\", \"no error\", err.message);\n", - " end\n", - "end" - ] - }, - { - "cell_type": "markdown", - "id": "c008", - "metadata": {}, - "source": [ - "## `WarpKriging`: continuous warpings, then a categorical input\n", - "\n", - "The categorical case: x₂ restricted to 5 levels (coded 0, …, 4), embedded in ℝ²." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c009", - "metadata": {}, - "outputs": [], - "source": [ - "if has_class(\"WarpKriging\")\n", - " try\n", - " kw = WarpKriging(y, X, {\"kumaraswamy\", \"kumaraswamy\"}, \"matern5_2\");\n", - " [mw, sw] = kw.predict(Xt, true, false, false);\n", - " s = lkb_check(s, \"WarpKriging(kumaraswamy): finite predictions\", all(isfinite([mw; sw])), sum(~isfinite([mw; sw])), \"0 non-finite\");\n", - " s = lkb_metric(s, \"branin\", \"WarpKriging kumaraswamy\", \"q2\", q2(yt, mw));\n", - " s = lkb_check(s, \"WarpKriging(kumaraswamy): Q2\", q2(yt, mw) > 0.9, q2(yt, mw), \"> 0.9\", \"\", \"warn\");\n", - " s = roundtrip(s, \"WarpKriging(kumaraswamy)\", kw, Xt, @load_kriging);\n", - " catch err\n", - " s = lkb_check(s, \"WarpKriging(kumaraswamy): fit + predict\", false, \"error\", \"no error\", err.message);\n", - " end\n", - " lev = @(U) [U(:, 1), round(U(:, 2) * 4)];\n", - " unlev = @(L) [L(:, 1), L(:, 2) / 4];\n", - " Xc = lev(lhs(40, 2)); yc = branin(unlev(Xc));\n", - " Xtc = lev(Xt); ytc = branin(unlev(Xtc));\n", - " try\n", - " kc = WarpKriging(yc, Xc, {\"none\", \"categorical(5,2)\"}, \"matern5_2\");\n", - " [mc, sc] = kc.predict(Xtc, true, false, false);\n", - " s = lkb_check(s, \"WarpKriging(categorical): finite predictions\", all(isfinite([mc; sc])), sum(~isfinite([mc; sc])), \"0 non-finite\");\n", - " s = lkb_metric(s, \"branin, x2 categorical\", \"WarpKriging categorical\", \"q2\", q2(ytc, mc));\n", - " s = lkb_check(s, \"WarpKriging(categorical): Q2\", q2(ytc, mc) > 0.8, q2(ytc, mc), \"> 0.8\", \"\", \"warn\");\n", - " s = roundtrip(s, \"WarpKriging(categorical)\", kc, Xtc, @load_kriging);\n", - " catch err\n", - " s = lkb_check(s, \"WarpKriging(categorical): fit + predict\", false, \"error\", \"no error\", err.message);\n", - " end\n", - "else\n", - " s = skip(s, \"WarpKriging\", \"not in this mLibKriging\");\n", - "end" - ] - }, - { - "cell_type": "markdown", - "id": "c010", - "metadata": {}, - "source": [ - "## `MLPKriging`: a joint feature map ℝ² → ℝ² before the kernel" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c011", - "metadata": {}, - "outputs": [], - "source": [ - "if has_class(\"MLPKriging\")\n", - " try\n", - " km = MLPKriging(y, X, [8], 2, \"tanh\", \"matern5_2\", \"constant\", false, \"BFGS10\");\n", - " [mm, sm] = km.predict(Xt, true, false, false);\n", - " s = lkb_check(s, \"MLPKriging: finite predictions\", all(isfinite([mm; sm])), sum(~isfinite([mm; sm])), \"0 non-finite\");\n", - " s = lkb_metric(s, \"branin\", \"MLPKriging tanh [8] -> 2, BFGS10\", \"q2\", q2(yt, mm));\n", - " s = lkb_check(s, \"MLPKriging: Q2\", q2(yt, mm) > 0.9, q2(yt, mm), \"> 0.9\", \"\", \"warn\");\n", - " s = roundtrip(s, \"MLPKriging\", km, Xt, @load_kriging);\n", - " kd = MLPKriging(y, X, [8], 2, \"tanh\", \"matern5_2\"); % default optimizer\n", - " qd = q2(yt, kd.predict(Xt, false, false, false));\n", - " s = lkb_metric(s, \"branin\", \"MLPKriging tanh [8] -> 2, default optim\", \"q2\", qd);\n", - " s = lkb_check(s, \"MLPKriging: Q2(default optim) - Q2(BFGS10)\", qd - q2(yt, mm) > -0.05, qd - q2(yt, mm), \"> -0.05\", \"\", \"warn\");\n", - " catch err\n", - " s = lkb_check(s, \"MLPKriging: fit + predict\", false, \"error\", \"no error\", err.message);\n", - " end\n", - "else\n", - " s = skip(s, \"MLPKriging\", \"not in this mLibKriging\");\n", - "end" - ] - }, - { - "cell_type": "markdown", - "id": "c012", - "metadata": {}, - "source": [ - "## `NestedKriging`: 4 groups of 100 points, `NK` and `PoE` aggregations" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c013", - "metadata": {}, - "outputs": [], - "source": [ - "if has_class(\"NestedKriging\")\n", - " for agg = {\"NK\", \"PoE\"}\n", - " agg = agg{1}; label = [\"NestedKriging(\" agg \")\"];\n", - " try\n", - " kk = NestedKriging(yl, Xl, \"matern5_2\", 4, agg, \"kmeans\", 3);\n", - " [mk, sk] = kk.predict(Xt); % stdev returned when asked for (2 outputs)\n", - " s = lkb_check(s, [label \": stdev >= 0\"], all(sk >= 0), min(sk), \">= 0\");\n", - " s = lkb_metric(s, \"branin n=400\", [\"NestedKriging \" agg], \"q2\", q2(yt, mk));\n", - " s = lkb_check(s, [label \": Q2\"], q2(yt, mk) > 0.95, q2(yt, mk), \"> 0.95\", \"\", \"warn\");\n", - " if strcmp(agg, \"NK\")\n", - " e = max(abs(kk.predict(Xl) - yl)) / std(yl);\n", - " s = lkb_check(s, \"NestedKriging(NK): interpolates (max |m(X) - y| / sd(y))\", e < 1e-3, e, \"< 1e-3\");\n", - " end\n", - " f = fullfile(tmp, [\"nested_\" agg \".json\"]);\n", - " kk.save(f); k2 = load_kriging(f);\n", - " s = lkb_close(s, [label \": load(save(k)) predicts the same mean\"], k2.predict(Xt), kk.predict(Xt), 1e-10);\n", - " catch err\n", - " s = lkb_check(s, [label \": fit + predict\"], false, \"error\", \"no error\", err.message);\n", - " end\n", - " end\n", - "else\n", - " s = skip(s, \"NestedKriging\", \"not in this mLibKriging\");\n", - "end" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c014", - "metadata": {}, - "outputs": [], - "source": [ - "q = strcmp(s.metrics(:, 5), \"q2\");\n", - "disp(s.metrics(q, [3, 4, 6]))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c015", - "metadata": {}, - "outputs": [], - "source": [ - "lkb_finalize(s);" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Octave", - "language": "octave", - "name": "octave" - }, - "language_info": { - "name": "octave" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/python_all-classes_save-load-pickle-sklearn.ipynb b/python_all-classes_save-load-pickle-sklearn.ipynb deleted file mode 100644 index ebdcf22..0000000 --- a/python_all-classes_save-load-pickle-sklearn.ipynb +++ /dev/null @@ -1,296 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "c000", - "metadata": {}, - "source": [ - "# Python · all classes — save/load, pickle, copy, accessors, scikit-learn estimators\n", - "\n", - "Persistence and introspection, for every model class:\n", - "\n", - "* `save` → `load` (class-specific and generic `pylibkriging.load`) gives a model\n", - " with **identical** predictions and hyperparameters;\n", - "* `pickle` / `copy` round trips, and a copy is independent of the original\n", - " (updating one leaves the other untouched);\n", - "* accessors mirror the constructor arguments (`kernel()`, `regmodel()`,\n", - " `optim()`, `objective()`, `X()`, `y()`, ...);\n", - "* scikit-learn estimators (`pylibkriging.sklearn`): `fit`/`predict`,\n", - " `get_params`/`set_params`/`clone`, use inside `Pipeline` and `GridSearchCV`,\n", - " same predictions as the native class." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c001", - "metadata": {}, - "outputs": [], - "source": [ - "import copy\n", - "import pickle\n", - "import tempfile\n", - "from pathlib import Path\n", - "\n", - "import numpy as np\n", - "import pylibkriging as lk\n", - "\n", - "import lkbench\n", - "from lkbench import TESTS, design, compat\n", - "\n", - "suite = lkbench.Suite(\"python_all-classes_save-load-pickle-sklearn\")\n", - "print(\"pylibkriging\", compat.version())\n", - "tmp = Path(tempfile.mkdtemp())\n", - "\n", - "tf = TESTS[\"hartmann3\"]\n", - "X, y, Xt, yt = design.dataset(tf, 40, n_test=100, seed=61)\n", - "\n", - "\n", - "def mean_sd(k, Xn):\n", - " out = k.predict(Xn, return_stdev=True)\n", - " return np.ravel(out[0]), np.ravel(out[1])\n", - "\n", - "\n", - "MODELS = {\n", - " \"Kriging\": lambda: lk.Kriging(y, X, \"matern5_2\", regmodel=\"linear\"),\n", - " \"Kriging nugget\": lambda: lk.Kriging(y, X, \"matern3_2\", noise=\"nugget\"),\n", - " \"Kriging noise vector\": lambda: lk.Kriging(y, X, \"gauss\", noise=np.full(len(y), 1e-4)),\n", - " \"Kriging LOO\": lambda: lk.Kriging(y, X, \"matern5_2\", objective=\"LOO\"),\n", - " \"Kriging LMP\": lambda: lk.Kriging(y, X, \"matern5_2\", objective=\"LMP\"),\n", - "}\n", - "if compat.has(\"method\"):\n", - " MODELS[\"Kriging vecchia\"] = lambda: lk.Kriging(y, X, \"matern5_2\", method=\"vecchia(10)\")\n", - " MODELS[\"Kriging nystrom\"] = lambda: lk.Kriging(y, X, \"matern5_2\", method=\"nystrom(20)\")\n", - "if compat.has(\"WarpKriging\"):\n", - " MODELS[\"WarpKriging\"] = lambda: lk.WarpKriging(y, X, [\"kumaraswamy\", \"none\", \"affine\"], \"matern5_2\")\n", - "if compat.has(\"MLPKriging\"):\n", - " MODELS[\"MLPKriging\"] = lambda: lk.MLPKriging(y, X, hidden_dims=[8], d_out=2, activation=\"tanh\",\n", - " kernel=\"matern5_2\")\n", - "if compat.has(\"NestedKriging\"):\n", - " MODELS[\"NestedKriging\"] = lambda: lk.NestedKriging(y, X, \"matern5_2\", nb_groups=4, seed=1)\n", - "\n", - "\n", - "def predict_any(k, Xn):\n", - " meth = k.method() if hasattr(k, \"method\") and isinstance(k, lk.Kriging) else None\n", - " if meth and meth != \"exact\":\n", - " out = k.predict(Xn, return_stdev=True, method=meth.split(\"(\")[0])\n", - " return np.ravel(out[0]), np.ravel(out[1])\n", - " return mean_sd(k, Xn)" - ] - }, - { - "cell_type": "markdown", - "id": "c002", - "metadata": {}, - "source": [ - "## save / load, pickle, copy" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c003", - "metadata": {}, - "outputs": [], - "source": [ - "for name, make in MODELS.items():\n", - " with suite.runs(f\"{name}: fit\"):\n", - " k = make()\n", - " m0, s0 = predict_any(k, Xt)\n", - " cls = type(k)\n", - " path = tmp / f\"{name.replace(' ', '_')}.json\"\n", - " if not hasattr(k, \"save\"):\n", - " suite.skip(f\"{name}: save + load\", \"no save() in this pylibkriging\")\n", - " continue\n", - " with suite.runs(f\"{name}: save + load\"):\n", - " k.save(str(path))\n", - " k1 = cls.load(str(path))\n", - " m1, s1 = predict_any(k1, Xt)\n", - " suite.close(f\"{name}: load(save(k)) predicts the same mean\", m1, m0, rtol=1e-12, atol=1e-12)\n", - " suite.close(f\"{name}: load(save(k)) predicts the same stdev\", s1, s0, rtol=1e-10, atol=1e-12)\n", - " if hasattr(k, \"theta\"):\n", - " suite.close(f\"{name}: same theta after load\", np.ravel(k1.theta()), np.ravel(k.theta()),\n", - " rtol=1e-14, atol=0)\n", - " if hasattr(lk, \"load\") and (compat.has(\"load(noise)\") or \"noise\" not in name and \"nugget\" not in name):\n", - " with suite.runs(f\"{name}: generic pylibkriging.load\"):\n", - " k2 = lk.load(str(path))\n", - " suite.true(f\"{name}: generic load returns {cls.__name__}\", type(k2).__name__ == cls.__name__,\n", - " detail=type(k2).__name__)\n", - " suite.close(f\"{name}: generic load predicts the same\", predict_any(k2, Xt)[0], m0, rtol=1e-12,\n", - " atol=1e-12)\n", - " if not compat.has(\"pickle\"):\n", - " suite.skip(f\"{name}: pickle / deepcopy\", \"models not picklable in this pylibkriging\")\n", - " continue\n", - " with suite.runs(f\"{name}: pickle round trip\"):\n", - " k3 = pickle.loads(pickle.dumps(k))\n", - " suite.close(f\"{name}: pickle predicts the same\", predict_any(k3, Xt)[0], m0, rtol=1e-12, atol=1e-12)\n", - " with suite.runs(f\"{name}: deepcopy\"):\n", - " k4 = copy.deepcopy(k)\n", - " suite.close(f\"{name}: deepcopy predicts the same\", predict_any(k4, Xt)[0], m0, rtol=1e-12, atol=1e-12)" - ] - }, - { - "cell_type": "markdown", - "id": "c004", - "metadata": {}, - "source": [ - "A copy is independent: updating it leaves the original model unchanged." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c005", - "metadata": {}, - "outputs": [], - "source": [ - "k = lk.Kriging(y[:30], X[:30], \"matern5_2\")\n", - "m_before = mean_sd(k, Xt)[0]\n", - "kc = k.copy() if hasattr(k, \"copy\") else copy.deepcopy(k)\n", - "kc.update(y[30:], X[30:], refit=False)\n", - "suite.close(\"update on a copy leaves the original unchanged\", mean_sd(k, Xt)[0], m_before, rtol=0, atol=0)\n", - "suite.true(\"the updated copy did change\", not np.allclose(mean_sd(kc, Xt)[0], m_before))" - ] - }, - { - "cell_type": "markdown", - "id": "c006", - "metadata": {}, - "source": [ - "## Accessors mirror the constructor" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c007", - "metadata": {}, - "outputs": [], - "source": [ - "k = lk.Kriging(y, X, \"matern3_2\", regmodel=\"linear\", normalize=False, optim=\"BFGS\", objective=\"LMP\")\n", - "for acc, expected in [(\"kernel\", \"matern3_2\"), (\"regmodel\", \"linear\"), (\"optim\", \"BFGS\"), (\"objective\", \"LMP\"),\n", - " (\"normalize\", False)]:\n", - " suite.true(f\"Kriging.{acc}() == {expected!r}\", getattr(k, acc)() == expected, detail=str(getattr(k, acc)()))\n", - "suite.close(\"Kriging.X() is the design\", np.asarray(k.X()), X, rtol=0, atol=0)\n", - "suite.close(\"Kriging.y() is the response\", np.ravel(k.y()), y, rtol=0, atol=0)\n", - "# with normalize=True: do the accessors return the data as given, or as rescaled internally?\n", - "kn = lk.Kriging(y, X * 10, \"matern3_2\", normalize=True)\n", - "suite.true(\"Kriging.normalize() == True\", kn.normalize() is True or kn.normalize() == 1)\n", - "suite.close(\"normalize=True: X() is the design as given\", np.asarray(kn.X()), X * 10, rtol=1e-12, atol=0,\n", - " severity=\"warn\")\n", - "suite.close(\"normalize=True: y() is the response as given\", np.ravel(kn.y()), y, rtol=1e-12, atol=0,\n", - " severity=\"warn\")\n", - "suite.true(\"Kriging.beta() has 1 + d coefficients (linear trend)\", np.ravel(k.beta()).size == 1 + X.shape[1])\n", - "suite.true(\"summary() is a non-empty string\", isinstance(k.summary(), str) and len(k.summary()) > 20)\n", - "suite.true(\"is_theta_estim() after an optimized fit\", bool(k.is_theta_estim()))" - ] - }, - { - "cell_type": "markdown", - "id": "c008", - "metadata": {}, - "source": [ - "## Installing scikit-learn\n", - "\n", - "Needed by the estimators of `pylibkriging.sklearn` below. With pip, into the\n", - "kernel's environment, numpy pinned to its installed version (a package that\n", - "would downgrade it is not installed: the libKriging extension is built\n", - "against it). A package that cannot be installed is skipped, and\n", - "reported." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c009", - "metadata": {}, - "outputs": [], - "source": [ - "import importlib\n", - "import subprocess\n", - "import sys\n", - "\n", - "import numpy\n", - "\n", - "for package in [\"scikit-learn\"]:\n", - " r = subprocess.run([sys.executable, \"-m\", \"pip\", \"install\", \"--quiet\", package, f\"numpy=={numpy.__version__}\"],\n", - " capture_output=True, text=True)\n", - " print(f\"{package}: \" + (\"installed\" if r.returncode == 0 else \"NOT installed\\n\" + r.stderr.strip()[-800:]))\n", - "importlib.invalidate_caches()" - ] - }, - { - "cell_type": "markdown", - "id": "c010", - "metadata": {}, - "source": [ - "## scikit-learn estimators" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c011", - "metadata": {}, - "outputs": [], - "source": [ - "if compat.has(\"sklearn\"):\n", - " from sklearn.base import clone\n", - " from sklearn.model_selection import GridSearchCV, cross_val_score\n", - " from sklearn.pipeline import make_pipeline\n", - " from sklearn.preprocessing import MinMaxScaler\n", - " from pylibkriging.sklearn import KrigingRegressor\n", - "\n", - " est = KrigingRegressor(kernel=\"matern5_2\").fit(X, y)\n", - " m_sk, s_sk = est.predict(Xt, return_std=True)\n", - " suite.true(\"KrigingRegressor.predict returns 1-D arrays (scikit-learn convention)\",\n", - " np.ndim(m_sk) == 1 and np.ndim(s_sk) == 1, detail=f\"shapes {np.shape(m_sk)}, {np.shape(s_sk)}\",\n", - " severity=\"warn\")\n", - " m_sk, s_sk = np.ravel(m_sk), np.ravel(s_sk)\n", - " m_nat, s_nat = mean_sd(lk.Kriging(y, X, \"matern5_2\"), Xt)\n", - " suite.close(\"KrigingRegressor == Kriging (mean)\", m_sk, m_nat, rtol=1e-8, atol=1e-10)\n", - " suite.close(\"KrigingRegressor == Kriging (stdev)\", s_sk, s_nat, rtol=1e-6, atol=1e-10)\n", - " est2 = clone(est).set_params(kernel=\"gauss\")\n", - " suite.true(\"clone + set_params\", est2.get_params()[\"kernel\"] == \"gauss\" and est.get_params()[\"kernel\"] ==\n", - " \"matern5_2\")\n", - " with suite.runs(\"Pipeline(MinMaxScaler, KrigingRegressor)\"):\n", - " pipe = make_pipeline(MinMaxScaler(), KrigingRegressor(kernel=\"matern5_2\")).fit(tf.scale(X), y)\n", - " suite.greater(\"pipeline Q2 on native inputs\", pipe.score(tf.scale(Xt), yt), 0.8)\n", - " with suite.runs(\"GridSearchCV over kernels\"):\n", - " gs = GridSearchCV(KrigingRegressor(), {\"kernel\": [\"gauss\", \"matern3_2\", \"matern5_2\"]}, cv=3).fit(X, y)\n", - " suite.true(\"GridSearchCV picked a kernel\", gs.best_params_[\"kernel\"] in (\"gauss\", \"matern3_2\", \"matern5_2\"),\n", - " detail=str(gs.best_params_))\n", - " with suite.runs(\"cross_val_score\"):\n", - " sc = cross_val_score(KrigingRegressor(kernel=\"matern5_2\"), X, y, cv=4)\n", - " suite.finite(\"cross_val_score finite\", sc)\n", - " with suite.runs(\"sample_y\"):\n", - " S = est.sample_y(Xt[:10], n_samples=5, random_state=1)\n", - " suite.true(\"sample_y shape (q, n_samples)\", np.shape(S) == (10, 5), detail=str(np.shape(S)))\n", - "else:\n", - " suite.skip(\"sklearn estimators\", \"pylibkriging.sklearn or scikit-learn not available\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c012", - "metadata": {}, - "outputs": [], - "source": [ - "suite.finalize()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/python_Kriging_robustness-invalid-inputs.ipynb b/robustness-invalid-inputs_Kriging_python.ipynb similarity index 99% rename from python_Kriging_robustness-invalid-inputs.ipynb rename to robustness-invalid-inputs_Kriging_python.ipynb index 2867784..b18b41d 100644 --- a/python_Kriging_robustness-invalid-inputs.ipynb +++ b/robustness-invalid-inputs_Kriging_python.ipynb @@ -40,7 +40,7 @@ "import lkbench\n", "from lkbench import TESTS, design, compat\n", "\n", - "suite = lkbench.Suite(\"python_Kriging_robustness-invalid-inputs\")\n", + "suite = lkbench.Suite(\"robustness-invalid-inputs_Kriging_python\")\n", "print(\"pylibkriging\", compat.version())\n", "tf = TESTS[\"branin\"]\n", "X, y, Xt, yt = design.dataset(tf, 20, n_test=200, seed=71)\n", diff --git a/save-load-pickle-copy-accessors_Kriging_python.ipynb b/save-load-pickle-copy-accessors_Kriging_python.ipynb new file mode 100644 index 0000000..3a6f3ae --- /dev/null +++ b/save-load-pickle-copy-accessors_Kriging_python.ipynb @@ -0,0 +1,171 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "c000", + "metadata": {}, + "source": [ + "# Python · Kriging — save/load, pickle, copy, accessors\n", + "\n", + "Persistence and introspection of `Kriging`, for each of its variants (noise\n", + "models, objectives, approximate fit methods):\n", + "\n", + "* `save` → `load` (class-specific and generic `pylibkriging.load`) gives a model\n", + " with **identical** predictions and hyperparameters;\n", + "* `pickle` / `copy` round trips, and a copy is independent of the original\n", + " (updating one leaves the other untouched);\n", + "* accessors mirror the constructor arguments (`kernel()`, `regmodel()`,\n", + " `optim()`, `objective()`, `X()`, `y()`, ...).\n", + "\n", + "The same persistence checks for the other classes are in their own notebooks\n", + "(`*_WarpKriging_python`, `*_MLPKriging_python`, `*_NestedKriging_python`); the\n", + "scikit-learn estimators in `sklearn-estimators_Kriging_python.ipynb`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c001", + "metadata": {}, + "outputs": [], + "source": [ + "import copy\n", + "\n", + "import numpy as np\n", + "import pylibkriging as lk\n", + "\n", + "import lkbench\n", + "from lkbench import TESTS, design, compat\n", + "\n", + "suite = lkbench.Suite(\"save-load-pickle-copy-accessors_Kriging_python\")\n", + "print(\"pylibkriging\", compat.version())\n", + "\n", + "tf = TESTS[\"hartmann3\"]\n", + "X, y, Xt, yt = design.dataset(tf, 40, n_test=100, seed=61)\n", + "\n", + "\n", + "def mean_sd(k, Xn):\n", + " out = k.predict(Xn, return_stdev=True)\n", + " return np.ravel(out[0]), np.ravel(out[1])\n", + "\n", + "\n", + "MODELS = {\n", + " \"Kriging\": lambda: lk.Kriging(y, X, \"matern5_2\", regmodel=\"linear\"),\n", + " \"Kriging nugget\": lambda: lk.Kriging(y, X, \"matern3_2\", noise=\"nugget\"),\n", + " \"Kriging noise vector\": lambda: lk.Kriging(y, X, \"gauss\", noise=np.full(len(y), 1e-4)),\n", + " \"Kriging LOO\": lambda: lk.Kriging(y, X, \"matern5_2\", objective=\"LOO\"),\n", + " \"Kriging LMP\": lambda: lk.Kriging(y, X, \"matern5_2\", objective=\"LMP\"),\n", + "}\n", + "if compat.has(\"method\"):\n", + " MODELS[\"Kriging vecchia\"] = lambda: lk.Kriging(y, X, \"matern5_2\", method=\"vecchia(10)\")\n", + " MODELS[\"Kriging nystrom\"] = lambda: lk.Kriging(y, X, \"matern5_2\", method=\"nystrom(20)\")\n", + "\n", + "\n", + "def predict_any(k, Xn):\n", + " meth = k.method() if hasattr(k, \"method\") and isinstance(k, lk.Kriging) else None\n", + " if meth and meth != \"exact\":\n", + " out = k.predict(Xn, return_stdev=True, method=meth.split(\"(\")[0])\n", + " return np.ravel(out[0]), np.ravel(out[1])\n", + " return mean_sd(k, Xn)" + ] + }, + { + "cell_type": "markdown", + "id": "c002", + "metadata": {}, + "source": [ + "## save / load, pickle, copy" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c003", + "metadata": {}, + "outputs": [], + "source": [ + "for name, make in MODELS.items():\n", + " with suite.runs(f\"{name}: fit\"):\n", + " lkbench.persistence.check(suite, name, make(), Xt, predict_any)" + ] + }, + { + "cell_type": "markdown", + "id": "c004", + "metadata": {}, + "source": [ + "A copy is independent: updating it leaves the original model unchanged." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c005", + "metadata": {}, + "outputs": [], + "source": [ + "k = lk.Kriging(y[:30], X[:30], \"matern5_2\")\n", + "m_before = mean_sd(k, Xt)[0]\n", + "kc = k.copy() if hasattr(k, \"copy\") else copy.deepcopy(k)\n", + "kc.update(y[30:], X[30:], refit=False)\n", + "suite.close(\"update on a copy leaves the original unchanged\", mean_sd(k, Xt)[0], m_before, rtol=0, atol=0)\n", + "suite.true(\"the updated copy did change\", not np.allclose(mean_sd(kc, Xt)[0], m_before))" + ] + }, + { + "cell_type": "markdown", + "id": "c006", + "metadata": {}, + "source": [ + "## Accessors mirror the constructor" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c007", + "metadata": {}, + "outputs": [], + "source": [ + "k = lk.Kriging(y, X, \"matern3_2\", regmodel=\"linear\", normalize=False, optim=\"BFGS\", objective=\"LMP\")\n", + "for acc, expected in [(\"kernel\", \"matern3_2\"), (\"regmodel\", \"linear\"), (\"optim\", \"BFGS\"), (\"objective\", \"LMP\"),\n", + " (\"normalize\", False)]:\n", + " suite.true(f\"Kriging.{acc}() == {expected!r}\", getattr(k, acc)() == expected, detail=str(getattr(k, acc)()))\n", + "suite.close(\"Kriging.X() is the design\", np.asarray(k.X()), X, rtol=0, atol=0)\n", + "suite.close(\"Kriging.y() is the response\", np.ravel(k.y()), y, rtol=0, atol=0)\n", + "# with normalize=True: do the accessors return the data as given, or as rescaled internally?\n", + "kn = lk.Kriging(y, X * 10, \"matern3_2\", normalize=True)\n", + "suite.true(\"Kriging.normalize() == True\", kn.normalize() is True or kn.normalize() == 1)\n", + "suite.close(\"normalize=True: X() is the design as given\", np.asarray(kn.X()), X * 10, rtol=1e-12, atol=0,\n", + " severity=\"warn\")\n", + "suite.close(\"normalize=True: y() is the response as given\", np.ravel(kn.y()), y, rtol=1e-12, atol=0,\n", + " severity=\"warn\")\n", + "suite.true(\"Kriging.beta() has 1 + d coefficients (linear trend)\", np.ravel(k.beta()).size == 1 + X.shape[1])\n", + "suite.true(\"summary() is a non-empty string\", isinstance(k.summary(), str) and len(k.summary()) > 20)\n", + "suite.true(\"is_theta_estim() after an optimized fit\", bool(k.is_theta_estim()))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c008", + "metadata": {}, + "outputs": [], + "source": [ + "suite.finalize()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/sklearn-estimators_Kriging_python.ipynb b/sklearn-estimators_Kriging_python.ipynb new file mode 100644 index 0000000..496b4fa --- /dev/null +++ b/sklearn-estimators_Kriging_python.ipynb @@ -0,0 +1,148 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "c000", + "metadata": {}, + "source": [ + "# Python · Kriging — scikit-learn estimators (`pylibkriging.sklearn`)\n", + "\n", + "`KrigingRegressor` (the scikit-learn face of `Kriging`) on Hartmann 3:\n", + "`fit`/`predict` (scikit-learn conventions), the same predictions as the native\n", + "class, `get_params`/`set_params`/`clone`, use inside `Pipeline`,\n", + "`GridSearchCV`, `cross_val_score`, and `sample_y`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c001", + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "import pylibkriging as lk\n", + "\n", + "import lkbench\n", + "from lkbench import TESTS, design, compat\n", + "\n", + "suite = lkbench.Suite(\"sklearn-estimators_Kriging_python\")\n", + "print(\"pylibkriging\", compat.version())\n", + "\n", + "tf = TESTS[\"hartmann3\"]\n", + "X, y, Xt, yt = design.dataset(tf, 40, n_test=100, seed=61)\n", + "\n", + "\n", + "def mean_sd(k, Xn):\n", + " out = k.predict(Xn, return_stdev=True)\n", + " return np.ravel(out[0]), np.ravel(out[1])" + ] + }, + { + "cell_type": "markdown", + "id": "c002", + "metadata": {}, + "source": [ + "## Installing scikit-learn\n", + "\n", + "With pip, into the kernel's environment, numpy pinned to its installed version\n", + "(a package that would downgrade it is not installed: the libKriging extension\n", + "is built against it). A package that cannot be installed is skipped, and\n", + "reported." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c003", + "metadata": {}, + "outputs": [], + "source": [ + "import importlib\n", + "import subprocess\n", + "import sys\n", + "\n", + "import numpy\n", + "\n", + "for package in [\"scikit-learn\"]:\n", + " r = subprocess.run([sys.executable, \"-m\", \"pip\", \"install\", \"--quiet\", package, f\"numpy=={numpy.__version__}\"],\n", + " capture_output=True, text=True)\n", + " print(f\"{package}: \" + (\"installed\" if r.returncode == 0 else \"NOT installed\\n\" + r.stderr.strip()[-800:]))\n", + "importlib.invalidate_caches()" + ] + }, + { + "cell_type": "markdown", + "id": "c004", + "metadata": {}, + "source": [ + "## scikit-learn estimators" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c005", + "metadata": {}, + "outputs": [], + "source": [ + "if compat.has(\"sklearn\"):\n", + " from sklearn.base import clone\n", + " from sklearn.model_selection import GridSearchCV, cross_val_score\n", + " from sklearn.pipeline import make_pipeline\n", + " from sklearn.preprocessing import MinMaxScaler\n", + " from pylibkriging.sklearn import KrigingRegressor\n", + "\n", + " est = KrigingRegressor(kernel=\"matern5_2\").fit(X, y)\n", + " m_sk, s_sk = est.predict(Xt, return_std=True)\n", + " suite.true(\"KrigingRegressor.predict returns 1-D arrays (scikit-learn convention)\",\n", + " np.ndim(m_sk) == 1 and np.ndim(s_sk) == 1, detail=f\"shapes {np.shape(m_sk)}, {np.shape(s_sk)}\",\n", + " severity=\"warn\")\n", + " m_sk, s_sk = np.ravel(m_sk), np.ravel(s_sk)\n", + " m_nat, s_nat = mean_sd(lk.Kriging(y, X, \"matern5_2\"), Xt)\n", + " suite.close(\"KrigingRegressor == Kriging (mean)\", m_sk, m_nat, rtol=1e-8, atol=1e-10)\n", + " suite.close(\"KrigingRegressor == Kriging (stdev)\", s_sk, s_nat, rtol=1e-6, atol=1e-10)\n", + " est2 = clone(est).set_params(kernel=\"gauss\")\n", + " suite.true(\"clone + set_params\", est2.get_params()[\"kernel\"] == \"gauss\" and est.get_params()[\"kernel\"] ==\n", + " \"matern5_2\")\n", + " with suite.runs(\"Pipeline(MinMaxScaler, KrigingRegressor)\"):\n", + " pipe = make_pipeline(MinMaxScaler(), KrigingRegressor(kernel=\"matern5_2\")).fit(tf.scale(X), y)\n", + " suite.greater(\"pipeline Q2 on native inputs\", pipe.score(tf.scale(Xt), yt), 0.8)\n", + " with suite.runs(\"GridSearchCV over kernels\"):\n", + " gs = GridSearchCV(KrigingRegressor(), {\"kernel\": [\"gauss\", \"matern3_2\", \"matern5_2\"]}, cv=3).fit(X, y)\n", + " suite.true(\"GridSearchCV picked a kernel\", gs.best_params_[\"kernel\"] in (\"gauss\", \"matern3_2\", \"matern5_2\"),\n", + " detail=str(gs.best_params_))\n", + " with suite.runs(\"cross_val_score\"):\n", + " sc = cross_val_score(KrigingRegressor(kernel=\"matern5_2\"), X, y, cv=4)\n", + " suite.finite(\"cross_val_score finite\", sc)\n", + " with suite.runs(\"sample_y\"):\n", + " S = est.sample_y(Xt[:10], n_samples=5, random_state=1)\n", + " suite.true(\"sample_y shape (q, n_samples)\", np.shape(S) == (10, 5), detail=str(np.shape(S)))\n", + "else:\n", + " suite.skip(\"sklearn estimators\", \"pylibkriging.sklearn or scikit-learn not available\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c006", + "metadata": {}, + "outputs": [], + "source": [ + "suite.finalize()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/python_no-model_test-functions-reference-values.ipynb b/test-functions-reference-values_no-model_python.ipynb similarity index 98% rename from python_no-model_test-functions-reference-values.ipynb rename to test-functions-reference-values_no-model_python.ipynb index ed2f7e9..9a2a0a8 100644 --- a/python_no-model_test-functions-reference-values.ipynb +++ b/test-functions-reference-values_no-model_python.ipynb @@ -28,7 +28,7 @@ "import lkbench\n", "from lkbench import TESTS, design\n", "\n", - "suite = lkbench.Suite(\"python_no-model_test-functions-reference-values\")" + "suite = lkbench.Suite(\"test-functions-reference-values_no-model_python\")" ] }, { diff --git a/tools/baseline/metrics-master.csv b/tools/baseline/metrics-master.csv index 88c120c..4f13d43 100644 --- a/tools/baseline/metrics-master.csv +++ b/tools/baseline/metrics-master.csv @@ -1,956 +1,956 @@ -suite,lang,case,package,metric,value -python_Kriging_kernels-trends-interpolation,python,forrester/gauss/constant,pylibkriging,fit_s,0.00161386 -python_Kriging_kernels-trends-interpolation,python,forrester/gauss/constant,pylibkriging,rmse,0.0848271 -python_Kriging_kernels-trends-interpolation,python,forrester/gauss/constant,pylibkriging,q2,0.999634 -python_Kriging_kernels-trends-interpolation,python,forrester/gauss/constant,pylibkriging,nlpd,-3.06486 -python_Kriging_kernels-trends-interpolation,python,forrester/gauss/linear,pylibkriging,fit_s,0.00126195 -python_Kriging_kernels-trends-interpolation,python,forrester/gauss/linear,pylibkriging,rmse,0.0855468 -python_Kriging_kernels-trends-interpolation,python,forrester/gauss/linear,pylibkriging,q2,0.999628 -python_Kriging_kernels-trends-interpolation,python,forrester/gauss/linear,pylibkriging,nlpd,-3.11029 -python_Kriging_kernels-trends-interpolation,python,forrester/gauss/interactive,pylibkriging,fit_s,0.00143647 -python_Kriging_kernels-trends-interpolation,python,forrester/gauss/interactive,pylibkriging,rmse,0.0855468 -python_Kriging_kernels-trends-interpolation,python,forrester/gauss/interactive,pylibkriging,q2,0.999628 -python_Kriging_kernels-trends-interpolation,python,forrester/gauss/interactive,pylibkriging,nlpd,-3.11029 -python_Kriging_kernels-trends-interpolation,python,forrester/gauss/quadratic,pylibkriging,fit_s,0.000566483 -python_Kriging_kernels-trends-interpolation,python,forrester/gauss/quadratic,pylibkriging,rmse,1.57249 -python_Kriging_kernels-trends-interpolation,python,forrester/gauss/quadratic,pylibkriging,q2,0.874394 -python_Kriging_kernels-trends-interpolation,python,forrester/gauss/quadratic,pylibkriging,nlpd,5.33879 -python_Kriging_kernels-trends-interpolation,python,forrester/exp/constant,pylibkriging,fit_s,0.00041604 -python_Kriging_kernels-trends-interpolation,python,forrester/exp/constant,pylibkriging,rmse,0.721582 -python_Kriging_kernels-trends-interpolation,python,forrester/exp/constant,pylibkriging,q2,0.973551 -python_Kriging_kernels-trends-interpolation,python,forrester/exp/constant,pylibkriging,nlpd,1.54551 -python_Kriging_kernels-trends-interpolation,python,forrester/exp/linear,pylibkriging,fit_s,0.000438213 -python_Kriging_kernels-trends-interpolation,python,forrester/exp/linear,pylibkriging,rmse,0.793413 -python_Kriging_kernels-trends-interpolation,python,forrester/exp/linear,pylibkriging,q2,0.968024 -python_Kriging_kernels-trends-interpolation,python,forrester/exp/linear,pylibkriging,nlpd,1.50639 -python_Kriging_kernels-trends-interpolation,python,forrester/exp/interactive,pylibkriging,fit_s,0.000480413 -python_Kriging_kernels-trends-interpolation,python,forrester/exp/interactive,pylibkriging,rmse,0.793413 -python_Kriging_kernels-trends-interpolation,python,forrester/exp/interactive,pylibkriging,q2,0.968024 -python_Kriging_kernels-trends-interpolation,python,forrester/exp/interactive,pylibkriging,nlpd,1.50639 -python_Kriging_kernels-trends-interpolation,python,forrester/exp/quadratic,pylibkriging,fit_s,0.000730038 -python_Kriging_kernels-trends-interpolation,python,forrester/exp/quadratic,pylibkriging,rmse,0.723267 -python_Kriging_kernels-trends-interpolation,python,forrester/exp/quadratic,pylibkriging,q2,0.973428 -python_Kriging_kernels-trends-interpolation,python,forrester/exp/quadratic,pylibkriging,nlpd,1.28782 -python_Kriging_kernels-trends-interpolation,python,forrester/matern3_2/constant,pylibkriging,fit_s,0.00060606 -python_Kriging_kernels-trends-interpolation,python,forrester/matern3_2/constant,pylibkriging,rmse,0.334007 -python_Kriging_kernels-trends-interpolation,python,forrester/matern3_2/constant,pylibkriging,q2,0.994333 -python_Kriging_kernels-trends-interpolation,python,forrester/matern3_2/constant,pylibkriging,nlpd,-0.227715 -python_Kriging_kernels-trends-interpolation,python,forrester/matern3_2/linear,pylibkriging,fit_s,0.000588894 -python_Kriging_kernels-trends-interpolation,python,forrester/matern3_2/linear,pylibkriging,rmse,0.364477 -python_Kriging_kernels-trends-interpolation,python,forrester/matern3_2/linear,pylibkriging,q2,0.993252 -python_Kriging_kernels-trends-interpolation,python,forrester/matern3_2/linear,pylibkriging,nlpd,-0.187262 -python_Kriging_kernels-trends-interpolation,python,forrester/matern3_2/interactive,pylibkriging,fit_s,0.000905514 -python_Kriging_kernels-trends-interpolation,python,forrester/matern3_2/interactive,pylibkriging,rmse,0.364477 -python_Kriging_kernels-trends-interpolation,python,forrester/matern3_2/interactive,pylibkriging,q2,0.993252 -python_Kriging_kernels-trends-interpolation,python,forrester/matern3_2/interactive,pylibkriging,nlpd,-0.187262 -python_Kriging_kernels-trends-interpolation,python,forrester/matern3_2/quadratic,pylibkriging,fit_s,0.000975609 -python_Kriging_kernels-trends-interpolation,python,forrester/matern3_2/quadratic,pylibkriging,rmse,0.435609 -python_Kriging_kernels-trends-interpolation,python,forrester/matern3_2/quadratic,pylibkriging,q2,0.990361 -python_Kriging_kernels-trends-interpolation,python,forrester/matern3_2/quadratic,pylibkriging,nlpd,-0.183076 -python_Kriging_kernels-trends-interpolation,python,forrester/matern5_2/constant,pylibkriging,fit_s,0.000726223 -python_Kriging_kernels-trends-interpolation,python,forrester/matern5_2/constant,pylibkriging,rmse,0.281945 -python_Kriging_kernels-trends-interpolation,python,forrester/matern5_2/constant,pylibkriging,q2,0.995962 -python_Kriging_kernels-trends-interpolation,python,forrester/matern5_2/constant,pylibkriging,nlpd,-1.01945 -python_Kriging_kernels-trends-interpolation,python,forrester/matern5_2/constant,pylibkriging BFGS10,rmse,0.281945 -python_Kriging_kernels-trends-interpolation,python,forrester/matern5_2/constant,pylibkriging BFGS10,q2,0.995962 -python_Kriging_kernels-trends-interpolation,python,forrester/matern5_2/constant,pylibkriging BFGS10,nlpd,-1.01945 -python_Kriging_kernels-trends-interpolation,python,forrester/matern5_2/linear,pylibkriging,fit_s,0.000932932 -python_Kriging_kernels-trends-interpolation,python,forrester/matern5_2/linear,pylibkriging,rmse,0.298959 -python_Kriging_kernels-trends-interpolation,python,forrester/matern5_2/linear,pylibkriging,q2,0.99546 -python_Kriging_kernels-trends-interpolation,python,forrester/matern5_2/linear,pylibkriging,nlpd,-0.95551 -python_Kriging_kernels-trends-interpolation,python,forrester/matern5_2/interactive,pylibkriging,fit_s,0.000725269 -python_Kriging_kernels-trends-interpolation,python,forrester/matern5_2/interactive,pylibkriging,rmse,0.298959 -python_Kriging_kernels-trends-interpolation,python,forrester/matern5_2/interactive,pylibkriging,q2,0.99546 -python_Kriging_kernels-trends-interpolation,python,forrester/matern5_2/interactive,pylibkriging,nlpd,-0.95551 -python_Kriging_kernels-trends-interpolation,python,forrester/matern5_2/quadratic,pylibkriging,fit_s,0.00131941 -python_Kriging_kernels-trends-interpolation,python,forrester/matern5_2/quadratic,pylibkriging,rmse,0.365039 -python_Kriging_kernels-trends-interpolation,python,forrester/matern5_2/quadratic,pylibkriging,q2,0.993231 -python_Kriging_kernels-trends-interpolation,python,forrester/matern5_2/quadratic,pylibkriging,nlpd,-0.790032 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/gauss/constant,pylibkriging,fit_s,0.0016458 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/gauss/constant,pylibkriging,rmse,0.191274 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/gauss/constant,pylibkriging,q2,0.978742 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/gauss/constant,pylibkriging,nlpd,-0.704948 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/gauss/linear,pylibkriging,fit_s,0.00319505 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/gauss/linear,pylibkriging,rmse,0.236028 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/gauss/linear,pylibkriging,q2,0.96763 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/gauss/linear,pylibkriging,nlpd,-0.580993 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/gauss/interactive,pylibkriging,fit_s,0.00250673 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/gauss/interactive,pylibkriging,rmse,0.236028 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/gauss/interactive,pylibkriging,q2,0.96763 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/gauss/interactive,pylibkriging,nlpd,-0.580993 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/gauss/quadratic,pylibkriging,fit_s,0.00243235 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/gauss/quadratic,pylibkriging,rmse,0.138869 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/gauss/quadratic,pylibkriging,q2,0.988795 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/gauss/quadratic,pylibkriging,nlpd,-0.530437 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/exp/constant,pylibkriging,fit_s,0.000752449 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/exp/constant,pylibkriging,rmse,0.196586 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/exp/constant,pylibkriging,q2,0.977545 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/exp/constant,pylibkriging,nlpd,-0.0621192 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/exp/linear,pylibkriging,fit_s,0.0010047 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/exp/linear,pylibkriging,rmse,0.20979 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/exp/linear,pylibkriging,q2,0.974427 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/exp/linear,pylibkriging,nlpd,-0.0466568 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/exp/interactive,pylibkriging,fit_s,0.000931501 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/exp/interactive,pylibkriging,rmse,0.20979 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/exp/interactive,pylibkriging,q2,0.974427 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/exp/interactive,pylibkriging,nlpd,-0.0466568 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/exp/quadratic,pylibkriging,fit_s,0.0025785 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/exp/quadratic,pylibkriging,rmse,0.220579 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/exp/quadratic,pylibkriging,q2,0.971729 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/exp/quadratic,pylibkriging,nlpd,0.136521 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern3_2/constant,pylibkriging,fit_s,0.000761509 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern3_2/constant,pylibkriging,rmse,0.177253 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern3_2/constant,pylibkriging,q2,0.981744 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern3_2/constant,pylibkriging,nlpd,-0.482212 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern3_2/linear,pylibkriging,fit_s,0.00132823 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern3_2/linear,pylibkriging,rmse,0.211499 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern3_2/linear,pylibkriging,q2,0.974009 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern3_2/linear,pylibkriging,nlpd,-0.375844 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern3_2/interactive,pylibkriging,fit_s,0.000978708 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern3_2/interactive,pylibkriging,rmse,0.211499 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern3_2/interactive,pylibkriging,q2,0.974009 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern3_2/interactive,pylibkriging,nlpd,-0.375844 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern3_2/quadratic,pylibkriging,fit_s,0.00228167 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern3_2/quadratic,pylibkriging,rmse,0.173012 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern3_2/quadratic,pylibkriging,q2,0.982607 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern3_2/quadratic,pylibkriging,nlpd,-0.223308 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern5_2/constant,pylibkriging,fit_s,0.00131416 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern5_2/constant,pylibkriging,rmse,0.177638 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern5_2/constant,pylibkriging,q2,0.981665 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern5_2/constant,pylibkriging,nlpd,-0.575421 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern5_2/constant,pylibkriging BFGS10,rmse,0.177638 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern5_2/constant,pylibkriging BFGS10,q2,0.981665 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern5_2/constant,pylibkriging BFGS10,nlpd,-0.575422 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern5_2/linear,pylibkriging,fit_s,0.00130415 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern5_2/linear,pylibkriging,rmse,0.219759 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern5_2/linear,pylibkriging,q2,0.971939 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern5_2/linear,pylibkriging,nlpd,-0.458933 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern5_2/interactive,pylibkriging,fit_s,0.00119758 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern5_2/interactive,pylibkriging,rmse,0.219759 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern5_2/interactive,pylibkriging,q2,0.971939 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern5_2/interactive,pylibkriging,nlpd,-0.458933 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern5_2/quadratic,pylibkriging,fit_s,0.00169826 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern5_2/quadratic,pylibkriging,rmse,0.160506 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern5_2/quadratic,pylibkriging,q2,0.985031 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern5_2/quadratic,pylibkriging,nlpd,-0.337276 -python_Kriging_kernels-trends-interpolation,python,branin/gauss/constant,pylibkriging,fit_s,0.00104284 -python_Kriging_kernels-trends-interpolation,python,branin/gauss/constant,pylibkriging,rmse,7.2629 -python_Kriging_kernels-trends-interpolation,python,branin/gauss/constant,pylibkriging,q2,0.980655 -python_Kriging_kernels-trends-interpolation,python,branin/gauss/constant,pylibkriging,nlpd,4.3152 -python_Kriging_kernels-trends-interpolation,python,branin/gauss/linear,pylibkriging,fit_s,0.00129008 -python_Kriging_kernels-trends-interpolation,python,branin/gauss/linear,pylibkriging,rmse,7.25733 -python_Kriging_kernels-trends-interpolation,python,branin/gauss/linear,pylibkriging,q2,0.980685 -python_Kriging_kernels-trends-interpolation,python,branin/gauss/linear,pylibkriging,nlpd,4.30813 -python_Kriging_kernels-trends-interpolation,python,branin/gauss/interactive,pylibkriging,fit_s,0.00280094 -python_Kriging_kernels-trends-interpolation,python,branin/gauss/interactive,pylibkriging,rmse,7.2576 -python_Kriging_kernels-trends-interpolation,python,branin/gauss/interactive,pylibkriging,q2,0.980683 -python_Kriging_kernels-trends-interpolation,python,branin/gauss/interactive,pylibkriging,nlpd,4.30176 -python_Kriging_kernels-trends-interpolation,python,branin/gauss/quadratic,pylibkriging,fit_s,0.00112319 -python_Kriging_kernels-trends-interpolation,python,branin/gauss/quadratic,pylibkriging,rmse,7.27808 -python_Kriging_kernels-trends-interpolation,python,branin/gauss/quadratic,pylibkriging,q2,0.980574 -python_Kriging_kernels-trends-interpolation,python,branin/gauss/quadratic,pylibkriging,nlpd,4.29399 -python_Kriging_kernels-trends-interpolation,python,branin/exp/constant,pylibkriging,fit_s,0.00222707 -python_Kriging_kernels-trends-interpolation,python,branin/exp/constant,pylibkriging,rmse,14.5887 -python_Kriging_kernels-trends-interpolation,python,branin/exp/constant,pylibkriging,q2,0.921947 -python_Kriging_kernels-trends-interpolation,python,branin/exp/constant,pylibkriging,nlpd,4.09457 -python_Kriging_kernels-trends-interpolation,python,branin/exp/linear,pylibkriging,fit_s,0.00213099 -python_Kriging_kernels-trends-interpolation,python,branin/exp/linear,pylibkriging,rmse,14.9622 -python_Kriging_kernels-trends-interpolation,python,branin/exp/linear,pylibkriging,q2,0.9179 -python_Kriging_kernels-trends-interpolation,python,branin/exp/linear,pylibkriging,nlpd,4.0909 -python_Kriging_kernels-trends-interpolation,python,branin/exp/interactive,pylibkriging,fit_s,0.00196409 -python_Kriging_kernels-trends-interpolation,python,branin/exp/interactive,pylibkriging,rmse,11.6963 -python_Kriging_kernels-trends-interpolation,python,branin/exp/interactive,pylibkriging,q2,0.94983 -python_Kriging_kernels-trends-interpolation,python,branin/exp/interactive,pylibkriging,nlpd,3.79671 -python_Kriging_kernels-trends-interpolation,python,branin/exp/quadratic,pylibkriging,fit_s,0.00227737 -python_Kriging_kernels-trends-interpolation,python,branin/exp/quadratic,pylibkriging,rmse,9.33909 -python_Kriging_kernels-trends-interpolation,python,branin/exp/quadratic,pylibkriging,q2,0.968014 -python_Kriging_kernels-trends-interpolation,python,branin/exp/quadratic,pylibkriging,nlpd,3.62362 -python_Kriging_kernels-trends-interpolation,python,branin/matern3_2/constant,pylibkriging,fit_s,0.00244212 -python_Kriging_kernels-trends-interpolation,python,branin/matern3_2/constant,pylibkriging,rmse,3.24796 -python_Kriging_kernels-trends-interpolation,python,branin/matern3_2/constant,pylibkriging,q2,0.996131 -python_Kriging_kernels-trends-interpolation,python,branin/matern3_2/constant,pylibkriging,nlpd,2.30444 -python_Kriging_kernels-trends-interpolation,python,branin/matern3_2/linear,pylibkriging,fit_s,0.00244641 -python_Kriging_kernels-trends-interpolation,python,branin/matern3_2/linear,pylibkriging,rmse,3.14402 -python_Kriging_kernels-trends-interpolation,python,branin/matern3_2/linear,pylibkriging,q2,0.996375 -python_Kriging_kernels-trends-interpolation,python,branin/matern3_2/linear,pylibkriging,nlpd,2.28067 -python_Kriging_kernels-trends-interpolation,python,branin/matern3_2/interactive,pylibkriging,fit_s,0.00703263 -python_Kriging_kernels-trends-interpolation,python,branin/matern3_2/interactive,pylibkriging,rmse,3.19226 -python_Kriging_kernels-trends-interpolation,python,branin/matern3_2/interactive,pylibkriging,q2,0.996263 -python_Kriging_kernels-trends-interpolation,python,branin/matern3_2/interactive,pylibkriging,nlpd,2.20359 -python_Kriging_kernels-trends-interpolation,python,branin/matern3_2/quadratic,pylibkriging,fit_s,0.00404716 -python_Kriging_kernels-trends-interpolation,python,branin/matern3_2/quadratic,pylibkriging,rmse,3.01232 -python_Kriging_kernels-trends-interpolation,python,branin/matern3_2/quadratic,pylibkriging,q2,0.996672 -python_Kriging_kernels-trends-interpolation,python,branin/matern3_2/quadratic,pylibkriging,nlpd,2.11627 -python_Kriging_kernels-trends-interpolation,python,branin/matern5_2/constant,pylibkriging,fit_s,0.0029068 -python_Kriging_kernels-trends-interpolation,python,branin/matern5_2/constant,pylibkriging,rmse,0.52012 -python_Kriging_kernels-trends-interpolation,python,branin/matern5_2/constant,pylibkriging,q2,0.999901 -python_Kriging_kernels-trends-interpolation,python,branin/matern5_2/constant,pylibkriging,nlpd,0.283399 -python_Kriging_kernels-trends-interpolation,python,branin/matern5_2/constant,pylibkriging BFGS10,rmse,0.52012 -python_Kriging_kernels-trends-interpolation,python,branin/matern5_2/constant,pylibkriging BFGS10,q2,0.999901 -python_Kriging_kernels-trends-interpolation,python,branin/matern5_2/constant,pylibkriging BFGS10,nlpd,0.283395 -python_Kriging_kernels-trends-interpolation,python,branin/matern5_2/linear,pylibkriging,fit_s,0.00212407 -python_Kriging_kernels-trends-interpolation,python,branin/matern5_2/linear,pylibkriging,rmse,0.460868 -python_Kriging_kernels-trends-interpolation,python,branin/matern5_2/linear,pylibkriging,q2,0.999922 -python_Kriging_kernels-trends-interpolation,python,branin/matern5_2/linear,pylibkriging,nlpd,0.209741 -python_Kriging_kernels-trends-interpolation,python,branin/matern5_2/interactive,pylibkriging,fit_s,0.0018971 -python_Kriging_kernels-trends-interpolation,python,branin/matern5_2/interactive,pylibkriging,rmse,0.498484 -python_Kriging_kernels-trends-interpolation,python,branin/matern5_2/interactive,pylibkriging,q2,0.999909 -python_Kriging_kernels-trends-interpolation,python,branin/matern5_2/interactive,pylibkriging,nlpd,0.239079 -python_Kriging_kernels-trends-interpolation,python,branin/matern5_2/quadratic,pylibkriging,fit_s,0.00570393 -python_Kriging_kernels-trends-interpolation,python,branin/matern5_2/quadratic,pylibkriging,rmse,0.67472 -python_Kriging_kernels-trends-interpolation,python,branin/matern5_2/quadratic,pylibkriging,q2,0.999833 -python_Kriging_kernels-trends-interpolation,python,branin/matern5_2/quadratic,pylibkriging,nlpd,0.419266 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/gauss/constant,pylibkriging,fit_s,0.00197244 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/gauss/constant,pylibkriging,rmse,18.9501 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/gauss/constant,pylibkriging,q2,0.536354 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/gauss/constant,pylibkriging,nlpd,84.151 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/gauss/linear,pylibkriging,fit_s,0.00170588 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/gauss/linear,pylibkriging,rmse,18.9579 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/gauss/linear,pylibkriging,q2,0.535976 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/gauss/linear,pylibkriging,nlpd,44.8696 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/gauss/interactive,pylibkriging,fit_s,0.0020082 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/gauss/interactive,pylibkriging,rmse,18.9819 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/gauss/interactive,pylibkriging,q2,0.5348 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/gauss/interactive,pylibkriging,nlpd,35.2765 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/gauss/quadratic,pylibkriging,fit_s,0.00248289 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/gauss/quadratic,pylibkriging,rmse,10.9058 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/gauss/quadratic,pylibkriging,q2,0.846441 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/gauss/quadratic,pylibkriging,nlpd,4.92017 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/exp/constant,pylibkriging,fit_s,0.00100708 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/exp/constant,pylibkriging,rmse,16.9653 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/exp/constant,pylibkriging,q2,0.628393 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/exp/constant,pylibkriging,nlpd,4.48595 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/exp/linear,pylibkriging,fit_s,0.00118208 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/exp/linear,pylibkriging,rmse,17.1781 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/exp/linear,pylibkriging,q2,0.619012 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/exp/linear,pylibkriging,nlpd,4.47061 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/exp/interactive,pylibkriging,fit_s,0.00123763 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/exp/interactive,pylibkriging,rmse,17.1856 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/exp/interactive,pylibkriging,q2,0.618679 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/exp/interactive,pylibkriging,nlpd,4.43133 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/exp/quadratic,pylibkriging,fit_s,0.0013876 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/exp/quadratic,pylibkriging,rmse,7.99484 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/exp/quadratic,pylibkriging,q2,0.917476 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/exp/quadratic,pylibkriging,nlpd,3.35612 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern3_2/constant,pylibkriging,fit_s,0.00175476 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern3_2/constant,pylibkriging,rmse,18.4958 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern3_2/constant,pylibkriging,q2,0.558321 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern3_2/constant,pylibkriging,nlpd,18.8349 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern3_2/linear,pylibkriging,fit_s,0.00206542 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern3_2/linear,pylibkriging,rmse,18.5562 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern3_2/linear,pylibkriging,q2,0.555433 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern3_2/linear,pylibkriging,nlpd,14.9494 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern3_2/interactive,pylibkriging,fit_s,0.00178266 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern3_2/interactive,pylibkriging,rmse,18.561 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern3_2/interactive,pylibkriging,q2,0.555202 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern3_2/interactive,pylibkriging,nlpd,12.9923 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern3_2/quadratic,pylibkriging,fit_s,0.00338221 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern3_2/quadratic,pylibkriging,rmse,8.91645 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern3_2/quadratic,pylibkriging,q2,0.897353 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern3_2/quadratic,pylibkriging,nlpd,3.94939 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern5_2/constant,pylibkriging,fit_s,0.00228882 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern5_2/constant,pylibkriging,rmse,18.3236 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern5_2/constant,pylibkriging,q2,0.566507 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern5_2/constant,pylibkriging,nlpd,18.4322 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern5_2/constant,pylibkriging BFGS10,rmse,1.36208 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern5_2/constant,pylibkriging BFGS10,q2,0.997605 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern5_2/constant,pylibkriging BFGS10,nlpd,1.03215 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern5_2/linear,pylibkriging,fit_s,0.00242186 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern5_2/linear,pylibkriging,rmse,18.7127 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern5_2/linear,pylibkriging,q2,0.547901 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern5_2/linear,pylibkriging,nlpd,24.5771 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern5_2/interactive,pylibkriging,fit_s,0.00234485 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern5_2/interactive,pylibkriging,rmse,18.7194 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern5_2/interactive,pylibkriging,q2,0.547577 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern5_2/interactive,pylibkriging,nlpd,20.4255 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern5_2/quadratic,pylibkriging,fit_s,0.00251341 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern5_2/quadratic,pylibkriging,rmse,10.2878 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern5_2/quadratic,pylibkriging,q2,0.863351 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern5_2/quadratic,pylibkriging,nlpd,4.46344 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/gauss/constant,pylibkriging,fit_s,0.00588727 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/gauss/constant,pylibkriging,rmse,0.295934 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/gauss/constant,pylibkriging,q2,0.920537 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/gauss/constant,pylibkriging,nlpd,1.58666 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/gauss/linear,pylibkriging,fit_s,0.00581551 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/gauss/linear,pylibkriging,rmse,0.296355 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/gauss/linear,pylibkriging,q2,0.92031 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/gauss/linear,pylibkriging,nlpd,1.2962 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/gauss/interactive,pylibkriging,fit_s,0.00644922 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/gauss/interactive,pylibkriging,rmse,0.285136 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/gauss/interactive,pylibkriging,q2,0.926229 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/gauss/interactive,pylibkriging,nlpd,1.04122 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/gauss/quadratic,pylibkriging,fit_s,0.00578475 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/gauss/quadratic,pylibkriging,rmse,0.237308 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/gauss/quadratic,pylibkriging,q2,0.948902 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/gauss/quadratic,pylibkriging,nlpd,0.146484 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/exp/constant,pylibkriging,fit_s,0.00258231 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/exp/constant,pylibkriging,rmse,0.334876 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/exp/constant,pylibkriging,q2,0.898247 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/exp/constant,pylibkriging,nlpd,0.339235 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/exp/linear,pylibkriging,fit_s,0.00211 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/exp/linear,pylibkriging,rmse,0.333447 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/exp/linear,pylibkriging,q2,0.899114 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/exp/linear,pylibkriging,nlpd,0.336515 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/exp/interactive,pylibkriging,fit_s,0.0025239 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/exp/interactive,pylibkriging,rmse,0.336819 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/exp/interactive,pylibkriging,q2,0.897063 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/exp/interactive,pylibkriging,nlpd,0.329566 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/exp/quadratic,pylibkriging,fit_s,0.00309515 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/exp/quadratic,pylibkriging,rmse,0.327816 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/exp/quadratic,pylibkriging,q2,0.902492 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/exp/quadratic,pylibkriging,nlpd,0.322056 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern3_2/constant,pylibkriging,fit_s,0.00425816 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern3_2/constant,pylibkriging,rmse,0.684362 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern3_2/constant,pylibkriging,q2,0.575038 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern3_2/constant,pylibkriging,nlpd,0.972803 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern3_2/linear,pylibkriging,fit_s,0.00413442 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern3_2/linear,pylibkriging,rmse,0.650201 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern3_2/linear,pylibkriging,q2,0.616404 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern3_2/linear,pylibkriging,nlpd,0.921455 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern3_2/interactive,pylibkriging,fit_s,0.00704145 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern3_2/interactive,pylibkriging,rmse,0.274326 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern3_2/interactive,pylibkriging,q2,0.931717 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern3_2/interactive,pylibkriging,nlpd,0.0549679 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern3_2/quadratic,pylibkriging,fit_s,0.00996184 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern3_2/quadratic,pylibkriging,rmse,0.263745 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern3_2/quadratic,pylibkriging,q2,0.936883 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern3_2/quadratic,pylibkriging,nlpd,0.0446267 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern5_2/constant,pylibkriging,fit_s,0.00410438 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern5_2/constant,pylibkriging,rmse,0.674391 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern5_2/constant,pylibkriging,q2,0.58733 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern5_2/constant,pylibkriging,nlpd,0.944765 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern5_2/constant,pylibkriging BFGS10,rmse,0.268216 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern5_2/constant,pylibkriging BFGS10,q2,0.934725 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern5_2/constant,pylibkriging BFGS10,nlpd,0.169585 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern5_2/linear,pylibkriging,fit_s,0.00728607 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern5_2/linear,pylibkriging,rmse,0.268808 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern5_2/linear,pylibkriging,q2,0.934436 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern5_2/linear,pylibkriging,nlpd,0.142382 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern5_2/interactive,pylibkriging,fit_s,0.00524259 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern5_2/interactive,pylibkriging,rmse,0.271088 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern5_2/interactive,pylibkriging,q2,0.933319 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern5_2/interactive,pylibkriging,nlpd,0.128788 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern5_2/quadratic,pylibkriging,fit_s,0.011641 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern5_2/quadratic,pylibkriging,rmse,0.253093 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern5_2/quadratic,pylibkriging,q2,0.941878 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern5_2/quadratic,pylibkriging,nlpd,0.025469 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/gauss/constant,pylibkriging,fit_s,0.00113583 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/gauss/constant,pylibkriging,rmse,0.026127 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/gauss/constant,pylibkriging,q2,0.999737 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/gauss/constant,pylibkriging,nlpd,-2.69817 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/gauss/linear,pylibkriging,fit_s,0.00152946 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/gauss/linear,pylibkriging,rmse,1.06243 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/gauss/linear,pylibkriging,q2,0.564691 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/gauss/linear,pylibkriging,nlpd,2.7394 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/gauss/interactive,pylibkriging,fit_s,0.00153446 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/gauss/interactive,pylibkriging,rmse,1.06504 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/gauss/interactive,pylibkriging,q2,0.562551 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/gauss/interactive,pylibkriging,nlpd,2.33575 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/gauss/quadratic,pylibkriging,fit_s,0.00196862 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/gauss/quadratic,pylibkriging,rmse,0.281816 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/gauss/quadratic,pylibkriging,q2,0.969371 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/gauss/quadratic,pylibkriging,nlpd,0.71682 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/exp/constant,pylibkriging,fit_s,0.000828028 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/exp/constant,pylibkriging,rmse,0.249023 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/exp/constant,pylibkriging,q2,0.976085 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/exp/constant,pylibkriging,nlpd,0.394159 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/exp/linear,pylibkriging,fit_s,0.000962973 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/exp/linear,pylibkriging,rmse,0.245558 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/exp/linear,pylibkriging,q2,0.976746 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/exp/linear,pylibkriging,nlpd,0.224593 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/exp/interactive,pylibkriging,fit_s,0.00143671 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/exp/interactive,pylibkriging,rmse,0.239032 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/exp/interactive,pylibkriging,q2,0.977965 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/exp/interactive,pylibkriging,nlpd,0.226133 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/exp/quadratic,pylibkriging,fit_s,0.00274634 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/exp/quadratic,pylibkriging,rmse,0.237713 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/exp/quadratic,pylibkriging,q2,0.978208 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/exp/quadratic,pylibkriging,nlpd,-0.139582 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern3_2/constant,pylibkriging,fit_s,0.0011251 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern3_2/constant,pylibkriging,rmse,0.0600911 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern3_2/constant,pylibkriging,q2,0.998607 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern3_2/constant,pylibkriging,nlpd,-1.34658 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern3_2/linear,pylibkriging,fit_s,0.000877857 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern3_2/linear,pylibkriging,rmse,0.0584376 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern3_2/linear,pylibkriging,q2,0.998683 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern3_2/linear,pylibkriging,nlpd,-1.34912 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern3_2/interactive,pylibkriging,fit_s,0.00140738 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern3_2/interactive,pylibkriging,rmse,0.0576096 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern3_2/interactive,pylibkriging,q2,0.99872 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern3_2/interactive,pylibkriging,nlpd,-1.35376 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern3_2/quadratic,pylibkriging,fit_s,0.00383472 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern3_2/quadratic,pylibkriging,rmse,0.215336 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern3_2/quadratic,pylibkriging,q2,0.982117 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern3_2/quadratic,pylibkriging,nlpd,-0.350837 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern5_2/constant,pylibkriging,fit_s,0.000718832 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern5_2/constant,pylibkriging,rmse,0.0246348 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern5_2/constant,pylibkriging,q2,0.999766 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern5_2/constant,pylibkriging,nlpd,-2.59598 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern5_2/constant,pylibkriging BFGS10,rmse,0.0246338 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern5_2/constant,pylibkriging BFGS10,q2,0.999766 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern5_2/constant,pylibkriging BFGS10,nlpd,-2.59598 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern5_2/linear,pylibkriging,fit_s,0.0013113 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern5_2/linear,pylibkriging,rmse,0.0232483 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern5_2/linear,pylibkriging,q2,0.999792 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern5_2/linear,pylibkriging,nlpd,-2.58843 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern5_2/interactive,pylibkriging,fit_s,0.00150442 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern5_2/interactive,pylibkriging,rmse,0.0230248 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern5_2/interactive,pylibkriging,q2,0.999796 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern5_2/interactive,pylibkriging,nlpd,-2.58839 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern5_2/quadratic,pylibkriging,fit_s,0.00300908 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern5_2/quadratic,pylibkriging,rmse,0.209802 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern5_2/quadratic,pylibkriging,q2,0.983025 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern5_2/quadratic,pylibkriging,nlpd,-0.403998 -python_Kriging_kernels-trends-interpolation,python,ishigami/gauss/constant,pylibkriging,fit_s,0.0101857 -python_Kriging_kernels-trends-interpolation,python,ishigami/gauss/constant,pylibkriging,rmse,2.07486 -python_Kriging_kernels-trends-interpolation,python,ishigami/gauss/constant,pylibkriging,q2,0.668191 -python_Kriging_kernels-trends-interpolation,python,ishigami/gauss/constant,pylibkriging,nlpd,2.80326 -python_Kriging_kernels-trends-interpolation,python,ishigami/gauss/linear,pylibkriging,fit_s,0.0193923 -python_Kriging_kernels-trends-interpolation,python,ishigami/gauss/linear,pylibkriging,rmse,2.8459 -python_Kriging_kernels-trends-interpolation,python,ishigami/gauss/linear,pylibkriging,q2,0.375762 -python_Kriging_kernels-trends-interpolation,python,ishigami/gauss/linear,pylibkriging,nlpd,3.41919 -python_Kriging_kernels-trends-interpolation,python,ishigami/gauss/interactive,pylibkriging,fit_s,0.0174267 -python_Kriging_kernels-trends-interpolation,python,ishigami/gauss/interactive,pylibkriging,rmse,2.85366 -python_Kriging_kernels-trends-interpolation,python,ishigami/gauss/interactive,pylibkriging,q2,0.372353 -python_Kriging_kernels-trends-interpolation,python,ishigami/gauss/interactive,pylibkriging,nlpd,3.28643 -python_Kriging_kernels-trends-interpolation,python,ishigami/gauss/quadratic,pylibkriging,fit_s,0.00865173 -python_Kriging_kernels-trends-interpolation,python,ishigami/gauss/quadratic,pylibkriging,rmse,2.08981 -python_Kriging_kernels-trends-interpolation,python,ishigami/gauss/quadratic,pylibkriging,q2,0.663392 -python_Kriging_kernels-trends-interpolation,python,ishigami/gauss/quadratic,pylibkriging,nlpd,2.80097 -python_Kriging_kernels-trends-interpolation,python,ishigami/exp/constant,pylibkriging,fit_s,0.00899458 -python_Kriging_kernels-trends-interpolation,python,ishigami/exp/constant,pylibkriging,rmse,1.41632 -python_Kriging_kernels-trends-interpolation,python,ishigami/exp/constant,pylibkriging,q2,0.845391 -python_Kriging_kernels-trends-interpolation,python,ishigami/exp/constant,pylibkriging,nlpd,1.75239 -python_Kriging_kernels-trends-interpolation,python,ishigami/exp/linear,pylibkriging,fit_s,0.0115979 -python_Kriging_kernels-trends-interpolation,python,ishigami/exp/linear,pylibkriging,rmse,1.4095 -python_Kriging_kernels-trends-interpolation,python,ishigami/exp/linear,pylibkriging,q2,0.846877 -python_Kriging_kernels-trends-interpolation,python,ishigami/exp/linear,pylibkriging,nlpd,1.74779 -python_Kriging_kernels-trends-interpolation,python,ishigami/exp/interactive,pylibkriging,fit_s,0.00745058 -python_Kriging_kernels-trends-interpolation,python,ishigami/exp/interactive,pylibkriging,rmse,1.41511 -python_Kriging_kernels-trends-interpolation,python,ishigami/exp/interactive,pylibkriging,q2,0.845657 -python_Kriging_kernels-trends-interpolation,python,ishigami/exp/interactive,pylibkriging,nlpd,1.75024 -python_Kriging_kernels-trends-interpolation,python,ishigami/exp/quadratic,pylibkriging,fit_s,0.00843048 -python_Kriging_kernels-trends-interpolation,python,ishigami/exp/quadratic,pylibkriging,rmse,1.40329 -python_Kriging_kernels-trends-interpolation,python,ishigami/exp/quadratic,pylibkriging,q2,0.848223 -python_Kriging_kernels-trends-interpolation,python,ishigami/exp/quadratic,pylibkriging,nlpd,1.73254 -python_Kriging_kernels-trends-interpolation,python,ishigami/matern3_2/constant,pylibkriging,fit_s,0.0480943 -python_Kriging_kernels-trends-interpolation,python,ishigami/matern3_2/constant,pylibkriging,rmse,1.32971 -python_Kriging_kernels-trends-interpolation,python,ishigami/matern3_2/constant,pylibkriging,q2,0.863723 -python_Kriging_kernels-trends-interpolation,python,ishigami/matern3_2/constant,pylibkriging,nlpd,1.50265 -python_Kriging_kernels-trends-interpolation,python,ishigami/matern3_2/linear,pylibkriging,fit_s,0.047653 -python_Kriging_kernels-trends-interpolation,python,ishigami/matern3_2/linear,pylibkriging,rmse,1.30244 -python_Kriging_kernels-trends-interpolation,python,ishigami/matern3_2/linear,pylibkriging,q2,0.869255 -python_Kriging_kernels-trends-interpolation,python,ishigami/matern3_2/linear,pylibkriging,nlpd,1.47906 -python_Kriging_kernels-trends-interpolation,python,ishigami/matern3_2/interactive,pylibkriging,fit_s,0.060349 -python_Kriging_kernels-trends-interpolation,python,ishigami/matern3_2/interactive,pylibkriging,rmse,1.30591 -python_Kriging_kernels-trends-interpolation,python,ishigami/matern3_2/interactive,pylibkriging,q2,0.868558 -python_Kriging_kernels-trends-interpolation,python,ishigami/matern3_2/interactive,pylibkriging,nlpd,1.47384 -python_Kriging_kernels-trends-interpolation,python,ishigami/matern3_2/quadratic,pylibkriging,fit_s,0.0471144 -python_Kriging_kernels-trends-interpolation,python,ishigami/matern3_2/quadratic,pylibkriging,rmse,1.32197 -python_Kriging_kernels-trends-interpolation,python,ishigami/matern3_2/quadratic,pylibkriging,q2,0.865305 -python_Kriging_kernels-trends-interpolation,python,ishigami/matern3_2/quadratic,pylibkriging,nlpd,1.46827 -python_Kriging_kernels-trends-interpolation,python,ishigami/matern5_2/constant,pylibkriging,fit_s,0.0145204 -python_Kriging_kernels-trends-interpolation,python,ishigami/matern5_2/constant,pylibkriging,rmse,3.00566 -python_Kriging_kernels-trends-interpolation,python,ishigami/matern5_2/constant,pylibkriging,q2,0.303709 -python_Kriging_kernels-trends-interpolation,python,ishigami/matern5_2/constant,pylibkriging,nlpd,3.42085 -python_Kriging_kernels-trends-interpolation,python,ishigami/matern5_2/constant,pylibkriging BFGS10,rmse,1.33903 -python_Kriging_kernels-trends-interpolation,python,ishigami/matern5_2/constant,pylibkriging BFGS10,q2,0.861806 -python_Kriging_kernels-trends-interpolation,python,ishigami/matern5_2/constant,pylibkriging BFGS10,nlpd,1.66132 -python_Kriging_kernels-trends-interpolation,python,ishigami/matern5_2/linear,pylibkriging,fit_s,0.0518982 -python_Kriging_kernels-trends-interpolation,python,ishigami/matern5_2/linear,pylibkriging,rmse,1.3434 -python_Kriging_kernels-trends-interpolation,python,ishigami/matern5_2/linear,pylibkriging,q2,0.860901 -python_Kriging_kernels-trends-interpolation,python,ishigami/matern5_2/linear,pylibkriging,nlpd,1.65103 -python_Kriging_kernels-trends-interpolation,python,ishigami/matern5_2/interactive,pylibkriging,fit_s,0.0539804 -python_Kriging_kernels-trends-interpolation,python,ishigami/matern5_2/interactive,pylibkriging,rmse,1.34479 -python_Kriging_kernels-trends-interpolation,python,ishigami/matern5_2/interactive,pylibkriging,q2,0.860615 -python_Kriging_kernels-trends-interpolation,python,ishigami/matern5_2/interactive,pylibkriging,nlpd,1.64153 -python_Kriging_kernels-trends-interpolation,python,ishigami/matern5_2/quadratic,pylibkriging,fit_s,0.052948 -python_Kriging_kernels-trends-interpolation,python,ishigami/matern5_2/quadratic,pylibkriging,rmse,1.37082 -python_Kriging_kernels-trends-interpolation,python,ishigami/matern5_2/quadratic,pylibkriging,q2,0.855166 -python_Kriging_kernels-trends-interpolation,python,ishigami/matern5_2/quadratic,pylibkriging,nlpd,1.62228 -python_Kriging_kernels-trends-interpolation,python,hartmann3/gauss/constant,pylibkriging,fit_s,0.00805306 -python_Kriging_kernels-trends-interpolation,python,hartmann3/gauss/constant,pylibkriging,rmse,0.156226 -python_Kriging_kernels-trends-interpolation,python,hartmann3/gauss/constant,pylibkriging,q2,0.971116 -python_Kriging_kernels-trends-interpolation,python,hartmann3/gauss/constant,pylibkriging,nlpd,-0.555623 -python_Kriging_kernels-trends-interpolation,python,hartmann3/gauss/linear,pylibkriging,fit_s,0.0129044 -python_Kriging_kernels-trends-interpolation,python,hartmann3/gauss/linear,pylibkriging,rmse,0.153297 -python_Kriging_kernels-trends-interpolation,python,hartmann3/gauss/linear,pylibkriging,q2,0.972189 -python_Kriging_kernels-trends-interpolation,python,hartmann3/gauss/linear,pylibkriging,nlpd,-0.568404 -python_Kriging_kernels-trends-interpolation,python,hartmann3/gauss/interactive,pylibkriging,fit_s,0.00773382 -python_Kriging_kernels-trends-interpolation,python,hartmann3/gauss/interactive,pylibkriging,rmse,0.712948 -python_Kriging_kernels-trends-interpolation,python,hartmann3/gauss/interactive,pylibkriging,q2,0.398455 -python_Kriging_kernels-trends-interpolation,python,hartmann3/gauss/interactive,pylibkriging,nlpd,1.74554 -python_Kriging_kernels-trends-interpolation,python,hartmann3/gauss/quadratic,pylibkriging,fit_s,0.00593877 -python_Kriging_kernels-trends-interpolation,python,hartmann3/gauss/quadratic,pylibkriging,rmse,0.73238 -python_Kriging_kernels-trends-interpolation,python,hartmann3/gauss/quadratic,pylibkriging,q2,0.365217 -python_Kriging_kernels-trends-interpolation,python,hartmann3/gauss/quadratic,pylibkriging,nlpd,2.14587 -python_Kriging_kernels-trends-interpolation,python,hartmann3/exp/constant,pylibkriging,fit_s,0.00374722 -python_Kriging_kernels-trends-interpolation,python,hartmann3/exp/constant,pylibkriging,rmse,0.347348 -python_Kriging_kernels-trends-interpolation,python,hartmann3/exp/constant,pylibkriging,q2,0.857215 -python_Kriging_kernels-trends-interpolation,python,hartmann3/exp/constant,pylibkriging,nlpd,0.352683 -python_Kriging_kernels-trends-interpolation,python,hartmann3/exp/linear,pylibkriging,fit_s,0.00216174 -python_Kriging_kernels-trends-interpolation,python,hartmann3/exp/linear,pylibkriging,rmse,0.347273 -python_Kriging_kernels-trends-interpolation,python,hartmann3/exp/linear,pylibkriging,q2,0.857276 -python_Kriging_kernels-trends-interpolation,python,hartmann3/exp/linear,pylibkriging,nlpd,0.343631 -python_Kriging_kernels-trends-interpolation,python,hartmann3/exp/interactive,pylibkriging,fit_s,0.0024302 -python_Kriging_kernels-trends-interpolation,python,hartmann3/exp/interactive,pylibkriging,rmse,0.378349 -python_Kriging_kernels-trends-interpolation,python,hartmann3/exp/interactive,pylibkriging,q2,0.83059 -python_Kriging_kernels-trends-interpolation,python,hartmann3/exp/interactive,pylibkriging,nlpd,0.385842 -python_Kriging_kernels-trends-interpolation,python,hartmann3/exp/quadratic,pylibkriging,fit_s,0.0039556 -python_Kriging_kernels-trends-interpolation,python,hartmann3/exp/quadratic,pylibkriging,rmse,0.355119 -python_Kriging_kernels-trends-interpolation,python,hartmann3/exp/quadratic,pylibkriging,q2,0.850755 -python_Kriging_kernels-trends-interpolation,python,hartmann3/exp/quadratic,pylibkriging,nlpd,0.334371 -python_Kriging_kernels-trends-interpolation,python,hartmann3/matern3_2/constant,pylibkriging,fit_s,0.0110524 -python_Kriging_kernels-trends-interpolation,python,hartmann3/matern3_2/constant,pylibkriging,rmse,0.227521 -python_Kriging_kernels-trends-interpolation,python,hartmann3/matern3_2/constant,pylibkriging,q2,0.938737 -python_Kriging_kernels-trends-interpolation,python,hartmann3/matern3_2/constant,pylibkriging,nlpd,-0.181207 -python_Kriging_kernels-trends-interpolation,python,hartmann3/matern3_2/linear,pylibkriging,fit_s,0.00763154 -python_Kriging_kernels-trends-interpolation,python,hartmann3/matern3_2/linear,pylibkriging,rmse,0.23038 -python_Kriging_kernels-trends-interpolation,python,hartmann3/matern3_2/linear,pylibkriging,q2,0.937188 -python_Kriging_kernels-trends-interpolation,python,hartmann3/matern3_2/linear,pylibkriging,nlpd,-0.162203 -python_Kriging_kernels-trends-interpolation,python,hartmann3/matern3_2/interactive,pylibkriging,fit_s,0.00412679 -python_Kriging_kernels-trends-interpolation,python,hartmann3/matern3_2/interactive,pylibkriging,rmse,0.315752 -python_Kriging_kernels-trends-interpolation,python,hartmann3/matern3_2/interactive,pylibkriging,q2,0.88201 -python_Kriging_kernels-trends-interpolation,python,hartmann3/matern3_2/interactive,pylibkriging,nlpd,0.898253 -python_Kriging_kernels-trends-interpolation,python,hartmann3/matern3_2/quadratic,pylibkriging,fit_s,0.00620675 -python_Kriging_kernels-trends-interpolation,python,hartmann3/matern3_2/quadratic,pylibkriging,rmse,0.291393 -python_Kriging_kernels-trends-interpolation,python,hartmann3/matern3_2/quadratic,pylibkriging,q2,0.899513 -python_Kriging_kernels-trends-interpolation,python,hartmann3/matern3_2/quadratic,pylibkriging,nlpd,0.53743 -python_Kriging_kernels-trends-interpolation,python,hartmann3/matern5_2/constant,pylibkriging,fit_s,0.00997186 -python_Kriging_kernels-trends-interpolation,python,hartmann3/matern5_2/constant,pylibkriging,rmse,0.190287 -python_Kriging_kernels-trends-interpolation,python,hartmann3/matern5_2/constant,pylibkriging,q2,0.957148 -python_Kriging_kernels-trends-interpolation,python,hartmann3/matern5_2/constant,pylibkriging,nlpd,-0.403054 -python_Kriging_kernels-trends-interpolation,python,hartmann3/matern5_2/constant,pylibkriging BFGS10,rmse,0.190286 -python_Kriging_kernels-trends-interpolation,python,hartmann3/matern5_2/constant,pylibkriging BFGS10,q2,0.957148 -python_Kriging_kernels-trends-interpolation,python,hartmann3/matern5_2/constant,pylibkriging BFGS10,nlpd,-0.40306 -python_Kriging_kernels-trends-interpolation,python,hartmann3/matern5_2/linear,pylibkriging,fit_s,0.0127335 -python_Kriging_kernels-trends-interpolation,python,hartmann3/matern5_2/linear,pylibkriging,rmse,0.189633 -python_Kriging_kernels-trends-interpolation,python,hartmann3/matern5_2/linear,pylibkriging,q2,0.957442 -python_Kriging_kernels-trends-interpolation,python,hartmann3/matern5_2/linear,pylibkriging,nlpd,-0.410075 -python_Kriging_kernels-trends-interpolation,python,hartmann3/matern5_2/interactive,pylibkriging,fit_s,0.0101018 -python_Kriging_kernels-trends-interpolation,python,hartmann3/matern5_2/interactive,pylibkriging,rmse,0.307114 -python_Kriging_kernels-trends-interpolation,python,hartmann3/matern5_2/interactive,pylibkriging,q2,0.888377 -python_Kriging_kernels-trends-interpolation,python,hartmann3/matern5_2/interactive,pylibkriging,nlpd,1.50063 -python_Kriging_kernels-trends-interpolation,python,hartmann3/matern5_2/quadratic,pylibkriging,fit_s,0.0101326 -python_Kriging_kernels-trends-interpolation,python,hartmann3/matern5_2/quadratic,pylibkriging,rmse,0.285355 -python_Kriging_kernels-trends-interpolation,python,hartmann3/matern5_2/quadratic,pylibkriging,q2,0.903634 -python_Kriging_kernels-trends-interpolation,python,hartmann3/matern5_2/quadratic,pylibkriging,nlpd,0.898128 -python_Kriging_objectives-LL-LOO-LMP-gradients,python,branin/LL,pylibkriging,rmse,0.833715 -python_Kriging_objectives-LL-LOO-LMP-gradients,python,branin/LL,pylibkriging,q2,0.999729 -python_Kriging_objectives-LL-LOO-LMP-gradients,python,branin/LL,pylibkriging,nlpd,0.697034 -python_Kriging_objectives-LL-LOO-LMP-gradients,python,branin/LOO,pylibkriging,rmse,0.550995 -python_Kriging_objectives-LL-LOO-LMP-gradients,python,branin/LOO,pylibkriging,q2,0.999882 -python_Kriging_objectives-LL-LOO-LMP-gradients,python,branin/LOO,pylibkriging,nlpd,0.471613 -python_Kriging_objectives-LL-LOO-LMP-gradients,python,branin/LMP,pylibkriging,rmse,0.66967 -python_Kriging_objectives-LL-LOO-LMP-gradients,python,branin/LMP,pylibkriging,q2,0.999825 -python_Kriging_objectives-LL-LOO-LMP-gradients,python,branin/LMP,pylibkriging,nlpd,0.617527 -python_Kriging_objectives-LL-LOO-LMP-gradients,python,hartmann3/LL,pylibkriging,rmse,0.157276 -python_Kriging_objectives-LL-LOO-LMP-gradients,python,hartmann3/LL,pylibkriging,q2,0.970945 -python_Kriging_objectives-LL-LOO-LMP-gradients,python,hartmann3/LL,pylibkriging,nlpd,-0.497548 -python_Kriging_objectives-LL-LOO-LMP-gradients,python,hartmann3/LOO,pylibkriging,rmse,0.144982 -python_Kriging_objectives-LL-LOO-LMP-gradients,python,hartmann3/LOO,pylibkriging,q2,0.97531 -python_Kriging_objectives-LL-LOO-LMP-gradients,python,hartmann3/LOO,pylibkriging,nlpd,-0.584885 -python_Kriging_objectives-LL-LOO-LMP-gradients,python,hartmann3/LMP,pylibkriging,rmse,0.150075 -python_Kriging_objectives-LL-LOO-LMP-gradients,python,hartmann3/LMP,pylibkriging,q2,0.973544 -python_Kriging_objectives-LL-LOO-LMP-gradients,python,hartmann3/LMP,pylibkriging,nlpd,-0.528379 -python_Kriging_objectives-LL-LOO-LMP-gradients,python,ishigami/LL,pylibkriging,rmse,3.41501 -python_Kriging_objectives-LL-LOO-LMP-gradients,python,ishigami/LL,pylibkriging,q2,0.203357 -python_Kriging_objectives-LL-LOO-LMP-gradients,python,ishigami/LL,pylibkriging,nlpd,4.86258 -python_Kriging_objectives-LL-LOO-LMP-gradients,python,ishigami/LOO,pylibkriging,rmse,2.45853 -python_Kriging_objectives-LL-LOO-LMP-gradients,python,ishigami/LOO,pylibkriging,q2,0.587112 -python_Kriging_objectives-LL-LOO-LMP-gradients,python,ishigami/LOO,pylibkriging,nlpd,2.72077 -python_Kriging_objectives-LL-LOO-LMP-gradients,python,ishigami/LMP,pylibkriging,rmse,3.47385 -python_Kriging_objectives-LL-LOO-LMP-gradients,python,ishigami/LMP,pylibkriging,q2,0.175667 -python_Kriging_objectives-LL-LOO-LMP-gradients,python,ishigami/LMP,pylibkriging,nlpd,5.27033 -python_Kriging_objectives-LL-LOO-LMP-gradients,python,friedman/LL,pylibkriging,rmse,0.273164 -python_Kriging_objectives-LL-LOO-LMP-gradients,python,friedman/LL,pylibkriging,q2,0.996855 -python_Kriging_objectives-LL-LOO-LMP-gradients,python,friedman/LL,pylibkriging,nlpd,-0.118408 -python_Kriging_objectives-LL-LOO-LMP-gradients,python,friedman/LOO,pylibkriging,rmse,0.233921 -python_Kriging_objectives-LL-LOO-LMP-gradients,python,friedman/LOO,pylibkriging,q2,0.997694 -python_Kriging_objectives-LL-LOO-LMP-gradients,python,friedman/LOO,pylibkriging,nlpd,-0.169369 -python_Kriging_objectives-LL-LOO-LMP-gradients,python,friedman/LMP,pylibkriging,rmse,0.269913 -python_Kriging_objectives-LL-LOO-LMP-gradients,python,friedman/LMP,pylibkriging,q2,0.996929 -python_Kriging_objectives-LL-LOO-LMP-gradients,python,friedman/LMP,pylibkriging,nlpd,-0.122709 -python_Kriging-nugget-noise_noisy-data,python,forrester,nugget,rmse,0.198091 -python_Kriging-nugget-noise_noisy-data,python,forrester,nugget,q2,0.998087 -python_Kriging-nugget-noise_noisy-data,python,forrester,nugget,nlpd,0.286631 -python_Kriging-nugget-noise_noisy-data,python,forrester,noise-free fit,rmse,0.487916 -python_Kriging-nugget-noise_noisy-data,python,forrester,noise-free fit,q2,0.988392 -python_Kriging-nugget-noise_noisy-data,python,forrester,noise-free fit,nlpd,862.67 -python_Kriging-nugget-noise_noisy-data,python,friedman,nugget,rmse,0.467335 -python_Kriging-nugget-noise_noisy-data,python,friedman,nugget,q2,0.99089 -python_Kriging-nugget-noise_noisy-data,python,friedman,nugget,nlpd,1.07073 -python_Kriging-nugget-noise_noisy-data,python,friedman,noise-free fit,rmse,1.27157 -python_Kriging-nugget-noise_noisy-data,python,friedman,noise-free fit,q2,0.932558 -python_Kriging-nugget-noise_noisy-data,python,friedman,noise-free fit,nlpd,1.64497 -python_Kriging-nugget-noise_noisy-data,python,gramacy_lee_2012/hetero,known variances,rmse,0.168741 -python_Kriging-nugget-noise_noisy-data,python,gramacy_lee_2012/hetero,known variances,q2,0.982515 -python_Kriging-nugget-noise_noisy-data,python,gramacy_lee_2012/hetero,known variances,nlpd,-0.570574 -python_Kriging-nugget-noise_noisy-data,python,gramacy_lee_2012/hetero,nugget,rmse,0.170945 -python_Kriging-nugget-noise_noisy-data,python,gramacy_lee_2012/hetero,nugget,q2,0.982055 -python_Kriging-nugget-noise_noisy-data,python,gramacy_lee_2012/hetero,nugget,nlpd,-0.221568 -python_WarpKriging_warpings-categorical-ordinal,python,gl2012,Kriging,rmse,0.167263 -python_WarpKriging_warpings-categorical-ordinal,python,gl2012,Kriging,q2,0.981176 -python_WarpKriging_warpings-categorical-ordinal,python,gl2012,Kriging,nlpd,-1.08917 -python_WarpKriging_warpings-categorical-ordinal,python,gl2012,affine,rmse,0.167263 -python_WarpKriging_warpings-categorical-ordinal,python,gl2012,affine,q2,0.981176 -python_WarpKriging_warpings-categorical-ordinal,python,gl2012,affine,nlpd,-1.08917 -python_WarpKriging_warpings-categorical-ordinal,python,gl2012,boxcox,rmse,0.16759 -python_WarpKriging_warpings-categorical-ordinal,python,gl2012,boxcox,q2,0.981103 -python_WarpKriging_warpings-categorical-ordinal,python,gl2012,boxcox,nlpd,282.422 -python_WarpKriging_warpings-categorical-ordinal,python,gl2012,kumaraswamy,rmse,0.168286 -python_WarpKriging_warpings-categorical-ordinal,python,gl2012,kumaraswamy,q2,0.980945 -python_WarpKriging_warpings-categorical-ordinal,python,gl2012,kumaraswamy,nlpd,2.04577e+06 -python_WarpKriging_warpings-categorical-ordinal,python,gl2012,knots(5),rmse,0.16791 -python_WarpKriging_warpings-categorical-ordinal,python,gl2012,knots(5),q2,0.98103 -python_WarpKriging_warpings-categorical-ordinal,python,gl2012,knots(5),nlpd,2.04376e+06 -python_WarpKriging_warpings-categorical-ordinal,python,gl2012,neural_mono(8),rmse,0.170824 -python_WarpKriging_warpings-categorical-ordinal,python,gl2012,neural_mono(8),q2,0.980366 -python_WarpKriging_warpings-categorical-ordinal,python,gl2012,neural_mono(8),nlpd,-0.265141 -python_WarpKriging_warpings-categorical-ordinal,python,gl2008,Kriging,rmse,0.0534649 -python_WarpKriging_warpings-categorical-ordinal,python,gl2008,Kriging,q2,0.585636 -python_WarpKriging_warpings-categorical-ordinal,python,gl2008,Kriging,nlpd,-1.74569 -python_WarpKriging_warpings-categorical-ordinal,python,gl2008,kumaraswamy,rmse,0.0535298 -python_WarpKriging_warpings-categorical-ordinal,python,gl2008,kumaraswamy,q2,0.58463 -python_WarpKriging_warpings-categorical-ordinal,python,gl2008,kumaraswamy,nlpd,-1.74552 -python_WarpKriging_warpings-categorical-ordinal,python,mixed,Kriging (level as number),rmse,16.7693 -python_WarpKriging_warpings-categorical-ordinal,python,mixed,Kriging (level as number),q2,0.838452 -python_WarpKriging_warpings-categorical-ordinal,python,mixed,Kriging (level as number),nlpd,3.81063 -python_WarpKriging_warpings-categorical-ordinal,python,mixed,"categorical(5,2)",rmse,16.1438 -python_WarpKriging_warpings-categorical-ordinal,python,mixed,"categorical(5,2)",q2,0.85028 -python_WarpKriging_warpings-categorical-ordinal,python,mixed,"categorical(5,2)",nlpd,3.3206 -python_WarpKriging_warpings-categorical-ordinal,python,ordinal,Kriging (rank as number),rmse,0.213845 -python_WarpKriging_warpings-categorical-ordinal,python,ordinal,Kriging (rank as number),q2,0.953177 -python_WarpKriging_warpings-categorical-ordinal,python,ordinal,Kriging (rank as number),nlpd,0.810969 -python_WarpKriging_warpings-categorical-ordinal,python,ordinal,ordinal(6),rmse,0.213043 -python_WarpKriging_warpings-categorical-ordinal,python,ordinal,ordinal(6),q2,0.953528 -python_WarpKriging_warpings-categorical-ordinal,python,ordinal,ordinal(6),nlpd,0.78888 -python_Kriging_high-dimension-normalize-screening,python,otl_circuit,unit cube,fit_s,0.00789523 -python_Kriging_high-dimension-normalize-screening,python,otl_circuit,unit cube,rmse,0.0150514 -python_Kriging_high-dimension-normalize-screening,python,otl_circuit,unit cube,q2,0.999836 -python_Kriging_high-dimension-normalize-screening,python,otl_circuit,unit cube,nlpd,-2.90452 -python_Kriging_high-dimension-normalize-screening,python,otl_circuit,native + normalize,rmse,0.0150514 -python_Kriging_high-dimension-normalize-screening,python,otl_circuit,native + normalize,q2,0.999836 -python_Kriging_high-dimension-normalize-screening,python,otl_circuit,native + normalize,nlpd,-2.90452 -python_Kriging_high-dimension-normalize-screening,python,piston,unit cube,fit_s,0.0197392 -python_Kriging_high-dimension-normalize-screening,python,piston,unit cube,rmse,0.0104258 -python_Kriging_high-dimension-normalize-screening,python,piston,unit cube,q2,0.994462 -python_Kriging_high-dimension-normalize-screening,python,piston,unit cube,nlpd,-3.49038 -python_Kriging_high-dimension-normalize-screening,python,piston,native + normalize,rmse,0.0104258 -python_Kriging_high-dimension-normalize-screening,python,piston,native + normalize,q2,0.994462 -python_Kriging_high-dimension-normalize-screening,python,piston,native + normalize,nlpd,-3.49038 -python_Kriging_high-dimension-normalize-screening,python,borehole,unit cube,fit_s,0.0140877 -python_Kriging_high-dimension-normalize-screening,python,borehole,unit cube,rmse,0.893341 -python_Kriging_high-dimension-normalize-screening,python,borehole,unit cube,q2,0.999599 -python_Kriging_high-dimension-normalize-screening,python,borehole,unit cube,nlpd,1.08433 -python_Kriging_high-dimension-normalize-screening,python,borehole,native + normalize,rmse,0.893341 -python_Kriging_high-dimension-normalize-screening,python,borehole,native + normalize,q2,0.999599 -python_Kriging_high-dimension-normalize-screening,python,borehole,native + normalize,nlpd,1.08433 -python_Kriging_high-dimension-normalize-screening,python,robot_arm,unit cube,fit_s,0.167482 -python_Kriging_high-dimension-normalize-screening,python,robot_arm,unit cube,rmse,0.358507 -python_Kriging_high-dimension-normalize-screening,python,robot_arm,unit cube,q2,0.571154 -python_Kriging_high-dimension-normalize-screening,python,robot_arm,unit cube,nlpd,0.355106 -python_Kriging_high-dimension-normalize-screening,python,robot_arm,native + normalize,rmse,0.358507 -python_Kriging_high-dimension-normalize-screening,python,robot_arm,native + normalize,q2,0.571154 -python_Kriging_high-dimension-normalize-screening,python,robot_arm,native + normalize,nlpd,0.355106 -python_Kriging_high-dimension-normalize-screening,python,dette_pepelyshev,unit cube,fit_s,0.0246282 -python_Kriging_high-dimension-normalize-screening,python,dette_pepelyshev,unit cube,rmse,0.424908 -python_Kriging_high-dimension-normalize-screening,python,dette_pepelyshev,unit cube,q2,0.997757 -python_Kriging_high-dimension-normalize-screening,python,dette_pepelyshev,unit cube,nlpd,0.510276 -python_Kriging_high-dimension-normalize-screening,python,dette_pepelyshev,native + normalize,rmse,0.424908 -python_Kriging_high-dimension-normalize-screening,python,dette_pepelyshev,native + normalize,q2,0.997757 -python_Kriging_high-dimension-normalize-screening,python,dette_pepelyshev,native + normalize,nlpd,0.510276 -python_Kriging_high-dimension-normalize-screening,python,wing_weight,unit cube,fit_s,0.0245481 -python_Kriging_high-dimension-normalize-screening,python,wing_weight,unit cube,rmse,0.809743 -python_Kriging_high-dimension-normalize-screening,python,wing_weight,unit cube,q2,0.999716 -python_Kriging_high-dimension-normalize-screening,python,wing_weight,unit cube,nlpd,1.07161 -python_Kriging_high-dimension-normalize-screening,python,wing_weight,native + normalize,rmse,0.809743 -python_Kriging_high-dimension-normalize-screening,python,wing_weight,native + normalize,q2,0.999716 -python_Kriging_high-dimension-normalize-screening,python,wing_weight,native + normalize,nlpd,1.07161 -python_Kriging_high-dimension-normalize-screening,python,welch,unit cube,fit_s,0.564758 -python_Kriging_high-dimension-normalize-screening,python,welch,unit cube,rmse,0.167729 -python_Kriging_high-dimension-normalize-screening,python,welch,unit cube,q2,0.993452 -python_Kriging_high-dimension-normalize-screening,python,welch,unit cube,nlpd,-0.429138 -python_Kriging_high-dimension-normalize-screening,python,welch,native + normalize,rmse,0.167692 -python_Kriging_high-dimension-normalize-screening,python,welch,native + normalize,q2,0.993454 -python_Kriging_high-dimension-normalize-screening,python,welch,native + normalize,nlpd,-0.429291 -python_MLPKriging_deep-kernel-projected-branin,python,projected_branin_10d,Kriging,rmse,4.42284 -python_MLPKriging_deep-kernel-projected-branin,python,projected_branin_10d,Kriging,q2,0.959122 -python_MLPKriging_deep-kernel-projected-branin,python,projected_branin_10d,Kriging,nlpd,2.89997 -python_MLPKriging_deep-kernel-projected-branin,python,projected_branin_10d,MLPKriging tanh BFGS,fit_s,0.130867 -python_MLPKriging_deep-kernel-projected-branin,python,projected_branin_10d,MLPKriging tanh BFGS,rmse,57.402 -python_MLPKriging_deep-kernel-projected-branin,python,projected_branin_10d,MLPKriging tanh BFGS,q2,-5.88553 -python_MLPKriging_deep-kernel-projected-branin,python,projected_branin_10d,MLPKriging tanh BFGS,nlpd,136.771 -python_MLPKriging_deep-kernel-projected-branin,python,projected_branin_10d,MLPKriging tanh BFGS10,fit_s,0.442791 -python_MLPKriging_deep-kernel-projected-branin,python,projected_branin_10d,MLPKriging tanh BFGS10,rmse,4.28606 -python_MLPKriging_deep-kernel-projected-branin,python,projected_branin_10d,MLPKriging tanh BFGS10,q2,0.961612 -python_MLPKriging_deep-kernel-projected-branin,python,projected_branin_10d,MLPKriging tanh BFGS10,nlpd,29.0854 -python_MLPKriging_deep-kernel-projected-branin,python,projected_branin_10d,MLPKriging selu BFGS,fit_s,0.139126 -python_MLPKriging_deep-kernel-projected-branin,python,projected_branin_10d,MLPKriging selu BFGS,rmse,58.9913 -python_MLPKriging_deep-kernel-projected-branin,python,projected_branin_10d,MLPKriging selu BFGS,q2,-6.2721 -python_MLPKriging_deep-kernel-projected-branin,python,projected_branin_10d,MLPKriging selu BFGS,nlpd,46.9788 -python_MLPKriging_deep-kernel-projected-branin,python,projected_branin_10d,MLPKriging selu BFGS10,fit_s,0.440179 -python_MLPKriging_deep-kernel-projected-branin,python,projected_branin_10d,MLPKriging selu BFGS10,rmse,4.39195 -python_MLPKriging_deep-kernel-projected-branin,python,projected_branin_10d,MLPKriging selu BFGS10,q2,0.959691 -python_MLPKriging_deep-kernel-projected-branin,python,projected_branin_10d,MLPKriging selu BFGS10,nlpd,10.6053 -python_Kriging-vecchia-nystrom_large-n,python,hartmann3/n=1500,exact,fit_s,11.1133 -python_NestedKriging_aggregations-large-n,python,hartmann3/n=1500,exact,fit_s,11.1133 -python_Kriging-vecchia-nystrom_large-n,python,hartmann3/n=1500,exact,rmse,0.000259233 -python_NestedKriging_aggregations-large-n,python,hartmann3/n=1500,exact,rmse,0.000259233 -python_Kriging-vecchia-nystrom_large-n,python,hartmann3/n=1500,exact,q2,1 -python_NestedKriging_aggregations-large-n,python,hartmann3/n=1500,exact,q2,1 -python_Kriging-vecchia-nystrom_large-n,python,hartmann3/n=1500,exact,nlpd,-6.92184 -python_NestedKriging_aggregations-large-n,python,hartmann3/n=1500,exact,nlpd,-6.92184 -python_Kriging-vecchia-nystrom_large-n,python,hartmann3/n=1500,vecchia(30),fit_s,5.43768 -python_Kriging-vecchia-nystrom_large-n,python,hartmann3/n=1500,vecchia(30),rmse,0.00353225 -python_Kriging-vecchia-nystrom_large-n,python,hartmann3/n=1500,vecchia(30),q2,0.999987 -python_Kriging-vecchia-nystrom_large-n,python,hartmann3/n=1500,vecchia(30),nlpd,-4.6266 -python_Kriging-vecchia-nystrom_large-n,python,hartmann3/n=5000,vecchia(30),fit_s,16.2077 -python_Kriging-vecchia-nystrom_large-n,python,hartmann3/n=5000,vecchia(30),rmse,0.000851738 -python_Kriging-vecchia-nystrom_large-n,python,hartmann3/n=5000,vecchia(30),q2,0.999999 -python_Kriging-vecchia-nystrom_large-n,python,hartmann3/n=5000,vecchia(30),nlpd,-5.95078 -python_Kriging-vecchia-nystrom_large-n,python,hartmann3/n=1500,nystrom(100),fit_s,2.04779 -python_Kriging-vecchia-nystrom_large-n,python,hartmann3/n=1500,nystrom(100),rmse,1.51127 -python_Kriging-vecchia-nystrom_large-n,python,hartmann3/n=1500,nystrom(100),q2,-1.39237 -python_Kriging-vecchia-nystrom_large-n,python,hartmann3/n=1500,nystrom(100),nlpd,1.14197e+24 -python_Kriging-vecchia-nystrom_large-n,python,hartmann3/n=5000,nystrom(100),fit_s,3.25622 -python_Kriging-vecchia-nystrom_large-n,python,hartmann3/n=5000,nystrom(100),rmse,9.95398 -python_Kriging-vecchia-nystrom_large-n,python,hartmann3/n=5000,nystrom(100),q2,-102.786 -python_Kriging-vecchia-nystrom_large-n,python,hartmann3/n=5000,nystrom(100),nlpd,4.95409e+25 -python_NestedKriging_aggregations-large-n,python,hartmann3/n=1500,nested NK (20 groups),fit_s,0.269932 -python_NestedKriging_aggregations-large-n,python,hartmann3/n=1500,nested NK (20 groups),rmse,0.0029908 -python_NestedKriging_aggregations-large-n,python,hartmann3/n=1500,nested NK (20 groups),q2,0.999991 -python_NestedKriging_aggregations-large-n,python,hartmann3/n=1500,nested NK (20 groups),nlpd,-4.85946 -python_NestedKriging_aggregations-large-n,python,hartmann3/n=5000,nested NK (20 groups),fit_s,2.87469 -python_NestedKriging_aggregations-large-n,python,hartmann3/n=5000,nested NK (20 groups),rmse,0.000222655 -python_NestedKriging_aggregations-large-n,python,hartmann3/n=5000,nested NK (20 groups),q2,1 -python_NestedKriging_aggregations-large-n,python,hartmann3/n=5000,nested NK (20 groups),nlpd,-7.44453 -python_NestedKriging_aggregations-large-n,python,hartmann3/n=1500,nested rBCM (20 groups),fit_s,0.25767 -python_NestedKriging_aggregations-large-n,python,hartmann3/n=1500,nested rBCM (20 groups),rmse,0.00304431 -python_NestedKriging_aggregations-large-n,python,hartmann3/n=1500,nested rBCM (20 groups),q2,0.99999 -python_NestedKriging_aggregations-large-n,python,hartmann3/n=1500,nested rBCM (20 groups),nlpd,-4.50201 -python_NestedKriging_aggregations-large-n,python,hartmann3/n=5000,nested rBCM (20 groups),fit_s,2.5369 -python_NestedKriging_aggregations-large-n,python,hartmann3/n=5000,nested rBCM (20 groups),rmse,0.000224517 -python_NestedKriging_aggregations-large-n,python,hartmann3/n=5000,nested rBCM (20 groups),q2,1 -python_NestedKriging_aggregations-large-n,python,hartmann3/n=5000,nested rBCM (20 groups),nlpd,-7.18182 -python_Kriging-vecchia-nystrom_large-n,python,hartmann3/n=1500,subset of data (500),fit_s,0.712767 -python_Kriging-vecchia-nystrom_large-n,python,hartmann3/n=1500,subset of data (500),rmse,0.00269153 -python_Kriging-vecchia-nystrom_large-n,python,hartmann3/n=1500,subset of data (500),q2,0.999992 -python_Kriging-vecchia-nystrom_large-n,python,hartmann3/n=1500,subset of data (500),nlpd,-4.68641 -python_Kriging-vecchia-nystrom_large-n,python,hartmann3/n=5000,subset of data (500),fit_s,0.950298 -python_Kriging-vecchia-nystrom_large-n,python,hartmann3/n=5000,subset of data (500),rmse,0.00247552 -python_Kriging-vecchia-nystrom_large-n,python,hartmann3/n=5000,subset of data (500),q2,0.999994 -python_Kriging-vecchia-nystrom_large-n,python,hartmann3/n=5000,subset of data (500),nlpd,-4.71359 -python_Kriging-vecchia-nystrom_large-n,python,otl_circuit/n=1500,exact,fit_s,10.6103 -python_NestedKriging_aggregations-large-n,python,otl_circuit/n=1500,exact,fit_s,10.6103 -python_Kriging-vecchia-nystrom_large-n,python,otl_circuit/n=1500,exact,rmse,9.81598e-05 -python_NestedKriging_aggregations-large-n,python,otl_circuit/n=1500,exact,rmse,9.81598e-05 -python_Kriging-vecchia-nystrom_large-n,python,otl_circuit/n=1500,exact,q2,1 -python_NestedKriging_aggregations-large-n,python,otl_circuit/n=1500,exact,q2,1 -python_Kriging-vecchia-nystrom_large-n,python,otl_circuit/n=1500,exact,nlpd,-8.21962 -python_NestedKriging_aggregations-large-n,python,otl_circuit/n=1500,exact,nlpd,-8.21962 -python_Kriging-vecchia-nystrom_large-n,python,otl_circuit/n=1500,vecchia(30),fit_s,4.73217 -python_Kriging-vecchia-nystrom_large-n,python,otl_circuit/n=1500,vecchia(30),rmse,0.00629979 -python_Kriging-vecchia-nystrom_large-n,python,otl_circuit/n=1500,vecchia(30),q2,0.999969 -python_Kriging-vecchia-nystrom_large-n,python,otl_circuit/n=1500,vecchia(30),nlpd,-3.95617 -python_Kriging-vecchia-nystrom_large-n,python,otl_circuit/n=5000,vecchia(30),fit_s,14.5549 -python_Kriging-vecchia-nystrom_large-n,python,otl_circuit/n=5000,vecchia(30),rmse,0.00312888 -python_Kriging-vecchia-nystrom_large-n,python,otl_circuit/n=5000,vecchia(30),q2,0.999992 -python_Kriging-vecchia-nystrom_large-n,python,otl_circuit/n=5000,vecchia(30),nlpd,-4.57612 -python_Kriging-vecchia-nystrom_large-n,python,otl_circuit/n=1500,nystrom(100),fit_s,1.47345 -python_Kriging-vecchia-nystrom_large-n,python,otl_circuit/n=1500,nystrom(100),rmse,0.392811 -python_Kriging-vecchia-nystrom_large-n,python,otl_circuit/n=1500,nystrom(100),q2,0.880908 -python_Kriging-vecchia-nystrom_large-n,python,otl_circuit/n=1500,nystrom(100),nlpd,7.06156e+22 -python_Kriging-vecchia-nystrom_large-n,python,otl_circuit/n=5000,nystrom(100),fit_s,4.69579 -python_Kriging-vecchia-nystrom_large-n,python,otl_circuit/n=5000,nystrom(100),rmse,1.11526 -python_Kriging-vecchia-nystrom_large-n,python,otl_circuit/n=5000,nystrom(100),q2,0.0400057 -python_Kriging-vecchia-nystrom_large-n,python,otl_circuit/n=5000,nystrom(100),nlpd,5.31511e+23 -python_NestedKriging_aggregations-large-n,python,otl_circuit/n=1500,nested NK (20 groups),fit_s,0.278129 -python_NestedKriging_aggregations-large-n,python,otl_circuit/n=1500,nested NK (20 groups),rmse,0.00226873 -python_NestedKriging_aggregations-large-n,python,otl_circuit/n=1500,nested NK (20 groups),q2,0.999996 -python_NestedKriging_aggregations-large-n,python,otl_circuit/n=1500,nested NK (20 groups),nlpd,-4.90542 -python_NestedKriging_aggregations-large-n,python,otl_circuit/n=5000,nested NK (20 groups),fit_s,3.62576 -python_NestedKriging_aggregations-large-n,python,otl_circuit/n=5000,nested NK (20 groups),rmse,0.000261686 -python_NestedKriging_aggregations-large-n,python,otl_circuit/n=5000,nested NK (20 groups),q2,1 -python_NestedKriging_aggregations-large-n,python,otl_circuit/n=5000,nested NK (20 groups),nlpd,-7.20623 -python_NestedKriging_aggregations-large-n,python,otl_circuit/n=1500,nested rBCM (20 groups),fit_s,0.315619 -python_NestedKriging_aggregations-large-n,python,otl_circuit/n=1500,nested rBCM (20 groups),rmse,0.00291869 -python_NestedKriging_aggregations-large-n,python,otl_circuit/n=1500,nested rBCM (20 groups),q2,0.999993 -python_NestedKriging_aggregations-large-n,python,otl_circuit/n=1500,nested rBCM (20 groups),nlpd,-2.55512 -python_NestedKriging_aggregations-large-n,python,otl_circuit/n=5000,nested rBCM (20 groups),fit_s,3.39037 -python_NestedKriging_aggregations-large-n,python,otl_circuit/n=5000,nested rBCM (20 groups),rmse,0.000278149 -python_NestedKriging_aggregations-large-n,python,otl_circuit/n=5000,nested rBCM (20 groups),q2,1 -python_NestedKriging_aggregations-large-n,python,otl_circuit/n=5000,nested rBCM (20 groups),nlpd,-4.34927 -python_Kriging-vecchia-nystrom_large-n,python,otl_circuit/n=1500,subset of data (500),fit_s,0.672431 -python_Kriging-vecchia-nystrom_large-n,python,otl_circuit/n=1500,subset of data (500),rmse,0.0004286 -python_Kriging-vecchia-nystrom_large-n,python,otl_circuit/n=1500,subset of data (500),q2,1 -python_Kriging-vecchia-nystrom_large-n,python,otl_circuit/n=1500,subset of data (500),nlpd,-6.72678 -python_Kriging-vecchia-nystrom_large-n,python,otl_circuit/n=5000,subset of data (500),fit_s,0.692672 -python_Kriging-vecchia-nystrom_large-n,python,otl_circuit/n=5000,subset of data (500),rmse,0.000446879 -python_Kriging-vecchia-nystrom_large-n,python,otl_circuit/n=5000,subset of data (500),q2,1 -python_Kriging-vecchia-nystrom_large-n,python,otl_circuit/n=5000,subset of data (500),nlpd,-6.78844 -python_NestedKriging_aggregations-large-n,python,branin/nested,NK,rmse,0.0343984 -python_NestedKriging_aggregations-large-n,python,branin/nested,NK,q2,1 -python_NestedKriging_aggregations-large-n,python,branin/nested,NK,nlpd,2.00082e+14 -python_NestedKriging_aggregations-large-n,python,branin/nested,PoE,rmse,0.0354265 -python_NestedKriging_aggregations-large-n,python,branin/nested,PoE,q2,1 -python_NestedKriging_aggregations-large-n,python,branin/nested,PoE,nlpd,-1.64533 -python_NestedKriging_aggregations-large-n,python,branin/nested,gPoE,rmse,0.0354265 -python_NestedKriging_aggregations-large-n,python,branin/nested,gPoE,q2,1 -python_NestedKriging_aggregations-large-n,python,branin/nested,gPoE,nlpd,-0.621836 -python_NestedKriging_aggregations-large-n,python,branin/nested,BCM,rmse,0.0354196 -python_NestedKriging_aggregations-large-n,python,branin/nested,BCM,q2,1 -python_NestedKriging_aggregations-large-n,python,branin/nested,BCM,nlpd,-1.64533 -python_NestedKriging_aggregations-large-n,python,branin/nested,rBCM,rmse,0.035287 -python_NestedKriging_aggregations-large-n,python,branin/nested,rBCM,q2,1 -python_NestedKriging_aggregations-large-n,python,branin/nested,rBCM,nlpd,-2.61369 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,branin,pylibkriging,q2,0.999944 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,branin,pylibkriging,nlpd,0.137225 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,branin,pylibkriging,fit_s,0.00261593 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,branin,pylibkriging BFGS10,q2,0.999945 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,branin,pylibkriging BFGS10,nlpd,0.137254 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,branin,pylibkriging BFGS10,fit_s,0.02301 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,branin,scikit-learn,q2,0.999597 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,branin,scikit-learn,nlpd,0.928508 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,branin,scikit-learn,fit_s,0.169693 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,branin,SMT,q2,0.999944 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,branin,SMT,nlpd,0.137259 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,branin,SMT,fit_s,0.833295 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,branin,OpenTURNS,q2,0.998221 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,branin,OpenTURNS,nlpd,1.55599 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,branin,OpenTURNS,fit_s,0.00490117 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,six_hump_camel,pylibkriging,q2,0.999418 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,six_hump_camel,pylibkriging,nlpd,0.0429342 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,six_hump_camel,pylibkriging,fit_s,0.00324678 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,six_hump_camel,pylibkriging BFGS10,q2,0.999418 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,six_hump_camel,pylibkriging BFGS10,nlpd,0.0429386 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,six_hump_camel,pylibkriging BFGS10,fit_s,0.060586 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,six_hump_camel,scikit-learn,q2,0.0163311 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,six_hump_camel,scikit-learn,nlpd,4.63763 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,six_hump_camel,scikit-learn,fit_s,0.206083 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,six_hump_camel,SMT,q2,0.999594 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,six_hump_camel,SMT,nlpd,-0.121007 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,six_hump_camel,SMT,fit_s,1.03758 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,six_hump_camel,OpenTURNS,q2,0.956713 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,six_hump_camel,OpenTURNS,nlpd,2.55446 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,six_hump_camel,OpenTURNS,fit_s,0.00741315 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann3,pylibkriging,q2,0.986148 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann3,pylibkriging,nlpd,-0.847247 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann3,pylibkriging,fit_s,0.0169713 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann3,pylibkriging BFGS10,q2,0.986148 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann3,pylibkriging BFGS10,nlpd,-0.847249 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann3,pylibkriging BFGS10,fit_s,0.139588 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann3,scikit-learn,q2,0.983889 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann3,scikit-learn,nlpd,-0.782697 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann3,scikit-learn,fit_s,0.188796 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann3,SMT,q2,0.986148 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann3,SMT,nlpd,-0.847249 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann3,SMT,fit_s,1.02907 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann3,OpenTURNS,q2,0.984712 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann3,OpenTURNS,nlpd,-0.809851 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann3,OpenTURNS,fit_s,0.0153861 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,ishigami,pylibkriging,q2,0.259556 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,ishigami,pylibkriging,nlpd,4.38991 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,ishigami,pylibkriging,fit_s,0.0256422 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,ishigami,pylibkriging BFGS10,q2,0.935722 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,ishigami,pylibkriging BFGS10,nlpd,0.857443 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,ishigami,pylibkriging BFGS10,fit_s,0.308839 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,ishigami,scikit-learn,q2,0.831142 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,ishigami,scikit-learn,nlpd,1.5455 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,ishigami,scikit-learn,fit_s,0.51733 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,ishigami,SMT,q2,0.51196 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,ishigami,SMT,nlpd,3.31349 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,ishigami,SMT,fit_s,2.77225 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,ishigami,OpenTURNS,q2,0.887251 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,ishigami,OpenTURNS,nlpd,1.39348 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,ishigami,OpenTURNS,fit_s,0.0499659 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,friedman,pylibkriging,q2,0.999771 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,friedman,pylibkriging,nlpd,-1.29726 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,friedman,pylibkriging,fit_s,0.0181558 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,friedman,pylibkriging BFGS10,q2,0.999771 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,friedman,pylibkriging BFGS10,nlpd,-1.29727 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,friedman,pylibkriging BFGS10,fit_s,0.215106 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,friedman,scikit-learn,q2,0.999754 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,friedman,scikit-learn,nlpd,-1.37702 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,friedman,scikit-learn,fit_s,0.283978 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,friedman,SMT,q2,0.999899 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,friedman,SMT,nlpd,-1.871 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,friedman,SMT,fit_s,4.3249 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,friedman,OpenTURNS,q2,0.990862 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,friedman,OpenTURNS,nlpd,0.469828 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,friedman,OpenTURNS,fit_s,0.0293689 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,otl_circuit,pylibkriging,q2,0.999922 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,otl_circuit,pylibkriging,nlpd,-3.46228 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,otl_circuit,pylibkriging,fit_s,0.013788 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,otl_circuit,pylibkriging BFGS10,q2,0.999922 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,otl_circuit,pylibkriging BFGS10,nlpd,-3.46227 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,otl_circuit,pylibkriging BFGS10,fit_s,0.141071 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,otl_circuit,scikit-learn,q2,0.999906 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,otl_circuit,scikit-learn,nlpd,-3.43062 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,otl_circuit,scikit-learn,fit_s,0.51218 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,otl_circuit,SMT,q2,0.999954 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,otl_circuit,SMT,nlpd,-3.35016 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,otl_circuit,SMT,fit_s,7.1224 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,otl_circuit,OpenTURNS,q2,0.99777 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,otl_circuit,OpenTURNS,nlpd,-1.61723 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,otl_circuit,OpenTURNS,fit_s,0.0603108 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,piston,pylibkriging,q2,0.997498 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,piston,pylibkriging,nlpd,-3.80667 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,piston,pylibkriging,fit_s,0.0395648 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,piston,pylibkriging BFGS10,q2,0.997498 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,piston,pylibkriging BFGS10,nlpd,-3.8067 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,piston,pylibkriging BFGS10,fit_s,0.327989 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,piston,scikit-learn,q2,0.998684 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,piston,scikit-learn,nlpd,-4.10497 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,piston,scikit-learn,fit_s,1.11386 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,piston,SMT,q2,0.997511 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,piston,SMT,nlpd,-3.86618 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,piston,SMT,fit_s,12.649 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,piston,OpenTURNS,q2,0.990631 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,piston,OpenTURNS,nlpd,-2.9906 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,piston,OpenTURNS,fit_s,0.11631 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,borehole,pylibkriging,q2,0.999694 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,borehole,pylibkriging,nlpd,1.00118 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,borehole,pylibkriging,fit_s,0.0149446 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,borehole,pylibkriging BFGS10,q2,0.999694 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,borehole,pylibkriging BFGS10,nlpd,1.00121 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,borehole,pylibkriging BFGS10,fit_s,0.164608 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,borehole,scikit-learn,q2,0.999929 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,borehole,scikit-learn,nlpd,0.194455 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,borehole,scikit-learn,fit_s,1.04563 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,borehole,SMT,q2,0.999925 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,borehole,SMT,nlpd,0.279122 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,borehole,SMT,fit_s,11.8531 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,borehole,OpenTURNS,q2,0.996206 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,borehole,OpenTURNS,nlpd,2.55431 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,borehole,OpenTURNS,fit_s,0.12395 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,wing_weight,pylibkriging,q2,0.99972 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,wing_weight,pylibkriging,nlpd,1.11517 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,wing_weight,pylibkriging,fit_s,0.0290489 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,wing_weight,pylibkriging BFGS10,q2,0.99972 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,wing_weight,pylibkriging BFGS10,nlpd,1.11512 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,wing_weight,pylibkriging BFGS10,fit_s,0.314707 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,wing_weight,scikit-learn,q2,0.998048 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,wing_weight,scikit-learn,nlpd,2.57304 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,wing_weight,scikit-learn,fit_s,0.729278 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,wing_weight,SMT,q2,0.999898 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,wing_weight,SMT,nlpd,0.604023 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,wing_weight,SMT,fit_s,25.8262 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,wing_weight,OpenTURNS,q2,0.993145 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,wing_weight,OpenTURNS,nlpd,2.83627 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,wing_weight,OpenTURNS,fit_s,0.280434 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann6,pylibkriging,q2,0.422938 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann6,pylibkriging,nlpd,0.0965167 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann6,pylibkriging,fit_s,0.0562823 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann6,pylibkriging BFGS10,q2,0.72352 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann6,pylibkriging BFGS10,nlpd,-0.156898 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann6,pylibkriging BFGS10,fit_s,0.822745 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann6,scikit-learn,q2,0.69049 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann6,scikit-learn,nlpd,-0.137845 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann6,scikit-learn,fit_s,1.3155 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann6,SMT,q2,0.708463 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann6,SMT,nlpd,-0.0177081 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann6,SMT,fit_s,12.0578 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann6,OpenTURNS,q2,0.696177 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann6,OpenTURNS,nlpd,-0.132838 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann6,OpenTURNS,fit_s,0.133332 +suite,lang,case,package,metric,value +kernels-trends-interpolation_Kriging_python,python,forrester/gauss/constant,pylibkriging,fit_s,0.00161386 +kernels-trends-interpolation_Kriging_python,python,forrester/gauss/constant,pylibkriging,rmse,0.0848271 +kernels-trends-interpolation_Kriging_python,python,forrester/gauss/constant,pylibkriging,q2,0.999634 +kernels-trends-interpolation_Kriging_python,python,forrester/gauss/constant,pylibkriging,nlpd,-3.06486 +kernels-trends-interpolation_Kriging_python,python,forrester/gauss/linear,pylibkriging,fit_s,0.00126195 +kernels-trends-interpolation_Kriging_python,python,forrester/gauss/linear,pylibkriging,rmse,0.0855468 +kernels-trends-interpolation_Kriging_python,python,forrester/gauss/linear,pylibkriging,q2,0.999628 +kernels-trends-interpolation_Kriging_python,python,forrester/gauss/linear,pylibkriging,nlpd,-3.11029 +kernels-trends-interpolation_Kriging_python,python,forrester/gauss/interactive,pylibkriging,fit_s,0.00143647 +kernels-trends-interpolation_Kriging_python,python,forrester/gauss/interactive,pylibkriging,rmse,0.0855468 +kernels-trends-interpolation_Kriging_python,python,forrester/gauss/interactive,pylibkriging,q2,0.999628 +kernels-trends-interpolation_Kriging_python,python,forrester/gauss/interactive,pylibkriging,nlpd,-3.11029 +kernels-trends-interpolation_Kriging_python,python,forrester/gauss/quadratic,pylibkriging,fit_s,0.000566483 +kernels-trends-interpolation_Kriging_python,python,forrester/gauss/quadratic,pylibkriging,rmse,1.57249 +kernels-trends-interpolation_Kriging_python,python,forrester/gauss/quadratic,pylibkriging,q2,0.874394 +kernels-trends-interpolation_Kriging_python,python,forrester/gauss/quadratic,pylibkriging,nlpd,5.33879 +kernels-trends-interpolation_Kriging_python,python,forrester/exp/constant,pylibkriging,fit_s,0.00041604 +kernels-trends-interpolation_Kriging_python,python,forrester/exp/constant,pylibkriging,rmse,0.721582 +kernels-trends-interpolation_Kriging_python,python,forrester/exp/constant,pylibkriging,q2,0.973551 +kernels-trends-interpolation_Kriging_python,python,forrester/exp/constant,pylibkriging,nlpd,1.54551 +kernels-trends-interpolation_Kriging_python,python,forrester/exp/linear,pylibkriging,fit_s,0.000438213 +kernels-trends-interpolation_Kriging_python,python,forrester/exp/linear,pylibkriging,rmse,0.793413 +kernels-trends-interpolation_Kriging_python,python,forrester/exp/linear,pylibkriging,q2,0.968024 +kernels-trends-interpolation_Kriging_python,python,forrester/exp/linear,pylibkriging,nlpd,1.50639 +kernels-trends-interpolation_Kriging_python,python,forrester/exp/interactive,pylibkriging,fit_s,0.000480413 +kernels-trends-interpolation_Kriging_python,python,forrester/exp/interactive,pylibkriging,rmse,0.793413 +kernels-trends-interpolation_Kriging_python,python,forrester/exp/interactive,pylibkriging,q2,0.968024 +kernels-trends-interpolation_Kriging_python,python,forrester/exp/interactive,pylibkriging,nlpd,1.50639 +kernels-trends-interpolation_Kriging_python,python,forrester/exp/quadratic,pylibkriging,fit_s,0.000730038 +kernels-trends-interpolation_Kriging_python,python,forrester/exp/quadratic,pylibkriging,rmse,0.723267 +kernels-trends-interpolation_Kriging_python,python,forrester/exp/quadratic,pylibkriging,q2,0.973428 +kernels-trends-interpolation_Kriging_python,python,forrester/exp/quadratic,pylibkriging,nlpd,1.28782 +kernels-trends-interpolation_Kriging_python,python,forrester/matern3_2/constant,pylibkriging,fit_s,0.00060606 +kernels-trends-interpolation_Kriging_python,python,forrester/matern3_2/constant,pylibkriging,rmse,0.334007 +kernels-trends-interpolation_Kriging_python,python,forrester/matern3_2/constant,pylibkriging,q2,0.994333 +kernels-trends-interpolation_Kriging_python,python,forrester/matern3_2/constant,pylibkriging,nlpd,-0.227715 +kernels-trends-interpolation_Kriging_python,python,forrester/matern3_2/linear,pylibkriging,fit_s,0.000588894 +kernels-trends-interpolation_Kriging_python,python,forrester/matern3_2/linear,pylibkriging,rmse,0.364477 +kernels-trends-interpolation_Kriging_python,python,forrester/matern3_2/linear,pylibkriging,q2,0.993252 +kernels-trends-interpolation_Kriging_python,python,forrester/matern3_2/linear,pylibkriging,nlpd,-0.187262 +kernels-trends-interpolation_Kriging_python,python,forrester/matern3_2/interactive,pylibkriging,fit_s,0.000905514 +kernels-trends-interpolation_Kriging_python,python,forrester/matern3_2/interactive,pylibkriging,rmse,0.364477 +kernels-trends-interpolation_Kriging_python,python,forrester/matern3_2/interactive,pylibkriging,q2,0.993252 +kernels-trends-interpolation_Kriging_python,python,forrester/matern3_2/interactive,pylibkriging,nlpd,-0.187262 +kernels-trends-interpolation_Kriging_python,python,forrester/matern3_2/quadratic,pylibkriging,fit_s,0.000975609 +kernels-trends-interpolation_Kriging_python,python,forrester/matern3_2/quadratic,pylibkriging,rmse,0.435609 +kernels-trends-interpolation_Kriging_python,python,forrester/matern3_2/quadratic,pylibkriging,q2,0.990361 +kernels-trends-interpolation_Kriging_python,python,forrester/matern3_2/quadratic,pylibkriging,nlpd,-0.183076 +kernels-trends-interpolation_Kriging_python,python,forrester/matern5_2/constant,pylibkriging,fit_s,0.000726223 +kernels-trends-interpolation_Kriging_python,python,forrester/matern5_2/constant,pylibkriging,rmse,0.281945 +kernels-trends-interpolation_Kriging_python,python,forrester/matern5_2/constant,pylibkriging,q2,0.995962 +kernels-trends-interpolation_Kriging_python,python,forrester/matern5_2/constant,pylibkriging,nlpd,-1.01945 +kernels-trends-interpolation_Kriging_python,python,forrester/matern5_2/constant,pylibkriging BFGS10,rmse,0.281945 +kernels-trends-interpolation_Kriging_python,python,forrester/matern5_2/constant,pylibkriging BFGS10,q2,0.995962 +kernels-trends-interpolation_Kriging_python,python,forrester/matern5_2/constant,pylibkriging BFGS10,nlpd,-1.01945 +kernels-trends-interpolation_Kriging_python,python,forrester/matern5_2/linear,pylibkriging,fit_s,0.000932932 +kernels-trends-interpolation_Kriging_python,python,forrester/matern5_2/linear,pylibkriging,rmse,0.298959 +kernels-trends-interpolation_Kriging_python,python,forrester/matern5_2/linear,pylibkriging,q2,0.99546 +kernels-trends-interpolation_Kriging_python,python,forrester/matern5_2/linear,pylibkriging,nlpd,-0.95551 +kernels-trends-interpolation_Kriging_python,python,forrester/matern5_2/interactive,pylibkriging,fit_s,0.000725269 +kernels-trends-interpolation_Kriging_python,python,forrester/matern5_2/interactive,pylibkriging,rmse,0.298959 +kernels-trends-interpolation_Kriging_python,python,forrester/matern5_2/interactive,pylibkriging,q2,0.99546 +kernels-trends-interpolation_Kriging_python,python,forrester/matern5_2/interactive,pylibkriging,nlpd,-0.95551 +kernels-trends-interpolation_Kriging_python,python,forrester/matern5_2/quadratic,pylibkriging,fit_s,0.00131941 +kernels-trends-interpolation_Kriging_python,python,forrester/matern5_2/quadratic,pylibkriging,rmse,0.365039 +kernels-trends-interpolation_Kriging_python,python,forrester/matern5_2/quadratic,pylibkriging,q2,0.993231 +kernels-trends-interpolation_Kriging_python,python,forrester/matern5_2/quadratic,pylibkriging,nlpd,-0.790032 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/gauss/constant,pylibkriging,fit_s,0.0016458 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/gauss/constant,pylibkriging,rmse,0.191274 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/gauss/constant,pylibkriging,q2,0.978742 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/gauss/constant,pylibkriging,nlpd,-0.704948 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/gauss/linear,pylibkriging,fit_s,0.00319505 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/gauss/linear,pylibkriging,rmse,0.236028 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/gauss/linear,pylibkriging,q2,0.96763 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/gauss/linear,pylibkriging,nlpd,-0.580993 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/gauss/interactive,pylibkriging,fit_s,0.00250673 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/gauss/interactive,pylibkriging,rmse,0.236028 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/gauss/interactive,pylibkriging,q2,0.96763 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/gauss/interactive,pylibkriging,nlpd,-0.580993 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/gauss/quadratic,pylibkriging,fit_s,0.00243235 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/gauss/quadratic,pylibkriging,rmse,0.138869 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/gauss/quadratic,pylibkriging,q2,0.988795 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/gauss/quadratic,pylibkriging,nlpd,-0.530437 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/exp/constant,pylibkriging,fit_s,0.000752449 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/exp/constant,pylibkriging,rmse,0.196586 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/exp/constant,pylibkriging,q2,0.977545 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/exp/constant,pylibkriging,nlpd,-0.0621192 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/exp/linear,pylibkriging,fit_s,0.0010047 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/exp/linear,pylibkriging,rmse,0.20979 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/exp/linear,pylibkriging,q2,0.974427 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/exp/linear,pylibkriging,nlpd,-0.0466568 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/exp/interactive,pylibkriging,fit_s,0.000931501 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/exp/interactive,pylibkriging,rmse,0.20979 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/exp/interactive,pylibkriging,q2,0.974427 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/exp/interactive,pylibkriging,nlpd,-0.0466568 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/exp/quadratic,pylibkriging,fit_s,0.0025785 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/exp/quadratic,pylibkriging,rmse,0.220579 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/exp/quadratic,pylibkriging,q2,0.971729 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/exp/quadratic,pylibkriging,nlpd,0.136521 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern3_2/constant,pylibkriging,fit_s,0.000761509 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern3_2/constant,pylibkriging,rmse,0.177253 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern3_2/constant,pylibkriging,q2,0.981744 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern3_2/constant,pylibkriging,nlpd,-0.482212 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern3_2/linear,pylibkriging,fit_s,0.00132823 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern3_2/linear,pylibkriging,rmse,0.211499 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern3_2/linear,pylibkriging,q2,0.974009 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern3_2/linear,pylibkriging,nlpd,-0.375844 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern3_2/interactive,pylibkriging,fit_s,0.000978708 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern3_2/interactive,pylibkriging,rmse,0.211499 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern3_2/interactive,pylibkriging,q2,0.974009 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern3_2/interactive,pylibkriging,nlpd,-0.375844 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern3_2/quadratic,pylibkriging,fit_s,0.00228167 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern3_2/quadratic,pylibkriging,rmse,0.173012 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern3_2/quadratic,pylibkriging,q2,0.982607 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern3_2/quadratic,pylibkriging,nlpd,-0.223308 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern5_2/constant,pylibkriging,fit_s,0.00131416 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern5_2/constant,pylibkriging,rmse,0.177638 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern5_2/constant,pylibkriging,q2,0.981665 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern5_2/constant,pylibkriging,nlpd,-0.575421 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern5_2/constant,pylibkriging BFGS10,rmse,0.177638 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern5_2/constant,pylibkriging BFGS10,q2,0.981665 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern5_2/constant,pylibkriging BFGS10,nlpd,-0.575422 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern5_2/linear,pylibkriging,fit_s,0.00130415 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern5_2/linear,pylibkriging,rmse,0.219759 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern5_2/linear,pylibkriging,q2,0.971939 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern5_2/linear,pylibkriging,nlpd,-0.458933 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern5_2/interactive,pylibkriging,fit_s,0.00119758 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern5_2/interactive,pylibkriging,rmse,0.219759 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern5_2/interactive,pylibkriging,q2,0.971939 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern5_2/interactive,pylibkriging,nlpd,-0.458933 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern5_2/quadratic,pylibkriging,fit_s,0.00169826 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern5_2/quadratic,pylibkriging,rmse,0.160506 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern5_2/quadratic,pylibkriging,q2,0.985031 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern5_2/quadratic,pylibkriging,nlpd,-0.337276 +kernels-trends-interpolation_Kriging_python,python,branin/gauss/constant,pylibkriging,fit_s,0.00104284 +kernels-trends-interpolation_Kriging_python,python,branin/gauss/constant,pylibkriging,rmse,7.2629 +kernels-trends-interpolation_Kriging_python,python,branin/gauss/constant,pylibkriging,q2,0.980655 +kernels-trends-interpolation_Kriging_python,python,branin/gauss/constant,pylibkriging,nlpd,4.3152 +kernels-trends-interpolation_Kriging_python,python,branin/gauss/linear,pylibkriging,fit_s,0.00129008 +kernels-trends-interpolation_Kriging_python,python,branin/gauss/linear,pylibkriging,rmse,7.25733 +kernels-trends-interpolation_Kriging_python,python,branin/gauss/linear,pylibkriging,q2,0.980685 +kernels-trends-interpolation_Kriging_python,python,branin/gauss/linear,pylibkriging,nlpd,4.30813 +kernels-trends-interpolation_Kriging_python,python,branin/gauss/interactive,pylibkriging,fit_s,0.00280094 +kernels-trends-interpolation_Kriging_python,python,branin/gauss/interactive,pylibkriging,rmse,7.2576 +kernels-trends-interpolation_Kriging_python,python,branin/gauss/interactive,pylibkriging,q2,0.980683 +kernels-trends-interpolation_Kriging_python,python,branin/gauss/interactive,pylibkriging,nlpd,4.30176 +kernels-trends-interpolation_Kriging_python,python,branin/gauss/quadratic,pylibkriging,fit_s,0.00112319 +kernels-trends-interpolation_Kriging_python,python,branin/gauss/quadratic,pylibkriging,rmse,7.27808 +kernels-trends-interpolation_Kriging_python,python,branin/gauss/quadratic,pylibkriging,q2,0.980574 +kernels-trends-interpolation_Kriging_python,python,branin/gauss/quadratic,pylibkriging,nlpd,4.29399 +kernels-trends-interpolation_Kriging_python,python,branin/exp/constant,pylibkriging,fit_s,0.00222707 +kernels-trends-interpolation_Kriging_python,python,branin/exp/constant,pylibkriging,rmse,14.5887 +kernels-trends-interpolation_Kriging_python,python,branin/exp/constant,pylibkriging,q2,0.921947 +kernels-trends-interpolation_Kriging_python,python,branin/exp/constant,pylibkriging,nlpd,4.09457 +kernels-trends-interpolation_Kriging_python,python,branin/exp/linear,pylibkriging,fit_s,0.00213099 +kernels-trends-interpolation_Kriging_python,python,branin/exp/linear,pylibkriging,rmse,14.9622 +kernels-trends-interpolation_Kriging_python,python,branin/exp/linear,pylibkriging,q2,0.9179 +kernels-trends-interpolation_Kriging_python,python,branin/exp/linear,pylibkriging,nlpd,4.0909 +kernels-trends-interpolation_Kriging_python,python,branin/exp/interactive,pylibkriging,fit_s,0.00196409 +kernels-trends-interpolation_Kriging_python,python,branin/exp/interactive,pylibkriging,rmse,11.6963 +kernels-trends-interpolation_Kriging_python,python,branin/exp/interactive,pylibkriging,q2,0.94983 +kernels-trends-interpolation_Kriging_python,python,branin/exp/interactive,pylibkriging,nlpd,3.79671 +kernels-trends-interpolation_Kriging_python,python,branin/exp/quadratic,pylibkriging,fit_s,0.00227737 +kernels-trends-interpolation_Kriging_python,python,branin/exp/quadratic,pylibkriging,rmse,9.33909 +kernels-trends-interpolation_Kriging_python,python,branin/exp/quadratic,pylibkriging,q2,0.968014 +kernels-trends-interpolation_Kriging_python,python,branin/exp/quadratic,pylibkriging,nlpd,3.62362 +kernels-trends-interpolation_Kriging_python,python,branin/matern3_2/constant,pylibkriging,fit_s,0.00244212 +kernels-trends-interpolation_Kriging_python,python,branin/matern3_2/constant,pylibkriging,rmse,3.24796 +kernels-trends-interpolation_Kriging_python,python,branin/matern3_2/constant,pylibkriging,q2,0.996131 +kernels-trends-interpolation_Kriging_python,python,branin/matern3_2/constant,pylibkriging,nlpd,2.30444 +kernels-trends-interpolation_Kriging_python,python,branin/matern3_2/linear,pylibkriging,fit_s,0.00244641 +kernels-trends-interpolation_Kriging_python,python,branin/matern3_2/linear,pylibkriging,rmse,3.14402 +kernels-trends-interpolation_Kriging_python,python,branin/matern3_2/linear,pylibkriging,q2,0.996375 +kernels-trends-interpolation_Kriging_python,python,branin/matern3_2/linear,pylibkriging,nlpd,2.28067 +kernels-trends-interpolation_Kriging_python,python,branin/matern3_2/interactive,pylibkriging,fit_s,0.00703263 +kernels-trends-interpolation_Kriging_python,python,branin/matern3_2/interactive,pylibkriging,rmse,3.19226 +kernels-trends-interpolation_Kriging_python,python,branin/matern3_2/interactive,pylibkriging,q2,0.996263 +kernels-trends-interpolation_Kriging_python,python,branin/matern3_2/interactive,pylibkriging,nlpd,2.20359 +kernels-trends-interpolation_Kriging_python,python,branin/matern3_2/quadratic,pylibkriging,fit_s,0.00404716 +kernels-trends-interpolation_Kriging_python,python,branin/matern3_2/quadratic,pylibkriging,rmse,3.01232 +kernels-trends-interpolation_Kriging_python,python,branin/matern3_2/quadratic,pylibkriging,q2,0.996672 +kernels-trends-interpolation_Kriging_python,python,branin/matern3_2/quadratic,pylibkriging,nlpd,2.11627 +kernels-trends-interpolation_Kriging_python,python,branin/matern5_2/constant,pylibkriging,fit_s,0.0029068 +kernels-trends-interpolation_Kriging_python,python,branin/matern5_2/constant,pylibkriging,rmse,0.52012 +kernels-trends-interpolation_Kriging_python,python,branin/matern5_2/constant,pylibkriging,q2,0.999901 +kernels-trends-interpolation_Kriging_python,python,branin/matern5_2/constant,pylibkriging,nlpd,0.283399 +kernels-trends-interpolation_Kriging_python,python,branin/matern5_2/constant,pylibkriging BFGS10,rmse,0.52012 +kernels-trends-interpolation_Kriging_python,python,branin/matern5_2/constant,pylibkriging BFGS10,q2,0.999901 +kernels-trends-interpolation_Kriging_python,python,branin/matern5_2/constant,pylibkriging BFGS10,nlpd,0.283395 +kernels-trends-interpolation_Kriging_python,python,branin/matern5_2/linear,pylibkriging,fit_s,0.00212407 +kernels-trends-interpolation_Kriging_python,python,branin/matern5_2/linear,pylibkriging,rmse,0.460868 +kernels-trends-interpolation_Kriging_python,python,branin/matern5_2/linear,pylibkriging,q2,0.999922 +kernels-trends-interpolation_Kriging_python,python,branin/matern5_2/linear,pylibkriging,nlpd,0.209741 +kernels-trends-interpolation_Kriging_python,python,branin/matern5_2/interactive,pylibkriging,fit_s,0.0018971 +kernels-trends-interpolation_Kriging_python,python,branin/matern5_2/interactive,pylibkriging,rmse,0.498484 +kernels-trends-interpolation_Kriging_python,python,branin/matern5_2/interactive,pylibkriging,q2,0.999909 +kernels-trends-interpolation_Kriging_python,python,branin/matern5_2/interactive,pylibkriging,nlpd,0.239079 +kernels-trends-interpolation_Kriging_python,python,branin/matern5_2/quadratic,pylibkriging,fit_s,0.00570393 +kernels-trends-interpolation_Kriging_python,python,branin/matern5_2/quadratic,pylibkriging,rmse,0.67472 +kernels-trends-interpolation_Kriging_python,python,branin/matern5_2/quadratic,pylibkriging,q2,0.999833 +kernels-trends-interpolation_Kriging_python,python,branin/matern5_2/quadratic,pylibkriging,nlpd,0.419266 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/gauss/constant,pylibkriging,fit_s,0.00197244 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/gauss/constant,pylibkriging,rmse,18.9501 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/gauss/constant,pylibkriging,q2,0.536354 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/gauss/constant,pylibkriging,nlpd,84.151 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/gauss/linear,pylibkriging,fit_s,0.00170588 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/gauss/linear,pylibkriging,rmse,18.9579 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/gauss/linear,pylibkriging,q2,0.535976 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/gauss/linear,pylibkriging,nlpd,44.8696 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/gauss/interactive,pylibkriging,fit_s,0.0020082 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/gauss/interactive,pylibkriging,rmse,18.9819 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/gauss/interactive,pylibkriging,q2,0.5348 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/gauss/interactive,pylibkriging,nlpd,35.2765 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/gauss/quadratic,pylibkriging,fit_s,0.00248289 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/gauss/quadratic,pylibkriging,rmse,10.9058 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/gauss/quadratic,pylibkriging,q2,0.846441 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/gauss/quadratic,pylibkriging,nlpd,4.92017 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/exp/constant,pylibkriging,fit_s,0.00100708 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/exp/constant,pylibkriging,rmse,16.9653 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/exp/constant,pylibkriging,q2,0.628393 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/exp/constant,pylibkriging,nlpd,4.48595 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/exp/linear,pylibkriging,fit_s,0.00118208 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/exp/linear,pylibkriging,rmse,17.1781 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/exp/linear,pylibkriging,q2,0.619012 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/exp/linear,pylibkriging,nlpd,4.47061 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/exp/interactive,pylibkriging,fit_s,0.00123763 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/exp/interactive,pylibkriging,rmse,17.1856 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/exp/interactive,pylibkriging,q2,0.618679 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/exp/interactive,pylibkriging,nlpd,4.43133 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/exp/quadratic,pylibkriging,fit_s,0.0013876 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/exp/quadratic,pylibkriging,rmse,7.99484 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/exp/quadratic,pylibkriging,q2,0.917476 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/exp/quadratic,pylibkriging,nlpd,3.35612 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern3_2/constant,pylibkriging,fit_s,0.00175476 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern3_2/constant,pylibkriging,rmse,18.4958 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern3_2/constant,pylibkriging,q2,0.558321 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern3_2/constant,pylibkriging,nlpd,18.8349 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern3_2/linear,pylibkriging,fit_s,0.00206542 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern3_2/linear,pylibkriging,rmse,18.5562 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern3_2/linear,pylibkriging,q2,0.555433 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern3_2/linear,pylibkriging,nlpd,14.9494 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern3_2/interactive,pylibkriging,fit_s,0.00178266 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern3_2/interactive,pylibkriging,rmse,18.561 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern3_2/interactive,pylibkriging,q2,0.555202 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern3_2/interactive,pylibkriging,nlpd,12.9923 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern3_2/quadratic,pylibkriging,fit_s,0.00338221 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern3_2/quadratic,pylibkriging,rmse,8.91645 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern3_2/quadratic,pylibkriging,q2,0.897353 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern3_2/quadratic,pylibkriging,nlpd,3.94939 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern5_2/constant,pylibkriging,fit_s,0.00228882 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern5_2/constant,pylibkriging,rmse,18.3236 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern5_2/constant,pylibkriging,q2,0.566507 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern5_2/constant,pylibkriging,nlpd,18.4322 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern5_2/constant,pylibkriging BFGS10,rmse,1.36208 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern5_2/constant,pylibkriging BFGS10,q2,0.997605 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern5_2/constant,pylibkriging BFGS10,nlpd,1.03215 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern5_2/linear,pylibkriging,fit_s,0.00242186 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern5_2/linear,pylibkriging,rmse,18.7127 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern5_2/linear,pylibkriging,q2,0.547901 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern5_2/linear,pylibkriging,nlpd,24.5771 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern5_2/interactive,pylibkriging,fit_s,0.00234485 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern5_2/interactive,pylibkriging,rmse,18.7194 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern5_2/interactive,pylibkriging,q2,0.547577 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern5_2/interactive,pylibkriging,nlpd,20.4255 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern5_2/quadratic,pylibkriging,fit_s,0.00251341 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern5_2/quadratic,pylibkriging,rmse,10.2878 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern5_2/quadratic,pylibkriging,q2,0.863351 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern5_2/quadratic,pylibkriging,nlpd,4.46344 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/gauss/constant,pylibkriging,fit_s,0.00588727 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/gauss/constant,pylibkriging,rmse,0.295934 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/gauss/constant,pylibkriging,q2,0.920537 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/gauss/constant,pylibkriging,nlpd,1.58666 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/gauss/linear,pylibkriging,fit_s,0.00581551 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/gauss/linear,pylibkriging,rmse,0.296355 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/gauss/linear,pylibkriging,q2,0.92031 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/gauss/linear,pylibkriging,nlpd,1.2962 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/gauss/interactive,pylibkriging,fit_s,0.00644922 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/gauss/interactive,pylibkriging,rmse,0.285136 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/gauss/interactive,pylibkriging,q2,0.926229 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/gauss/interactive,pylibkriging,nlpd,1.04122 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/gauss/quadratic,pylibkriging,fit_s,0.00578475 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/gauss/quadratic,pylibkriging,rmse,0.237308 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/gauss/quadratic,pylibkriging,q2,0.948902 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/gauss/quadratic,pylibkriging,nlpd,0.146484 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/exp/constant,pylibkriging,fit_s,0.00258231 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/exp/constant,pylibkriging,rmse,0.334876 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/exp/constant,pylibkriging,q2,0.898247 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/exp/constant,pylibkriging,nlpd,0.339235 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/exp/linear,pylibkriging,fit_s,0.00211 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/exp/linear,pylibkriging,rmse,0.333447 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/exp/linear,pylibkriging,q2,0.899114 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/exp/linear,pylibkriging,nlpd,0.336515 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/exp/interactive,pylibkriging,fit_s,0.0025239 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/exp/interactive,pylibkriging,rmse,0.336819 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/exp/interactive,pylibkriging,q2,0.897063 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/exp/interactive,pylibkriging,nlpd,0.329566 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/exp/quadratic,pylibkriging,fit_s,0.00309515 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/exp/quadratic,pylibkriging,rmse,0.327816 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/exp/quadratic,pylibkriging,q2,0.902492 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/exp/quadratic,pylibkriging,nlpd,0.322056 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern3_2/constant,pylibkriging,fit_s,0.00425816 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern3_2/constant,pylibkriging,rmse,0.684362 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern3_2/constant,pylibkriging,q2,0.575038 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern3_2/constant,pylibkriging,nlpd,0.972803 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern3_2/linear,pylibkriging,fit_s,0.00413442 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern3_2/linear,pylibkriging,rmse,0.650201 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern3_2/linear,pylibkriging,q2,0.616404 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern3_2/linear,pylibkriging,nlpd,0.921455 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern3_2/interactive,pylibkriging,fit_s,0.00704145 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern3_2/interactive,pylibkriging,rmse,0.274326 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern3_2/interactive,pylibkriging,q2,0.931717 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern3_2/interactive,pylibkriging,nlpd,0.0549679 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern3_2/quadratic,pylibkriging,fit_s,0.00996184 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern3_2/quadratic,pylibkriging,rmse,0.263745 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern3_2/quadratic,pylibkriging,q2,0.936883 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern3_2/quadratic,pylibkriging,nlpd,0.0446267 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern5_2/constant,pylibkriging,fit_s,0.00410438 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern5_2/constant,pylibkriging,rmse,0.674391 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern5_2/constant,pylibkriging,q2,0.58733 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern5_2/constant,pylibkriging,nlpd,0.944765 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern5_2/constant,pylibkriging BFGS10,rmse,0.268216 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern5_2/constant,pylibkriging BFGS10,q2,0.934725 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern5_2/constant,pylibkriging BFGS10,nlpd,0.169585 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern5_2/linear,pylibkriging,fit_s,0.00728607 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern5_2/linear,pylibkriging,rmse,0.268808 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern5_2/linear,pylibkriging,q2,0.934436 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern5_2/linear,pylibkriging,nlpd,0.142382 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern5_2/interactive,pylibkriging,fit_s,0.00524259 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern5_2/interactive,pylibkriging,rmse,0.271088 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern5_2/interactive,pylibkriging,q2,0.933319 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern5_2/interactive,pylibkriging,nlpd,0.128788 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern5_2/quadratic,pylibkriging,fit_s,0.011641 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern5_2/quadratic,pylibkriging,rmse,0.253093 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern5_2/quadratic,pylibkriging,q2,0.941878 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern5_2/quadratic,pylibkriging,nlpd,0.025469 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/gauss/constant,pylibkriging,fit_s,0.00113583 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/gauss/constant,pylibkriging,rmse,0.026127 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/gauss/constant,pylibkriging,q2,0.999737 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/gauss/constant,pylibkriging,nlpd,-2.69817 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/gauss/linear,pylibkriging,fit_s,0.00152946 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/gauss/linear,pylibkriging,rmse,1.06243 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/gauss/linear,pylibkriging,q2,0.564691 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/gauss/linear,pylibkriging,nlpd,2.7394 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/gauss/interactive,pylibkriging,fit_s,0.00153446 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/gauss/interactive,pylibkriging,rmse,1.06504 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/gauss/interactive,pylibkriging,q2,0.562551 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/gauss/interactive,pylibkriging,nlpd,2.33575 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/gauss/quadratic,pylibkriging,fit_s,0.00196862 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/gauss/quadratic,pylibkriging,rmse,0.281816 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/gauss/quadratic,pylibkriging,q2,0.969371 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/gauss/quadratic,pylibkriging,nlpd,0.71682 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/exp/constant,pylibkriging,fit_s,0.000828028 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/exp/constant,pylibkriging,rmse,0.249023 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/exp/constant,pylibkriging,q2,0.976085 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/exp/constant,pylibkriging,nlpd,0.394159 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/exp/linear,pylibkriging,fit_s,0.000962973 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/exp/linear,pylibkriging,rmse,0.245558 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/exp/linear,pylibkriging,q2,0.976746 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/exp/linear,pylibkriging,nlpd,0.224593 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/exp/interactive,pylibkriging,fit_s,0.00143671 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/exp/interactive,pylibkriging,rmse,0.239032 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/exp/interactive,pylibkriging,q2,0.977965 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/exp/interactive,pylibkriging,nlpd,0.226133 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/exp/quadratic,pylibkriging,fit_s,0.00274634 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/exp/quadratic,pylibkriging,rmse,0.237713 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/exp/quadratic,pylibkriging,q2,0.978208 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/exp/quadratic,pylibkriging,nlpd,-0.139582 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern3_2/constant,pylibkriging,fit_s,0.0011251 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern3_2/constant,pylibkriging,rmse,0.0600911 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern3_2/constant,pylibkriging,q2,0.998607 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern3_2/constant,pylibkriging,nlpd,-1.34658 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern3_2/linear,pylibkriging,fit_s,0.000877857 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern3_2/linear,pylibkriging,rmse,0.0584376 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern3_2/linear,pylibkriging,q2,0.998683 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern3_2/linear,pylibkriging,nlpd,-1.34912 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern3_2/interactive,pylibkriging,fit_s,0.00140738 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern3_2/interactive,pylibkriging,rmse,0.0576096 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern3_2/interactive,pylibkriging,q2,0.99872 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern3_2/interactive,pylibkriging,nlpd,-1.35376 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern3_2/quadratic,pylibkriging,fit_s,0.00383472 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern3_2/quadratic,pylibkriging,rmse,0.215336 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern3_2/quadratic,pylibkriging,q2,0.982117 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern3_2/quadratic,pylibkriging,nlpd,-0.350837 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern5_2/constant,pylibkriging,fit_s,0.000718832 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern5_2/constant,pylibkriging,rmse,0.0246348 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern5_2/constant,pylibkriging,q2,0.999766 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern5_2/constant,pylibkriging,nlpd,-2.59598 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern5_2/constant,pylibkriging BFGS10,rmse,0.0246338 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern5_2/constant,pylibkriging BFGS10,q2,0.999766 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern5_2/constant,pylibkriging BFGS10,nlpd,-2.59598 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern5_2/linear,pylibkriging,fit_s,0.0013113 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern5_2/linear,pylibkriging,rmse,0.0232483 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern5_2/linear,pylibkriging,q2,0.999792 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern5_2/linear,pylibkriging,nlpd,-2.58843 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern5_2/interactive,pylibkriging,fit_s,0.00150442 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern5_2/interactive,pylibkriging,rmse,0.0230248 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern5_2/interactive,pylibkriging,q2,0.999796 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern5_2/interactive,pylibkriging,nlpd,-2.58839 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern5_2/quadratic,pylibkriging,fit_s,0.00300908 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern5_2/quadratic,pylibkriging,rmse,0.209802 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern5_2/quadratic,pylibkriging,q2,0.983025 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern5_2/quadratic,pylibkriging,nlpd,-0.403998 +kernels-trends-interpolation_Kriging_python,python,ishigami/gauss/constant,pylibkriging,fit_s,0.0101857 +kernels-trends-interpolation_Kriging_python,python,ishigami/gauss/constant,pylibkriging,rmse,2.07486 +kernels-trends-interpolation_Kriging_python,python,ishigami/gauss/constant,pylibkriging,q2,0.668191 +kernels-trends-interpolation_Kriging_python,python,ishigami/gauss/constant,pylibkriging,nlpd,2.80326 +kernels-trends-interpolation_Kriging_python,python,ishigami/gauss/linear,pylibkriging,fit_s,0.0193923 +kernels-trends-interpolation_Kriging_python,python,ishigami/gauss/linear,pylibkriging,rmse,2.8459 +kernels-trends-interpolation_Kriging_python,python,ishigami/gauss/linear,pylibkriging,q2,0.375762 +kernels-trends-interpolation_Kriging_python,python,ishigami/gauss/linear,pylibkriging,nlpd,3.41919 +kernels-trends-interpolation_Kriging_python,python,ishigami/gauss/interactive,pylibkriging,fit_s,0.0174267 +kernels-trends-interpolation_Kriging_python,python,ishigami/gauss/interactive,pylibkriging,rmse,2.85366 +kernels-trends-interpolation_Kriging_python,python,ishigami/gauss/interactive,pylibkriging,q2,0.372353 +kernels-trends-interpolation_Kriging_python,python,ishigami/gauss/interactive,pylibkriging,nlpd,3.28643 +kernels-trends-interpolation_Kriging_python,python,ishigami/gauss/quadratic,pylibkriging,fit_s,0.00865173 +kernels-trends-interpolation_Kriging_python,python,ishigami/gauss/quadratic,pylibkriging,rmse,2.08981 +kernels-trends-interpolation_Kriging_python,python,ishigami/gauss/quadratic,pylibkriging,q2,0.663392 +kernels-trends-interpolation_Kriging_python,python,ishigami/gauss/quadratic,pylibkriging,nlpd,2.80097 +kernels-trends-interpolation_Kriging_python,python,ishigami/exp/constant,pylibkriging,fit_s,0.00899458 +kernels-trends-interpolation_Kriging_python,python,ishigami/exp/constant,pylibkriging,rmse,1.41632 +kernels-trends-interpolation_Kriging_python,python,ishigami/exp/constant,pylibkriging,q2,0.845391 +kernels-trends-interpolation_Kriging_python,python,ishigami/exp/constant,pylibkriging,nlpd,1.75239 +kernels-trends-interpolation_Kriging_python,python,ishigami/exp/linear,pylibkriging,fit_s,0.0115979 +kernels-trends-interpolation_Kriging_python,python,ishigami/exp/linear,pylibkriging,rmse,1.4095 +kernels-trends-interpolation_Kriging_python,python,ishigami/exp/linear,pylibkriging,q2,0.846877 +kernels-trends-interpolation_Kriging_python,python,ishigami/exp/linear,pylibkriging,nlpd,1.74779 +kernels-trends-interpolation_Kriging_python,python,ishigami/exp/interactive,pylibkriging,fit_s,0.00745058 +kernels-trends-interpolation_Kriging_python,python,ishigami/exp/interactive,pylibkriging,rmse,1.41511 +kernels-trends-interpolation_Kriging_python,python,ishigami/exp/interactive,pylibkriging,q2,0.845657 +kernels-trends-interpolation_Kriging_python,python,ishigami/exp/interactive,pylibkriging,nlpd,1.75024 +kernels-trends-interpolation_Kriging_python,python,ishigami/exp/quadratic,pylibkriging,fit_s,0.00843048 +kernels-trends-interpolation_Kriging_python,python,ishigami/exp/quadratic,pylibkriging,rmse,1.40329 +kernels-trends-interpolation_Kriging_python,python,ishigami/exp/quadratic,pylibkriging,q2,0.848223 +kernels-trends-interpolation_Kriging_python,python,ishigami/exp/quadratic,pylibkriging,nlpd,1.73254 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern3_2/constant,pylibkriging,fit_s,0.0480943 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern3_2/constant,pylibkriging,rmse,1.32971 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern3_2/constant,pylibkriging,q2,0.863723 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern3_2/constant,pylibkriging,nlpd,1.50265 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern3_2/linear,pylibkriging,fit_s,0.047653 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern3_2/linear,pylibkriging,rmse,1.30244 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern3_2/linear,pylibkriging,q2,0.869255 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern3_2/linear,pylibkriging,nlpd,1.47906 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern3_2/interactive,pylibkriging,fit_s,0.060349 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern3_2/interactive,pylibkriging,rmse,1.30591 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern3_2/interactive,pylibkriging,q2,0.868558 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern3_2/interactive,pylibkriging,nlpd,1.47384 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern3_2/quadratic,pylibkriging,fit_s,0.0471144 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern3_2/quadratic,pylibkriging,rmse,1.32197 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern3_2/quadratic,pylibkriging,q2,0.865305 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern3_2/quadratic,pylibkriging,nlpd,1.46827 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern5_2/constant,pylibkriging,fit_s,0.0145204 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern5_2/constant,pylibkriging,rmse,3.00566 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern5_2/constant,pylibkriging,q2,0.303709 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern5_2/constant,pylibkriging,nlpd,3.42085 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern5_2/constant,pylibkriging BFGS10,rmse,1.33903 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern5_2/constant,pylibkriging BFGS10,q2,0.861806 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern5_2/constant,pylibkriging BFGS10,nlpd,1.66132 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern5_2/linear,pylibkriging,fit_s,0.0518982 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern5_2/linear,pylibkriging,rmse,1.3434 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern5_2/linear,pylibkriging,q2,0.860901 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern5_2/linear,pylibkriging,nlpd,1.65103 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern5_2/interactive,pylibkriging,fit_s,0.0539804 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern5_2/interactive,pylibkriging,rmse,1.34479 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern5_2/interactive,pylibkriging,q2,0.860615 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern5_2/interactive,pylibkriging,nlpd,1.64153 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern5_2/quadratic,pylibkriging,fit_s,0.052948 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern5_2/quadratic,pylibkriging,rmse,1.37082 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern5_2/quadratic,pylibkriging,q2,0.855166 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern5_2/quadratic,pylibkriging,nlpd,1.62228 +kernels-trends-interpolation_Kriging_python,python,hartmann3/gauss/constant,pylibkriging,fit_s,0.00805306 +kernels-trends-interpolation_Kriging_python,python,hartmann3/gauss/constant,pylibkriging,rmse,0.156226 +kernels-trends-interpolation_Kriging_python,python,hartmann3/gauss/constant,pylibkriging,q2,0.971116 +kernels-trends-interpolation_Kriging_python,python,hartmann3/gauss/constant,pylibkriging,nlpd,-0.555623 +kernels-trends-interpolation_Kriging_python,python,hartmann3/gauss/linear,pylibkriging,fit_s,0.0129044 +kernels-trends-interpolation_Kriging_python,python,hartmann3/gauss/linear,pylibkriging,rmse,0.153297 +kernels-trends-interpolation_Kriging_python,python,hartmann3/gauss/linear,pylibkriging,q2,0.972189 +kernels-trends-interpolation_Kriging_python,python,hartmann3/gauss/linear,pylibkriging,nlpd,-0.568404 +kernels-trends-interpolation_Kriging_python,python,hartmann3/gauss/interactive,pylibkriging,fit_s,0.00773382 +kernels-trends-interpolation_Kriging_python,python,hartmann3/gauss/interactive,pylibkriging,rmse,0.712948 +kernels-trends-interpolation_Kriging_python,python,hartmann3/gauss/interactive,pylibkriging,q2,0.398455 +kernels-trends-interpolation_Kriging_python,python,hartmann3/gauss/interactive,pylibkriging,nlpd,1.74554 +kernels-trends-interpolation_Kriging_python,python,hartmann3/gauss/quadratic,pylibkriging,fit_s,0.00593877 +kernels-trends-interpolation_Kriging_python,python,hartmann3/gauss/quadratic,pylibkriging,rmse,0.73238 +kernels-trends-interpolation_Kriging_python,python,hartmann3/gauss/quadratic,pylibkriging,q2,0.365217 +kernels-trends-interpolation_Kriging_python,python,hartmann3/gauss/quadratic,pylibkriging,nlpd,2.14587 +kernels-trends-interpolation_Kriging_python,python,hartmann3/exp/constant,pylibkriging,fit_s,0.00374722 +kernels-trends-interpolation_Kriging_python,python,hartmann3/exp/constant,pylibkriging,rmse,0.347348 +kernels-trends-interpolation_Kriging_python,python,hartmann3/exp/constant,pylibkriging,q2,0.857215 +kernels-trends-interpolation_Kriging_python,python,hartmann3/exp/constant,pylibkriging,nlpd,0.352683 +kernels-trends-interpolation_Kriging_python,python,hartmann3/exp/linear,pylibkriging,fit_s,0.00216174 +kernels-trends-interpolation_Kriging_python,python,hartmann3/exp/linear,pylibkriging,rmse,0.347273 +kernels-trends-interpolation_Kriging_python,python,hartmann3/exp/linear,pylibkriging,q2,0.857276 +kernels-trends-interpolation_Kriging_python,python,hartmann3/exp/linear,pylibkriging,nlpd,0.343631 +kernels-trends-interpolation_Kriging_python,python,hartmann3/exp/interactive,pylibkriging,fit_s,0.0024302 +kernels-trends-interpolation_Kriging_python,python,hartmann3/exp/interactive,pylibkriging,rmse,0.378349 +kernels-trends-interpolation_Kriging_python,python,hartmann3/exp/interactive,pylibkriging,q2,0.83059 +kernels-trends-interpolation_Kriging_python,python,hartmann3/exp/interactive,pylibkriging,nlpd,0.385842 +kernels-trends-interpolation_Kriging_python,python,hartmann3/exp/quadratic,pylibkriging,fit_s,0.0039556 +kernels-trends-interpolation_Kriging_python,python,hartmann3/exp/quadratic,pylibkriging,rmse,0.355119 +kernels-trends-interpolation_Kriging_python,python,hartmann3/exp/quadratic,pylibkriging,q2,0.850755 +kernels-trends-interpolation_Kriging_python,python,hartmann3/exp/quadratic,pylibkriging,nlpd,0.334371 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern3_2/constant,pylibkriging,fit_s,0.0110524 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern3_2/constant,pylibkriging,rmse,0.227521 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern3_2/constant,pylibkriging,q2,0.938737 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern3_2/constant,pylibkriging,nlpd,-0.181207 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern3_2/linear,pylibkriging,fit_s,0.00763154 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern3_2/linear,pylibkriging,rmse,0.23038 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern3_2/linear,pylibkriging,q2,0.937188 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern3_2/linear,pylibkriging,nlpd,-0.162203 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern3_2/interactive,pylibkriging,fit_s,0.00412679 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern3_2/interactive,pylibkriging,rmse,0.315752 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern3_2/interactive,pylibkriging,q2,0.88201 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern3_2/interactive,pylibkriging,nlpd,0.898253 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern3_2/quadratic,pylibkriging,fit_s,0.00620675 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern3_2/quadratic,pylibkriging,rmse,0.291393 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern3_2/quadratic,pylibkriging,q2,0.899513 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern3_2/quadratic,pylibkriging,nlpd,0.53743 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern5_2/constant,pylibkriging,fit_s,0.00997186 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern5_2/constant,pylibkriging,rmse,0.190287 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern5_2/constant,pylibkriging,q2,0.957148 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern5_2/constant,pylibkriging,nlpd,-0.403054 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern5_2/constant,pylibkriging BFGS10,rmse,0.190286 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern5_2/constant,pylibkriging BFGS10,q2,0.957148 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern5_2/constant,pylibkriging BFGS10,nlpd,-0.40306 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern5_2/linear,pylibkriging,fit_s,0.0127335 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern5_2/linear,pylibkriging,rmse,0.189633 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern5_2/linear,pylibkriging,q2,0.957442 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern5_2/linear,pylibkriging,nlpd,-0.410075 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern5_2/interactive,pylibkriging,fit_s,0.0101018 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern5_2/interactive,pylibkriging,rmse,0.307114 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern5_2/interactive,pylibkriging,q2,0.888377 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern5_2/interactive,pylibkriging,nlpd,1.50063 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern5_2/quadratic,pylibkriging,fit_s,0.0101326 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern5_2/quadratic,pylibkriging,rmse,0.285355 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern5_2/quadratic,pylibkriging,q2,0.903634 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern5_2/quadratic,pylibkriging,nlpd,0.898128 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,branin/LL,pylibkriging,rmse,0.833715 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,branin/LL,pylibkriging,q2,0.999729 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,branin/LL,pylibkriging,nlpd,0.697034 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,branin/LOO,pylibkriging,rmse,0.550995 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,branin/LOO,pylibkriging,q2,0.999882 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,branin/LOO,pylibkriging,nlpd,0.471613 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,branin/LMP,pylibkriging,rmse,0.66967 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,branin/LMP,pylibkriging,q2,0.999825 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,branin/LMP,pylibkriging,nlpd,0.617527 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,hartmann3/LL,pylibkriging,rmse,0.157276 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,hartmann3/LL,pylibkriging,q2,0.970945 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,hartmann3/LL,pylibkriging,nlpd,-0.497548 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,hartmann3/LOO,pylibkriging,rmse,0.144982 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,hartmann3/LOO,pylibkriging,q2,0.97531 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,hartmann3/LOO,pylibkriging,nlpd,-0.584885 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,hartmann3/LMP,pylibkriging,rmse,0.150075 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,hartmann3/LMP,pylibkriging,q2,0.973544 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,hartmann3/LMP,pylibkriging,nlpd,-0.528379 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,ishigami/LL,pylibkriging,rmse,3.41501 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,ishigami/LL,pylibkriging,q2,0.203357 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,ishigami/LL,pylibkriging,nlpd,4.86258 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,ishigami/LOO,pylibkriging,rmse,2.45853 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,ishigami/LOO,pylibkriging,q2,0.587112 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,ishigami/LOO,pylibkriging,nlpd,2.72077 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,ishigami/LMP,pylibkriging,rmse,3.47385 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,ishigami/LMP,pylibkriging,q2,0.175667 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,ishigami/LMP,pylibkriging,nlpd,5.27033 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,friedman/LL,pylibkriging,rmse,0.273164 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,friedman/LL,pylibkriging,q2,0.996855 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,friedman/LL,pylibkriging,nlpd,-0.118408 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,friedman/LOO,pylibkriging,rmse,0.233921 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,friedman/LOO,pylibkriging,q2,0.997694 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,friedman/LOO,pylibkriging,nlpd,-0.169369 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,friedman/LMP,pylibkriging,rmse,0.269913 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,friedman/LMP,pylibkriging,q2,0.996929 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,friedman/LMP,pylibkriging,nlpd,-0.122709 +noisy-data_Kriging-nugget-noise_python,python,forrester,nugget,rmse,0.198091 +noisy-data_Kriging-nugget-noise_python,python,forrester,nugget,q2,0.998087 +noisy-data_Kriging-nugget-noise_python,python,forrester,nugget,nlpd,0.286631 +noisy-data_Kriging-nugget-noise_python,python,forrester,noise-free fit,rmse,0.487916 +noisy-data_Kriging-nugget-noise_python,python,forrester,noise-free fit,q2,0.988392 +noisy-data_Kriging-nugget-noise_python,python,forrester,noise-free fit,nlpd,862.67 +noisy-data_Kriging-nugget-noise_python,python,friedman,nugget,rmse,0.467335 +noisy-data_Kriging-nugget-noise_python,python,friedman,nugget,q2,0.99089 +noisy-data_Kriging-nugget-noise_python,python,friedman,nugget,nlpd,1.07073 +noisy-data_Kriging-nugget-noise_python,python,friedman,noise-free fit,rmse,1.27157 +noisy-data_Kriging-nugget-noise_python,python,friedman,noise-free fit,q2,0.932558 +noisy-data_Kriging-nugget-noise_python,python,friedman,noise-free fit,nlpd,1.64497 +noisy-data_Kriging-nugget-noise_python,python,gramacy_lee_2012/hetero,known variances,rmse,0.168741 +noisy-data_Kriging-nugget-noise_python,python,gramacy_lee_2012/hetero,known variances,q2,0.982515 +noisy-data_Kriging-nugget-noise_python,python,gramacy_lee_2012/hetero,known variances,nlpd,-0.570574 +noisy-data_Kriging-nugget-noise_python,python,gramacy_lee_2012/hetero,nugget,rmse,0.170945 +noisy-data_Kriging-nugget-noise_python,python,gramacy_lee_2012/hetero,nugget,q2,0.982055 +noisy-data_Kriging-nugget-noise_python,python,gramacy_lee_2012/hetero,nugget,nlpd,-0.221568 +warpings-categorical-ordinal_WarpKriging_python,python,gl2012,Kriging,rmse,0.167263 +warpings-categorical-ordinal_WarpKriging_python,python,gl2012,Kriging,q2,0.981176 +warpings-categorical-ordinal_WarpKriging_python,python,gl2012,Kriging,nlpd,-1.08917 +warpings-categorical-ordinal_WarpKriging_python,python,gl2012,affine,rmse,0.167263 +warpings-categorical-ordinal_WarpKriging_python,python,gl2012,affine,q2,0.981176 +warpings-categorical-ordinal_WarpKriging_python,python,gl2012,affine,nlpd,-1.08917 +warpings-categorical-ordinal_WarpKriging_python,python,gl2012,boxcox,rmse,0.16759 +warpings-categorical-ordinal_WarpKriging_python,python,gl2012,boxcox,q2,0.981103 +warpings-categorical-ordinal_WarpKriging_python,python,gl2012,boxcox,nlpd,282.422 +warpings-categorical-ordinal_WarpKriging_python,python,gl2012,kumaraswamy,rmse,0.168286 +warpings-categorical-ordinal_WarpKriging_python,python,gl2012,kumaraswamy,q2,0.980945 +warpings-categorical-ordinal_WarpKriging_python,python,gl2012,kumaraswamy,nlpd,2.04577e+06 +warpings-categorical-ordinal_WarpKriging_python,python,gl2012,knots(5),rmse,0.16791 +warpings-categorical-ordinal_WarpKriging_python,python,gl2012,knots(5),q2,0.98103 +warpings-categorical-ordinal_WarpKriging_python,python,gl2012,knots(5),nlpd,2.04376e+06 +warpings-categorical-ordinal_WarpKriging_python,python,gl2012,neural_mono(8),rmse,0.170824 +warpings-categorical-ordinal_WarpKriging_python,python,gl2012,neural_mono(8),q2,0.980366 +warpings-categorical-ordinal_WarpKriging_python,python,gl2012,neural_mono(8),nlpd,-0.265141 +warpings-categorical-ordinal_WarpKriging_python,python,gl2008,Kriging,rmse,0.0534649 +warpings-categorical-ordinal_WarpKriging_python,python,gl2008,Kriging,q2,0.585636 +warpings-categorical-ordinal_WarpKriging_python,python,gl2008,Kriging,nlpd,-1.74569 +warpings-categorical-ordinal_WarpKriging_python,python,gl2008,kumaraswamy,rmse,0.0535298 +warpings-categorical-ordinal_WarpKriging_python,python,gl2008,kumaraswamy,q2,0.58463 +warpings-categorical-ordinal_WarpKriging_python,python,gl2008,kumaraswamy,nlpd,-1.74552 +warpings-categorical-ordinal_WarpKriging_python,python,mixed,Kriging (level as number),rmse,16.7693 +warpings-categorical-ordinal_WarpKriging_python,python,mixed,Kriging (level as number),q2,0.838452 +warpings-categorical-ordinal_WarpKriging_python,python,mixed,Kriging (level as number),nlpd,3.81063 +warpings-categorical-ordinal_WarpKriging_python,python,mixed,"categorical(5,2)",rmse,16.1438 +warpings-categorical-ordinal_WarpKriging_python,python,mixed,"categorical(5,2)",q2,0.85028 +warpings-categorical-ordinal_WarpKriging_python,python,mixed,"categorical(5,2)",nlpd,3.3206 +warpings-categorical-ordinal_WarpKriging_python,python,ordinal,Kriging (rank as number),rmse,0.213845 +warpings-categorical-ordinal_WarpKriging_python,python,ordinal,Kriging (rank as number),q2,0.953177 +warpings-categorical-ordinal_WarpKriging_python,python,ordinal,Kriging (rank as number),nlpd,0.810969 +warpings-categorical-ordinal_WarpKriging_python,python,ordinal,ordinal(6),rmse,0.213043 +warpings-categorical-ordinal_WarpKriging_python,python,ordinal,ordinal(6),q2,0.953528 +warpings-categorical-ordinal_WarpKriging_python,python,ordinal,ordinal(6),nlpd,0.78888 +high-dimension-normalize-screening_Kriging_python,python,otl_circuit,unit cube,fit_s,0.00789523 +high-dimension-normalize-screening_Kriging_python,python,otl_circuit,unit cube,rmse,0.0150514 +high-dimension-normalize-screening_Kriging_python,python,otl_circuit,unit cube,q2,0.999836 +high-dimension-normalize-screening_Kriging_python,python,otl_circuit,unit cube,nlpd,-2.90452 +high-dimension-normalize-screening_Kriging_python,python,otl_circuit,native + normalize,rmse,0.0150514 +high-dimension-normalize-screening_Kriging_python,python,otl_circuit,native + normalize,q2,0.999836 +high-dimension-normalize-screening_Kriging_python,python,otl_circuit,native + normalize,nlpd,-2.90452 +high-dimension-normalize-screening_Kriging_python,python,piston,unit cube,fit_s,0.0197392 +high-dimension-normalize-screening_Kriging_python,python,piston,unit cube,rmse,0.0104258 +high-dimension-normalize-screening_Kriging_python,python,piston,unit cube,q2,0.994462 +high-dimension-normalize-screening_Kriging_python,python,piston,unit cube,nlpd,-3.49038 +high-dimension-normalize-screening_Kriging_python,python,piston,native + normalize,rmse,0.0104258 +high-dimension-normalize-screening_Kriging_python,python,piston,native + normalize,q2,0.994462 +high-dimension-normalize-screening_Kriging_python,python,piston,native + normalize,nlpd,-3.49038 +high-dimension-normalize-screening_Kriging_python,python,borehole,unit cube,fit_s,0.0140877 +high-dimension-normalize-screening_Kriging_python,python,borehole,unit cube,rmse,0.893341 +high-dimension-normalize-screening_Kriging_python,python,borehole,unit cube,q2,0.999599 +high-dimension-normalize-screening_Kriging_python,python,borehole,unit cube,nlpd,1.08433 +high-dimension-normalize-screening_Kriging_python,python,borehole,native + normalize,rmse,0.893341 +high-dimension-normalize-screening_Kriging_python,python,borehole,native + normalize,q2,0.999599 +high-dimension-normalize-screening_Kriging_python,python,borehole,native + normalize,nlpd,1.08433 +high-dimension-normalize-screening_Kriging_python,python,robot_arm,unit cube,fit_s,0.167482 +high-dimension-normalize-screening_Kriging_python,python,robot_arm,unit cube,rmse,0.358507 +high-dimension-normalize-screening_Kriging_python,python,robot_arm,unit cube,q2,0.571154 +high-dimension-normalize-screening_Kriging_python,python,robot_arm,unit cube,nlpd,0.355106 +high-dimension-normalize-screening_Kriging_python,python,robot_arm,native + normalize,rmse,0.358507 +high-dimension-normalize-screening_Kriging_python,python,robot_arm,native + normalize,q2,0.571154 +high-dimension-normalize-screening_Kriging_python,python,robot_arm,native + normalize,nlpd,0.355106 +high-dimension-normalize-screening_Kriging_python,python,dette_pepelyshev,unit cube,fit_s,0.0246282 +high-dimension-normalize-screening_Kriging_python,python,dette_pepelyshev,unit cube,rmse,0.424908 +high-dimension-normalize-screening_Kriging_python,python,dette_pepelyshev,unit cube,q2,0.997757 +high-dimension-normalize-screening_Kriging_python,python,dette_pepelyshev,unit cube,nlpd,0.510276 +high-dimension-normalize-screening_Kriging_python,python,dette_pepelyshev,native + normalize,rmse,0.424908 +high-dimension-normalize-screening_Kriging_python,python,dette_pepelyshev,native + normalize,q2,0.997757 +high-dimension-normalize-screening_Kriging_python,python,dette_pepelyshev,native + normalize,nlpd,0.510276 +high-dimension-normalize-screening_Kriging_python,python,wing_weight,unit cube,fit_s,0.0245481 +high-dimension-normalize-screening_Kriging_python,python,wing_weight,unit cube,rmse,0.809743 +high-dimension-normalize-screening_Kriging_python,python,wing_weight,unit cube,q2,0.999716 +high-dimension-normalize-screening_Kriging_python,python,wing_weight,unit cube,nlpd,1.07161 +high-dimension-normalize-screening_Kriging_python,python,wing_weight,native + normalize,rmse,0.809743 +high-dimension-normalize-screening_Kriging_python,python,wing_weight,native + normalize,q2,0.999716 +high-dimension-normalize-screening_Kriging_python,python,wing_weight,native + normalize,nlpd,1.07161 +high-dimension-normalize-screening_Kriging_python,python,welch,unit cube,fit_s,0.564758 +high-dimension-normalize-screening_Kriging_python,python,welch,unit cube,rmse,0.167729 +high-dimension-normalize-screening_Kriging_python,python,welch,unit cube,q2,0.993452 +high-dimension-normalize-screening_Kriging_python,python,welch,unit cube,nlpd,-0.429138 +high-dimension-normalize-screening_Kriging_python,python,welch,native + normalize,rmse,0.167692 +high-dimension-normalize-screening_Kriging_python,python,welch,native + normalize,q2,0.993454 +high-dimension-normalize-screening_Kriging_python,python,welch,native + normalize,nlpd,-0.429291 +deep-kernel-projected-branin_MLPKriging_python,python,projected_branin_10d,Kriging,rmse,4.42284 +deep-kernel-projected-branin_MLPKriging_python,python,projected_branin_10d,Kriging,q2,0.959122 +deep-kernel-projected-branin_MLPKriging_python,python,projected_branin_10d,Kriging,nlpd,2.89997 +deep-kernel-projected-branin_MLPKriging_python,python,projected_branin_10d,MLPKriging tanh BFGS,fit_s,0.130867 +deep-kernel-projected-branin_MLPKriging_python,python,projected_branin_10d,MLPKriging tanh BFGS,rmse,57.402 +deep-kernel-projected-branin_MLPKriging_python,python,projected_branin_10d,MLPKriging tanh BFGS,q2,-5.88553 +deep-kernel-projected-branin_MLPKriging_python,python,projected_branin_10d,MLPKriging tanh BFGS,nlpd,136.771 +deep-kernel-projected-branin_MLPKriging_python,python,projected_branin_10d,MLPKriging tanh BFGS10,fit_s,0.442791 +deep-kernel-projected-branin_MLPKriging_python,python,projected_branin_10d,MLPKriging tanh BFGS10,rmse,4.28606 +deep-kernel-projected-branin_MLPKriging_python,python,projected_branin_10d,MLPKriging tanh BFGS10,q2,0.961612 +deep-kernel-projected-branin_MLPKriging_python,python,projected_branin_10d,MLPKriging tanh BFGS10,nlpd,29.0854 +deep-kernel-projected-branin_MLPKriging_python,python,projected_branin_10d,MLPKriging selu BFGS,fit_s,0.139126 +deep-kernel-projected-branin_MLPKriging_python,python,projected_branin_10d,MLPKriging selu BFGS,rmse,58.9913 +deep-kernel-projected-branin_MLPKriging_python,python,projected_branin_10d,MLPKriging selu BFGS,q2,-6.2721 +deep-kernel-projected-branin_MLPKriging_python,python,projected_branin_10d,MLPKriging selu BFGS,nlpd,46.9788 +deep-kernel-projected-branin_MLPKriging_python,python,projected_branin_10d,MLPKriging selu BFGS10,fit_s,0.440179 +deep-kernel-projected-branin_MLPKriging_python,python,projected_branin_10d,MLPKriging selu BFGS10,rmse,4.39195 +deep-kernel-projected-branin_MLPKriging_python,python,projected_branin_10d,MLPKriging selu BFGS10,q2,0.959691 +deep-kernel-projected-branin_MLPKriging_python,python,projected_branin_10d,MLPKriging selu BFGS10,nlpd,10.6053 +large-n_Kriging-vecchia-nystrom_python,python,hartmann3/n=1500,exact,fit_s,11.1133 +aggregations-large-n_NestedKriging_python,python,hartmann3/n=1500,exact,fit_s,11.1133 +large-n_Kriging-vecchia-nystrom_python,python,hartmann3/n=1500,exact,rmse,0.000259233 +aggregations-large-n_NestedKriging_python,python,hartmann3/n=1500,exact,rmse,0.000259233 +large-n_Kriging-vecchia-nystrom_python,python,hartmann3/n=1500,exact,q2,1 +aggregations-large-n_NestedKriging_python,python,hartmann3/n=1500,exact,q2,1 +large-n_Kriging-vecchia-nystrom_python,python,hartmann3/n=1500,exact,nlpd,-6.92184 +aggregations-large-n_NestedKriging_python,python,hartmann3/n=1500,exact,nlpd,-6.92184 +large-n_Kriging-vecchia-nystrom_python,python,hartmann3/n=1500,vecchia(30),fit_s,5.43768 +large-n_Kriging-vecchia-nystrom_python,python,hartmann3/n=1500,vecchia(30),rmse,0.00353225 +large-n_Kriging-vecchia-nystrom_python,python,hartmann3/n=1500,vecchia(30),q2,0.999987 +large-n_Kriging-vecchia-nystrom_python,python,hartmann3/n=1500,vecchia(30),nlpd,-4.6266 +large-n_Kriging-vecchia-nystrom_python,python,hartmann3/n=5000,vecchia(30),fit_s,16.2077 +large-n_Kriging-vecchia-nystrom_python,python,hartmann3/n=5000,vecchia(30),rmse,0.000851738 +large-n_Kriging-vecchia-nystrom_python,python,hartmann3/n=5000,vecchia(30),q2,0.999999 +large-n_Kriging-vecchia-nystrom_python,python,hartmann3/n=5000,vecchia(30),nlpd,-5.95078 +large-n_Kriging-vecchia-nystrom_python,python,hartmann3/n=1500,nystrom(100),fit_s,2.04779 +large-n_Kriging-vecchia-nystrom_python,python,hartmann3/n=1500,nystrom(100),rmse,1.51127 +large-n_Kriging-vecchia-nystrom_python,python,hartmann3/n=1500,nystrom(100),q2,-1.39237 +large-n_Kriging-vecchia-nystrom_python,python,hartmann3/n=1500,nystrom(100),nlpd,1.14197e+24 +large-n_Kriging-vecchia-nystrom_python,python,hartmann3/n=5000,nystrom(100),fit_s,3.25622 +large-n_Kriging-vecchia-nystrom_python,python,hartmann3/n=5000,nystrom(100),rmse,9.95398 +large-n_Kriging-vecchia-nystrom_python,python,hartmann3/n=5000,nystrom(100),q2,-102.786 +large-n_Kriging-vecchia-nystrom_python,python,hartmann3/n=5000,nystrom(100),nlpd,4.95409e+25 +aggregations-large-n_NestedKriging_python,python,hartmann3/n=1500,nested NK (20 groups),fit_s,0.269932 +aggregations-large-n_NestedKriging_python,python,hartmann3/n=1500,nested NK (20 groups),rmse,0.0029908 +aggregations-large-n_NestedKriging_python,python,hartmann3/n=1500,nested NK (20 groups),q2,0.999991 +aggregations-large-n_NestedKriging_python,python,hartmann3/n=1500,nested NK (20 groups),nlpd,-4.85946 +aggregations-large-n_NestedKriging_python,python,hartmann3/n=5000,nested NK (20 groups),fit_s,2.87469 +aggregations-large-n_NestedKriging_python,python,hartmann3/n=5000,nested NK (20 groups),rmse,0.000222655 +aggregations-large-n_NestedKriging_python,python,hartmann3/n=5000,nested NK (20 groups),q2,1 +aggregations-large-n_NestedKriging_python,python,hartmann3/n=5000,nested NK (20 groups),nlpd,-7.44453 +aggregations-large-n_NestedKriging_python,python,hartmann3/n=1500,nested rBCM (20 groups),fit_s,0.25767 +aggregations-large-n_NestedKriging_python,python,hartmann3/n=1500,nested rBCM (20 groups),rmse,0.00304431 +aggregations-large-n_NestedKriging_python,python,hartmann3/n=1500,nested rBCM (20 groups),q2,0.99999 +aggregations-large-n_NestedKriging_python,python,hartmann3/n=1500,nested rBCM (20 groups),nlpd,-4.50201 +aggregations-large-n_NestedKriging_python,python,hartmann3/n=5000,nested rBCM (20 groups),fit_s,2.5369 +aggregations-large-n_NestedKriging_python,python,hartmann3/n=5000,nested rBCM (20 groups),rmse,0.000224517 +aggregations-large-n_NestedKriging_python,python,hartmann3/n=5000,nested rBCM (20 groups),q2,1 +aggregations-large-n_NestedKriging_python,python,hartmann3/n=5000,nested rBCM (20 groups),nlpd,-7.18182 +large-n_Kriging-vecchia-nystrom_python,python,hartmann3/n=1500,subset of data (500),fit_s,0.712767 +large-n_Kriging-vecchia-nystrom_python,python,hartmann3/n=1500,subset of data (500),rmse,0.00269153 +large-n_Kriging-vecchia-nystrom_python,python,hartmann3/n=1500,subset of data (500),q2,0.999992 +large-n_Kriging-vecchia-nystrom_python,python,hartmann3/n=1500,subset of data (500),nlpd,-4.68641 +large-n_Kriging-vecchia-nystrom_python,python,hartmann3/n=5000,subset of data (500),fit_s,0.950298 +large-n_Kriging-vecchia-nystrom_python,python,hartmann3/n=5000,subset of data (500),rmse,0.00247552 +large-n_Kriging-vecchia-nystrom_python,python,hartmann3/n=5000,subset of data (500),q2,0.999994 +large-n_Kriging-vecchia-nystrom_python,python,hartmann3/n=5000,subset of data (500),nlpd,-4.71359 +large-n_Kriging-vecchia-nystrom_python,python,otl_circuit/n=1500,exact,fit_s,10.6103 +aggregations-large-n_NestedKriging_python,python,otl_circuit/n=1500,exact,fit_s,10.6103 +large-n_Kriging-vecchia-nystrom_python,python,otl_circuit/n=1500,exact,rmse,9.81598e-05 +aggregations-large-n_NestedKriging_python,python,otl_circuit/n=1500,exact,rmse,9.81598e-05 +large-n_Kriging-vecchia-nystrom_python,python,otl_circuit/n=1500,exact,q2,1 +aggregations-large-n_NestedKriging_python,python,otl_circuit/n=1500,exact,q2,1 +large-n_Kriging-vecchia-nystrom_python,python,otl_circuit/n=1500,exact,nlpd,-8.21962 +aggregations-large-n_NestedKriging_python,python,otl_circuit/n=1500,exact,nlpd,-8.21962 +large-n_Kriging-vecchia-nystrom_python,python,otl_circuit/n=1500,vecchia(30),fit_s,4.73217 +large-n_Kriging-vecchia-nystrom_python,python,otl_circuit/n=1500,vecchia(30),rmse,0.00629979 +large-n_Kriging-vecchia-nystrom_python,python,otl_circuit/n=1500,vecchia(30),q2,0.999969 +large-n_Kriging-vecchia-nystrom_python,python,otl_circuit/n=1500,vecchia(30),nlpd,-3.95617 +large-n_Kriging-vecchia-nystrom_python,python,otl_circuit/n=5000,vecchia(30),fit_s,14.5549 +large-n_Kriging-vecchia-nystrom_python,python,otl_circuit/n=5000,vecchia(30),rmse,0.00312888 +large-n_Kriging-vecchia-nystrom_python,python,otl_circuit/n=5000,vecchia(30),q2,0.999992 +large-n_Kriging-vecchia-nystrom_python,python,otl_circuit/n=5000,vecchia(30),nlpd,-4.57612 +large-n_Kriging-vecchia-nystrom_python,python,otl_circuit/n=1500,nystrom(100),fit_s,1.47345 +large-n_Kriging-vecchia-nystrom_python,python,otl_circuit/n=1500,nystrom(100),rmse,0.392811 +large-n_Kriging-vecchia-nystrom_python,python,otl_circuit/n=1500,nystrom(100),q2,0.880908 +large-n_Kriging-vecchia-nystrom_python,python,otl_circuit/n=1500,nystrom(100),nlpd,7.06156e+22 +large-n_Kriging-vecchia-nystrom_python,python,otl_circuit/n=5000,nystrom(100),fit_s,4.69579 +large-n_Kriging-vecchia-nystrom_python,python,otl_circuit/n=5000,nystrom(100),rmse,1.11526 +large-n_Kriging-vecchia-nystrom_python,python,otl_circuit/n=5000,nystrom(100),q2,0.0400057 +large-n_Kriging-vecchia-nystrom_python,python,otl_circuit/n=5000,nystrom(100),nlpd,5.31511e+23 +aggregations-large-n_NestedKriging_python,python,otl_circuit/n=1500,nested NK (20 groups),fit_s,0.278129 +aggregations-large-n_NestedKriging_python,python,otl_circuit/n=1500,nested NK (20 groups),rmse,0.00226873 +aggregations-large-n_NestedKriging_python,python,otl_circuit/n=1500,nested NK (20 groups),q2,0.999996 +aggregations-large-n_NestedKriging_python,python,otl_circuit/n=1500,nested NK (20 groups),nlpd,-4.90542 +aggregations-large-n_NestedKriging_python,python,otl_circuit/n=5000,nested NK (20 groups),fit_s,3.62576 +aggregations-large-n_NestedKriging_python,python,otl_circuit/n=5000,nested NK (20 groups),rmse,0.000261686 +aggregations-large-n_NestedKriging_python,python,otl_circuit/n=5000,nested NK (20 groups),q2,1 +aggregations-large-n_NestedKriging_python,python,otl_circuit/n=5000,nested NK (20 groups),nlpd,-7.20623 +aggregations-large-n_NestedKriging_python,python,otl_circuit/n=1500,nested rBCM (20 groups),fit_s,0.315619 +aggregations-large-n_NestedKriging_python,python,otl_circuit/n=1500,nested rBCM (20 groups),rmse,0.00291869 +aggregations-large-n_NestedKriging_python,python,otl_circuit/n=1500,nested rBCM (20 groups),q2,0.999993 +aggregations-large-n_NestedKriging_python,python,otl_circuit/n=1500,nested rBCM (20 groups),nlpd,-2.55512 +aggregations-large-n_NestedKriging_python,python,otl_circuit/n=5000,nested rBCM (20 groups),fit_s,3.39037 +aggregations-large-n_NestedKriging_python,python,otl_circuit/n=5000,nested rBCM (20 groups),rmse,0.000278149 +aggregations-large-n_NestedKriging_python,python,otl_circuit/n=5000,nested rBCM (20 groups),q2,1 +aggregations-large-n_NestedKriging_python,python,otl_circuit/n=5000,nested rBCM (20 groups),nlpd,-4.34927 +large-n_Kriging-vecchia-nystrom_python,python,otl_circuit/n=1500,subset of data (500),fit_s,0.672431 +large-n_Kriging-vecchia-nystrom_python,python,otl_circuit/n=1500,subset of data (500),rmse,0.0004286 +large-n_Kriging-vecchia-nystrom_python,python,otl_circuit/n=1500,subset of data (500),q2,1 +large-n_Kriging-vecchia-nystrom_python,python,otl_circuit/n=1500,subset of data (500),nlpd,-6.72678 +large-n_Kriging-vecchia-nystrom_python,python,otl_circuit/n=5000,subset of data (500),fit_s,0.692672 +large-n_Kriging-vecchia-nystrom_python,python,otl_circuit/n=5000,subset of data (500),rmse,0.000446879 +large-n_Kriging-vecchia-nystrom_python,python,otl_circuit/n=5000,subset of data (500),q2,1 +large-n_Kriging-vecchia-nystrom_python,python,otl_circuit/n=5000,subset of data (500),nlpd,-6.78844 +aggregations-large-n_NestedKriging_python,python,branin/nested,NK,rmse,0.0343984 +aggregations-large-n_NestedKriging_python,python,branin/nested,NK,q2,1 +aggregations-large-n_NestedKriging_python,python,branin/nested,NK,nlpd,2.00082e+14 +aggregations-large-n_NestedKriging_python,python,branin/nested,PoE,rmse,0.0354265 +aggregations-large-n_NestedKriging_python,python,branin/nested,PoE,q2,1 +aggregations-large-n_NestedKriging_python,python,branin/nested,PoE,nlpd,-1.64533 +aggregations-large-n_NestedKriging_python,python,branin/nested,gPoE,rmse,0.0354265 +aggregations-large-n_NestedKriging_python,python,branin/nested,gPoE,q2,1 +aggregations-large-n_NestedKriging_python,python,branin/nested,gPoE,nlpd,-0.621836 +aggregations-large-n_NestedKriging_python,python,branin/nested,BCM,rmse,0.0354196 +aggregations-large-n_NestedKriging_python,python,branin/nested,BCM,q2,1 +aggregations-large-n_NestedKriging_python,python,branin/nested,BCM,nlpd,-1.64533 +aggregations-large-n_NestedKriging_python,python,branin/nested,rBCM,rmse,0.035287 +aggregations-large-n_NestedKriging_python,python,branin/nested,rBCM,q2,1 +aggregations-large-n_NestedKriging_python,python,branin/nested,rBCM,nlpd,-2.61369 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,branin,pylibkriging,q2,0.999944 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,branin,pylibkriging,nlpd,0.137225 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,branin,pylibkriging,fit_s,0.00261593 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,branin,pylibkriging BFGS10,q2,0.999945 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,branin,pylibkriging BFGS10,nlpd,0.137254 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,branin,pylibkriging BFGS10,fit_s,0.02301 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,branin,scikit-learn,q2,0.999597 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,branin,scikit-learn,nlpd,0.928508 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,branin,scikit-learn,fit_s,0.169693 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,branin,SMT,q2,0.999944 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,branin,SMT,nlpd,0.137259 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,branin,SMT,fit_s,0.833295 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,branin,OpenTURNS,q2,0.998221 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,branin,OpenTURNS,nlpd,1.55599 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,branin,OpenTURNS,fit_s,0.00490117 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,six_hump_camel,pylibkriging,q2,0.999418 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,six_hump_camel,pylibkriging,nlpd,0.0429342 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,six_hump_camel,pylibkriging,fit_s,0.00324678 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,six_hump_camel,pylibkriging BFGS10,q2,0.999418 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,six_hump_camel,pylibkriging BFGS10,nlpd,0.0429386 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,six_hump_camel,pylibkriging BFGS10,fit_s,0.060586 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,six_hump_camel,scikit-learn,q2,0.0163311 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,six_hump_camel,scikit-learn,nlpd,4.63763 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,six_hump_camel,scikit-learn,fit_s,0.206083 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,six_hump_camel,SMT,q2,0.999594 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,six_hump_camel,SMT,nlpd,-0.121007 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,six_hump_camel,SMT,fit_s,1.03758 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,six_hump_camel,OpenTURNS,q2,0.956713 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,six_hump_camel,OpenTURNS,nlpd,2.55446 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,six_hump_camel,OpenTURNS,fit_s,0.00741315 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann3,pylibkriging,q2,0.986148 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann3,pylibkriging,nlpd,-0.847247 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann3,pylibkriging,fit_s,0.0169713 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann3,pylibkriging BFGS10,q2,0.986148 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann3,pylibkriging BFGS10,nlpd,-0.847249 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann3,pylibkriging BFGS10,fit_s,0.139588 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann3,scikit-learn,q2,0.983889 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann3,scikit-learn,nlpd,-0.782697 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann3,scikit-learn,fit_s,0.188796 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann3,SMT,q2,0.986148 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann3,SMT,nlpd,-0.847249 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann3,SMT,fit_s,1.02907 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann3,OpenTURNS,q2,0.984712 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann3,OpenTURNS,nlpd,-0.809851 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann3,OpenTURNS,fit_s,0.0153861 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,ishigami,pylibkriging,q2,0.259556 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,ishigami,pylibkriging,nlpd,4.38991 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,ishigami,pylibkriging,fit_s,0.0256422 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,ishigami,pylibkriging BFGS10,q2,0.935722 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,ishigami,pylibkriging BFGS10,nlpd,0.857443 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,ishigami,pylibkriging BFGS10,fit_s,0.308839 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,ishigami,scikit-learn,q2,0.831142 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,ishigami,scikit-learn,nlpd,1.5455 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,ishigami,scikit-learn,fit_s,0.51733 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,ishigami,SMT,q2,0.51196 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,ishigami,SMT,nlpd,3.31349 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,ishigami,SMT,fit_s,2.77225 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,ishigami,OpenTURNS,q2,0.887251 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,ishigami,OpenTURNS,nlpd,1.39348 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,ishigami,OpenTURNS,fit_s,0.0499659 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,friedman,pylibkriging,q2,0.999771 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,friedman,pylibkriging,nlpd,-1.29726 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,friedman,pylibkriging,fit_s,0.0181558 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,friedman,pylibkriging BFGS10,q2,0.999771 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,friedman,pylibkriging BFGS10,nlpd,-1.29727 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,friedman,pylibkriging BFGS10,fit_s,0.215106 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,friedman,scikit-learn,q2,0.999754 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,friedman,scikit-learn,nlpd,-1.37702 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,friedman,scikit-learn,fit_s,0.283978 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,friedman,SMT,q2,0.999899 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,friedman,SMT,nlpd,-1.871 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,friedman,SMT,fit_s,4.3249 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,friedman,OpenTURNS,q2,0.990862 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,friedman,OpenTURNS,nlpd,0.469828 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,friedman,OpenTURNS,fit_s,0.0293689 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,otl_circuit,pylibkriging,q2,0.999922 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,otl_circuit,pylibkriging,nlpd,-3.46228 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,otl_circuit,pylibkriging,fit_s,0.013788 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,otl_circuit,pylibkriging BFGS10,q2,0.999922 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,otl_circuit,pylibkriging BFGS10,nlpd,-3.46227 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,otl_circuit,pylibkriging BFGS10,fit_s,0.141071 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,otl_circuit,scikit-learn,q2,0.999906 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,otl_circuit,scikit-learn,nlpd,-3.43062 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,otl_circuit,scikit-learn,fit_s,0.51218 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,otl_circuit,SMT,q2,0.999954 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,otl_circuit,SMT,nlpd,-3.35016 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,otl_circuit,SMT,fit_s,7.1224 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,otl_circuit,OpenTURNS,q2,0.99777 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,otl_circuit,OpenTURNS,nlpd,-1.61723 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,otl_circuit,OpenTURNS,fit_s,0.0603108 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,piston,pylibkriging,q2,0.997498 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,piston,pylibkriging,nlpd,-3.80667 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,piston,pylibkriging,fit_s,0.0395648 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,piston,pylibkriging BFGS10,q2,0.997498 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,piston,pylibkriging BFGS10,nlpd,-3.8067 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,piston,pylibkriging BFGS10,fit_s,0.327989 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,piston,scikit-learn,q2,0.998684 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,piston,scikit-learn,nlpd,-4.10497 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,piston,scikit-learn,fit_s,1.11386 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,piston,SMT,q2,0.997511 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,piston,SMT,nlpd,-3.86618 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,piston,SMT,fit_s,12.649 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,piston,OpenTURNS,q2,0.990631 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,piston,OpenTURNS,nlpd,-2.9906 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,piston,OpenTURNS,fit_s,0.11631 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,borehole,pylibkriging,q2,0.999694 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,borehole,pylibkriging,nlpd,1.00118 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,borehole,pylibkriging,fit_s,0.0149446 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,borehole,pylibkriging BFGS10,q2,0.999694 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,borehole,pylibkriging BFGS10,nlpd,1.00121 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,borehole,pylibkriging BFGS10,fit_s,0.164608 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,borehole,scikit-learn,q2,0.999929 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,borehole,scikit-learn,nlpd,0.194455 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,borehole,scikit-learn,fit_s,1.04563 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,borehole,SMT,q2,0.999925 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,borehole,SMT,nlpd,0.279122 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,borehole,SMT,fit_s,11.8531 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,borehole,OpenTURNS,q2,0.996206 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,borehole,OpenTURNS,nlpd,2.55431 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,borehole,OpenTURNS,fit_s,0.12395 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,wing_weight,pylibkriging,q2,0.99972 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,wing_weight,pylibkriging,nlpd,1.11517 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,wing_weight,pylibkriging,fit_s,0.0290489 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,wing_weight,pylibkriging BFGS10,q2,0.99972 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,wing_weight,pylibkriging BFGS10,nlpd,1.11512 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,wing_weight,pylibkriging BFGS10,fit_s,0.314707 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,wing_weight,scikit-learn,q2,0.998048 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,wing_weight,scikit-learn,nlpd,2.57304 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,wing_weight,scikit-learn,fit_s,0.729278 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,wing_weight,SMT,q2,0.999898 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,wing_weight,SMT,nlpd,0.604023 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,wing_weight,SMT,fit_s,25.8262 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,wing_weight,OpenTURNS,q2,0.993145 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,wing_weight,OpenTURNS,nlpd,2.83627 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,wing_weight,OpenTURNS,fit_s,0.280434 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann6,pylibkriging,q2,0.422938 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann6,pylibkriging,nlpd,0.0965167 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann6,pylibkriging,fit_s,0.0562823 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann6,pylibkriging BFGS10,q2,0.72352 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann6,pylibkriging BFGS10,nlpd,-0.156898 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann6,pylibkriging BFGS10,fit_s,0.822745 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann6,scikit-learn,q2,0.69049 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann6,scikit-learn,nlpd,-0.137845 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann6,scikit-learn,fit_s,1.3155 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann6,SMT,q2,0.708463 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann6,SMT,nlpd,-0.0177081 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann6,SMT,fit_s,12.0578 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann6,OpenTURNS,q2,0.696177 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann6,OpenTURNS,nlpd,-0.132838 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann6,OpenTURNS,fit_s,0.133332 diff --git a/tools/baseline/metrics-release.csv b/tools/baseline/metrics-release.csv index 5c4790d..d9414f2 100644 --- a/tools/baseline/metrics-release.csv +++ b/tools/baseline/metrics-release.csv @@ -1,924 +1,924 @@ -suite,lang,case,package,metric,value -python_Kriging_kernels-trends-interpolation,python,forrester/gauss/constant,pylibkriging,fit_s,0.0039537 -python_Kriging_kernels-trends-interpolation,python,forrester/gauss/constant,pylibkriging,rmse,0.0848271 -python_Kriging_kernels-trends-interpolation,python,forrester/gauss/constant,pylibkriging,q2,0.999634 -python_Kriging_kernels-trends-interpolation,python,forrester/gauss/constant,pylibkriging,nlpd,-3.06486 -python_Kriging_kernels-trends-interpolation,python,forrester/gauss/linear,pylibkriging,fit_s,0.0008564 -python_Kriging_kernels-trends-interpolation,python,forrester/gauss/linear,pylibkriging,rmse,0.0855468 -python_Kriging_kernels-trends-interpolation,python,forrester/gauss/linear,pylibkriging,q2,0.999628 -python_Kriging_kernels-trends-interpolation,python,forrester/gauss/linear,pylibkriging,nlpd,-3.11029 -python_Kriging_kernels-trends-interpolation,python,forrester/gauss/interactive,pylibkriging,fit_s,0.000818729 -python_Kriging_kernels-trends-interpolation,python,forrester/gauss/interactive,pylibkriging,rmse,0.0855468 -python_Kriging_kernels-trends-interpolation,python,forrester/gauss/interactive,pylibkriging,q2,0.999628 -python_Kriging_kernels-trends-interpolation,python,forrester/gauss/interactive,pylibkriging,nlpd,-3.11029 -python_Kriging_kernels-trends-interpolation,python,forrester/gauss/quadratic,pylibkriging,fit_s,0.000473261 -python_Kriging_kernels-trends-interpolation,python,forrester/gauss/quadratic,pylibkriging,rmse,1.57249 -python_Kriging_kernels-trends-interpolation,python,forrester/gauss/quadratic,pylibkriging,q2,0.874394 -python_Kriging_kernels-trends-interpolation,python,forrester/gauss/quadratic,pylibkriging,nlpd,5.33879 -python_Kriging_kernels-trends-interpolation,python,forrester/exp/constant,pylibkriging,fit_s,0.00042963 -python_Kriging_kernels-trends-interpolation,python,forrester/exp/constant,pylibkriging,rmse,0.721582 -python_Kriging_kernels-trends-interpolation,python,forrester/exp/constant,pylibkriging,q2,0.973551 -python_Kriging_kernels-trends-interpolation,python,forrester/exp/constant,pylibkriging,nlpd,1.54551 -python_Kriging_kernels-trends-interpolation,python,forrester/exp/linear,pylibkriging,fit_s,0.000402451 -python_Kriging_kernels-trends-interpolation,python,forrester/exp/linear,pylibkriging,rmse,0.793413 -python_Kriging_kernels-trends-interpolation,python,forrester/exp/linear,pylibkriging,q2,0.968024 -python_Kriging_kernels-trends-interpolation,python,forrester/exp/linear,pylibkriging,nlpd,1.50639 -python_Kriging_kernels-trends-interpolation,python,forrester/exp/interactive,pylibkriging,fit_s,0.000428438 -python_Kriging_kernels-trends-interpolation,python,forrester/exp/interactive,pylibkriging,rmse,0.793413 -python_Kriging_kernels-trends-interpolation,python,forrester/exp/interactive,pylibkriging,q2,0.968024 -python_Kriging_kernels-trends-interpolation,python,forrester/exp/interactive,pylibkriging,nlpd,1.50639 -python_Kriging_kernels-trends-interpolation,python,forrester/exp/quadratic,pylibkriging,fit_s,0.000418425 -python_Kriging_kernels-trends-interpolation,python,forrester/exp/quadratic,pylibkriging,rmse,0.723267 -python_Kriging_kernels-trends-interpolation,python,forrester/exp/quadratic,pylibkriging,q2,0.973428 -python_Kriging_kernels-trends-interpolation,python,forrester/exp/quadratic,pylibkriging,nlpd,1.28782 -python_Kriging_kernels-trends-interpolation,python,forrester/matern3_2/constant,pylibkriging,fit_s,0.000519753 -python_Kriging_kernels-trends-interpolation,python,forrester/matern3_2/constant,pylibkriging,rmse,0.334007 -python_Kriging_kernels-trends-interpolation,python,forrester/matern3_2/constant,pylibkriging,q2,0.994333 -python_Kriging_kernels-trends-interpolation,python,forrester/matern3_2/constant,pylibkriging,nlpd,-0.227715 -python_Kriging_kernels-trends-interpolation,python,forrester/matern3_2/linear,pylibkriging,fit_s,0.000559092 -python_Kriging_kernels-trends-interpolation,python,forrester/matern3_2/linear,pylibkriging,rmse,0.364477 -python_Kriging_kernels-trends-interpolation,python,forrester/matern3_2/linear,pylibkriging,q2,0.993252 -python_Kriging_kernels-trends-interpolation,python,forrester/matern3_2/linear,pylibkriging,nlpd,-0.187262 -python_Kriging_kernels-trends-interpolation,python,forrester/matern3_2/interactive,pylibkriging,fit_s,0.00060463 -python_Kriging_kernels-trends-interpolation,python,forrester/matern3_2/interactive,pylibkriging,rmse,0.364477 -python_Kriging_kernels-trends-interpolation,python,forrester/matern3_2/interactive,pylibkriging,q2,0.993252 -python_Kriging_kernels-trends-interpolation,python,forrester/matern3_2/interactive,pylibkriging,nlpd,-0.187262 -python_Kriging_kernels-trends-interpolation,python,forrester/matern3_2/quadratic,pylibkriging,fit_s,0.000493765 -python_Kriging_kernels-trends-interpolation,python,forrester/matern3_2/quadratic,pylibkriging,rmse,0.435609 -python_Kriging_kernels-trends-interpolation,python,forrester/matern3_2/quadratic,pylibkriging,q2,0.990361 -python_Kriging_kernels-trends-interpolation,python,forrester/matern3_2/quadratic,pylibkriging,nlpd,-0.183076 -python_Kriging_kernels-trends-interpolation,python,forrester/matern5_2/constant,pylibkriging,fit_s,0.000758886 -python_Kriging_kernels-trends-interpolation,python,forrester/matern5_2/constant,pylibkriging,rmse,0.281945 -python_Kriging_kernels-trends-interpolation,python,forrester/matern5_2/constant,pylibkriging,q2,0.995962 -python_Kriging_kernels-trends-interpolation,python,forrester/matern5_2/constant,pylibkriging,nlpd,-1.01945 -python_Kriging_kernels-trends-interpolation,python,forrester/matern5_2/constant,pylibkriging BFGS10,rmse,0.281945 -python_Kriging_kernels-trends-interpolation,python,forrester/matern5_2/constant,pylibkriging BFGS10,q2,0.995962 -python_Kriging_kernels-trends-interpolation,python,forrester/matern5_2/constant,pylibkriging BFGS10,nlpd,-1.01945 -python_Kriging_kernels-trends-interpolation,python,forrester/matern5_2/linear,pylibkriging,fit_s,0.000576973 -python_Kriging_kernels-trends-interpolation,python,forrester/matern5_2/linear,pylibkriging,rmse,0.298959 -python_Kriging_kernels-trends-interpolation,python,forrester/matern5_2/linear,pylibkriging,q2,0.99546 -python_Kriging_kernels-trends-interpolation,python,forrester/matern5_2/linear,pylibkriging,nlpd,-0.95551 -python_Kriging_kernels-trends-interpolation,python,forrester/matern5_2/interactive,pylibkriging,fit_s,0.00054884 -python_Kriging_kernels-trends-interpolation,python,forrester/matern5_2/interactive,pylibkriging,rmse,0.298959 -python_Kriging_kernels-trends-interpolation,python,forrester/matern5_2/interactive,pylibkriging,q2,0.99546 -python_Kriging_kernels-trends-interpolation,python,forrester/matern5_2/interactive,pylibkriging,nlpd,-0.95551 -python_Kriging_kernels-trends-interpolation,python,forrester/matern5_2/quadratic,pylibkriging,fit_s,0.000477076 -python_Kriging_kernels-trends-interpolation,python,forrester/matern5_2/quadratic,pylibkriging,rmse,0.365039 -python_Kriging_kernels-trends-interpolation,python,forrester/matern5_2/quadratic,pylibkriging,q2,0.993231 -python_Kriging_kernels-trends-interpolation,python,forrester/matern5_2/quadratic,pylibkriging,nlpd,-0.790032 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/gauss/constant,pylibkriging,fit_s,0.0016222 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/gauss/constant,pylibkriging,rmse,0.191274 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/gauss/constant,pylibkriging,q2,0.978742 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/gauss/constant,pylibkriging,nlpd,-0.704948 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/gauss/linear,pylibkriging,fit_s,0.00240469 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/gauss/linear,pylibkriging,rmse,0.236028 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/gauss/linear,pylibkriging,q2,0.96763 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/gauss/linear,pylibkriging,nlpd,-0.580993 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/gauss/interactive,pylibkriging,fit_s,0.00205922 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/gauss/interactive,pylibkriging,rmse,0.236028 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/gauss/interactive,pylibkriging,q2,0.96763 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/gauss/interactive,pylibkriging,nlpd,-0.580993 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/gauss/quadratic,pylibkriging,fit_s,0.00200701 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/gauss/quadratic,pylibkriging,rmse,0.150914 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/gauss/quadratic,pylibkriging,q2,0.986767 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/gauss/quadratic,pylibkriging,nlpd,-0.458431 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/exp/constant,pylibkriging,fit_s,0.00123 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/exp/constant,pylibkriging,rmse,0.196586 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/exp/constant,pylibkriging,q2,0.977545 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/exp/constant,pylibkriging,nlpd,-0.0621192 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/exp/linear,pylibkriging,fit_s,0.00172782 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/exp/linear,pylibkriging,rmse,0.20979 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/exp/linear,pylibkriging,q2,0.974427 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/exp/linear,pylibkriging,nlpd,-0.0466568 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/exp/interactive,pylibkriging,fit_s,0.0017736 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/exp/interactive,pylibkriging,rmse,0.20979 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/exp/interactive,pylibkriging,q2,0.974427 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/exp/interactive,pylibkriging,nlpd,-0.0466568 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/exp/quadratic,pylibkriging,fit_s,0.00394821 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/exp/quadratic,pylibkriging,rmse,0.252565 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/exp/quadratic,pylibkriging,q2,0.962935 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/exp/quadratic,pylibkriging,nlpd,0.220609 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern3_2/constant,pylibkriging,fit_s,0.000825167 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern3_2/constant,pylibkriging,rmse,0.177253 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern3_2/constant,pylibkriging,q2,0.981744 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern3_2/constant,pylibkriging,nlpd,-0.482212 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern3_2/linear,pylibkriging,fit_s,0.000857115 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern3_2/linear,pylibkriging,rmse,0.211499 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern3_2/linear,pylibkriging,q2,0.974009 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern3_2/linear,pylibkriging,nlpd,-0.375844 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern3_2/interactive,pylibkriging,fit_s,0.00114012 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern3_2/interactive,pylibkriging,rmse,0.211499 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern3_2/interactive,pylibkriging,q2,0.974009 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern3_2/interactive,pylibkriging,nlpd,-0.375844 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern3_2/quadratic,pylibkriging,fit_s,0.00237894 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern3_2/quadratic,pylibkriging,rmse,0.184528 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern3_2/quadratic,pylibkriging,q2,0.980215 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern3_2/quadratic,pylibkriging,nlpd,-0.171795 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern5_2/constant,pylibkriging,fit_s,0.000836849 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern5_2/constant,pylibkriging,rmse,0.177638 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern5_2/constant,pylibkriging,q2,0.981665 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern5_2/constant,pylibkriging,nlpd,-0.575421 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern5_2/constant,pylibkriging BFGS10,rmse,0.177638 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern5_2/constant,pylibkriging BFGS10,q2,0.981665 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern5_2/constant,pylibkriging BFGS10,nlpd,-0.575422 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern5_2/linear,pylibkriging,fit_s,0.000859499 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern5_2/linear,pylibkriging,rmse,0.219759 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern5_2/linear,pylibkriging,q2,0.971939 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern5_2/linear,pylibkriging,nlpd,-0.458933 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern5_2/interactive,pylibkriging,fit_s,0.00080061 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern5_2/interactive,pylibkriging,rmse,0.219759 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern5_2/interactive,pylibkriging,q2,0.971939 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern5_2/interactive,pylibkriging,nlpd,-0.458933 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern5_2/quadratic,pylibkriging,fit_s,0.00249147 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern5_2/quadratic,pylibkriging,rmse,0.169941 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern5_2/quadratic,pylibkriging,q2,0.983219 -python_Kriging_kernels-trends-interpolation,python,gramacy_lee_2012/matern5_2/quadratic,pylibkriging,nlpd,-0.28938 -python_Kriging_kernels-trends-interpolation,python,branin/gauss/constant,pylibkriging,fit_s,0.00172019 -python_Kriging_kernels-trends-interpolation,python,branin/gauss/constant,pylibkriging,rmse,7.2629 -python_Kriging_kernels-trends-interpolation,python,branin/gauss/constant,pylibkriging,q2,0.980655 -python_Kriging_kernels-trends-interpolation,python,branin/gauss/constant,pylibkriging,nlpd,4.3152 -python_Kriging_kernels-trends-interpolation,python,branin/gauss/linear,pylibkriging,fit_s,0.00188136 -python_Kriging_kernels-trends-interpolation,python,branin/gauss/linear,pylibkriging,rmse,7.25733 -python_Kriging_kernels-trends-interpolation,python,branin/gauss/linear,pylibkriging,q2,0.980685 -python_Kriging_kernels-trends-interpolation,python,branin/gauss/linear,pylibkriging,nlpd,4.30813 -python_Kriging_kernels-trends-interpolation,python,branin/gauss/interactive,pylibkriging,fit_s,0.00456238 -python_Kriging_kernels-trends-interpolation,python,branin/gauss/interactive,pylibkriging,rmse,7.2576 -python_Kriging_kernels-trends-interpolation,python,branin/gauss/interactive,pylibkriging,q2,0.980683 -python_Kriging_kernels-trends-interpolation,python,branin/gauss/interactive,pylibkriging,nlpd,4.30176 -python_Kriging_kernels-trends-interpolation,python,branin/gauss/quadratic,pylibkriging,fit_s,0.00182772 -python_Kriging_kernels-trends-interpolation,python,branin/gauss/quadratic,pylibkriging,rmse,7.27808 -python_Kriging_kernels-trends-interpolation,python,branin/gauss/quadratic,pylibkriging,q2,0.980574 -python_Kriging_kernels-trends-interpolation,python,branin/gauss/quadratic,pylibkriging,nlpd,4.29399 -python_Kriging_kernels-trends-interpolation,python,branin/exp/constant,pylibkriging,fit_s,0.00215006 -python_Kriging_kernels-trends-interpolation,python,branin/exp/constant,pylibkriging,rmse,14.5887 -python_Kriging_kernels-trends-interpolation,python,branin/exp/constant,pylibkriging,q2,0.921947 -python_Kriging_kernels-trends-interpolation,python,branin/exp/constant,pylibkriging,nlpd,4.09457 -python_Kriging_kernels-trends-interpolation,python,branin/exp/linear,pylibkriging,fit_s,0.00210071 -python_Kriging_kernels-trends-interpolation,python,branin/exp/linear,pylibkriging,rmse,14.9622 -python_Kriging_kernels-trends-interpolation,python,branin/exp/linear,pylibkriging,q2,0.9179 -python_Kriging_kernels-trends-interpolation,python,branin/exp/linear,pylibkriging,nlpd,4.0909 -python_Kriging_kernels-trends-interpolation,python,branin/exp/interactive,pylibkriging,fit_s,0.00220799 -python_Kriging_kernels-trends-interpolation,python,branin/exp/interactive,pylibkriging,rmse,11.6963 -python_Kriging_kernels-trends-interpolation,python,branin/exp/interactive,pylibkriging,q2,0.94983 -python_Kriging_kernels-trends-interpolation,python,branin/exp/interactive,pylibkriging,nlpd,3.79671 -python_Kriging_kernels-trends-interpolation,python,branin/exp/quadratic,pylibkriging,fit_s,0.00237751 -python_Kriging_kernels-trends-interpolation,python,branin/exp/quadratic,pylibkriging,rmse,9.33909 -python_Kriging_kernels-trends-interpolation,python,branin/exp/quadratic,pylibkriging,q2,0.968014 -python_Kriging_kernels-trends-interpolation,python,branin/exp/quadratic,pylibkriging,nlpd,3.62362 -python_Kriging_kernels-trends-interpolation,python,branin/matern3_2/constant,pylibkriging,fit_s,0.00252843 -python_Kriging_kernels-trends-interpolation,python,branin/matern3_2/constant,pylibkriging,rmse,3.24796 -python_Kriging_kernels-trends-interpolation,python,branin/matern3_2/constant,pylibkriging,q2,0.996131 -python_Kriging_kernels-trends-interpolation,python,branin/matern3_2/constant,pylibkriging,nlpd,2.30444 -python_Kriging_kernels-trends-interpolation,python,branin/matern3_2/linear,pylibkriging,fit_s,0.00200725 -python_Kriging_kernels-trends-interpolation,python,branin/matern3_2/linear,pylibkriging,rmse,3.14402 -python_Kriging_kernels-trends-interpolation,python,branin/matern3_2/linear,pylibkriging,q2,0.996375 -python_Kriging_kernels-trends-interpolation,python,branin/matern3_2/linear,pylibkriging,nlpd,2.28067 -python_Kriging_kernels-trends-interpolation,python,branin/matern3_2/interactive,pylibkriging,fit_s,0.00414944 -python_Kriging_kernels-trends-interpolation,python,branin/matern3_2/interactive,pylibkriging,rmse,3.19226 -python_Kriging_kernels-trends-interpolation,python,branin/matern3_2/interactive,pylibkriging,q2,0.996263 -python_Kriging_kernels-trends-interpolation,python,branin/matern3_2/interactive,pylibkriging,nlpd,2.20359 -python_Kriging_kernels-trends-interpolation,python,branin/matern3_2/quadratic,pylibkriging,fit_s,0.00253224 -python_Kriging_kernels-trends-interpolation,python,branin/matern3_2/quadratic,pylibkriging,rmse,3.01232 -python_Kriging_kernels-trends-interpolation,python,branin/matern3_2/quadratic,pylibkriging,q2,0.996672 -python_Kriging_kernels-trends-interpolation,python,branin/matern3_2/quadratic,pylibkriging,nlpd,2.11627 -python_Kriging_kernels-trends-interpolation,python,branin/matern5_2/constant,pylibkriging,fit_s,0.00171685 -python_Kriging_kernels-trends-interpolation,python,branin/matern5_2/constant,pylibkriging,rmse,0.52012 -python_Kriging_kernels-trends-interpolation,python,branin/matern5_2/constant,pylibkriging,q2,0.999901 -python_Kriging_kernels-trends-interpolation,python,branin/matern5_2/constant,pylibkriging,nlpd,0.283399 -python_Kriging_kernels-trends-interpolation,python,branin/matern5_2/constant,pylibkriging BFGS10,rmse,0.52012 -python_Kriging_kernels-trends-interpolation,python,branin/matern5_2/constant,pylibkriging BFGS10,q2,0.999901 -python_Kriging_kernels-trends-interpolation,python,branin/matern5_2/constant,pylibkriging BFGS10,nlpd,0.283395 -python_Kriging_kernels-trends-interpolation,python,branin/matern5_2/linear,pylibkriging,fit_s,0.0021255 -python_Kriging_kernels-trends-interpolation,python,branin/matern5_2/linear,pylibkriging,rmse,0.460868 -python_Kriging_kernels-trends-interpolation,python,branin/matern5_2/linear,pylibkriging,q2,0.999922 -python_Kriging_kernels-trends-interpolation,python,branin/matern5_2/linear,pylibkriging,nlpd,0.209741 -python_Kriging_kernels-trends-interpolation,python,branin/matern5_2/interactive,pylibkriging,fit_s,0.00307012 -python_Kriging_kernels-trends-interpolation,python,branin/matern5_2/interactive,pylibkriging,rmse,0.498484 -python_Kriging_kernels-trends-interpolation,python,branin/matern5_2/interactive,pylibkriging,q2,0.999909 -python_Kriging_kernels-trends-interpolation,python,branin/matern5_2/interactive,pylibkriging,nlpd,0.239079 -python_Kriging_kernels-trends-interpolation,python,branin/matern5_2/quadratic,pylibkriging,fit_s,0.0092895 -python_Kriging_kernels-trends-interpolation,python,branin/matern5_2/quadratic,pylibkriging,rmse,0.67472 -python_Kriging_kernels-trends-interpolation,python,branin/matern5_2/quadratic,pylibkriging,q2,0.999833 -python_Kriging_kernels-trends-interpolation,python,branin/matern5_2/quadratic,pylibkriging,nlpd,0.419266 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/gauss/constant,pylibkriging,fit_s,0.00194049 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/gauss/constant,pylibkriging,rmse,18.9501 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/gauss/constant,pylibkriging,q2,0.536354 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/gauss/constant,pylibkriging,nlpd,84.151 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/gauss/linear,pylibkriging,fit_s,0.00170755 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/gauss/linear,pylibkriging,rmse,18.9579 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/gauss/linear,pylibkriging,q2,0.535976 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/gauss/linear,pylibkriging,nlpd,44.8696 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/gauss/interactive,pylibkriging,fit_s,0.00180864 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/gauss/interactive,pylibkriging,rmse,18.9819 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/gauss/interactive,pylibkriging,q2,0.5348 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/gauss/interactive,pylibkriging,nlpd,35.2765 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/gauss/quadratic,pylibkriging,fit_s,0.00178218 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/gauss/quadratic,pylibkriging,rmse,10.9058 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/gauss/quadratic,pylibkriging,q2,0.846441 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/gauss/quadratic,pylibkriging,nlpd,4.92017 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/exp/constant,pylibkriging,fit_s,0.000977039 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/exp/constant,pylibkriging,rmse,16.9653 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/exp/constant,pylibkriging,q2,0.628393 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/exp/constant,pylibkriging,nlpd,4.48595 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/exp/linear,pylibkriging,fit_s,0.00107837 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/exp/linear,pylibkriging,rmse,17.1781 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/exp/linear,pylibkriging,q2,0.619012 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/exp/linear,pylibkriging,nlpd,4.47061 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/exp/interactive,pylibkriging,fit_s,0.00123405 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/exp/interactive,pylibkriging,rmse,17.1856 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/exp/interactive,pylibkriging,q2,0.618679 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/exp/interactive,pylibkriging,nlpd,4.43133 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/exp/quadratic,pylibkriging,fit_s,0.00120115 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/exp/quadratic,pylibkriging,rmse,7.99484 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/exp/quadratic,pylibkriging,q2,0.917476 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/exp/quadratic,pylibkriging,nlpd,3.35612 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern3_2/constant,pylibkriging,fit_s,0.00264788 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern3_2/constant,pylibkriging,rmse,18.4958 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern3_2/constant,pylibkriging,q2,0.558321 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern3_2/constant,pylibkriging,nlpd,18.8349 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern3_2/linear,pylibkriging,fit_s,0.00251079 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern3_2/linear,pylibkriging,rmse,18.5562 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern3_2/linear,pylibkriging,q2,0.555433 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern3_2/linear,pylibkriging,nlpd,14.9494 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern3_2/interactive,pylibkriging,fit_s,0.00148797 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern3_2/interactive,pylibkriging,rmse,18.561 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern3_2/interactive,pylibkriging,q2,0.555202 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern3_2/interactive,pylibkriging,nlpd,12.9923 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern3_2/quadratic,pylibkriging,fit_s,0.00289869 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern3_2/quadratic,pylibkriging,rmse,8.91645 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern3_2/quadratic,pylibkriging,q2,0.897353 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern3_2/quadratic,pylibkriging,nlpd,3.94939 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern5_2/constant,pylibkriging,fit_s,0.00238967 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern5_2/constant,pylibkriging,rmse,18.3236 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern5_2/constant,pylibkriging,q2,0.566507 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern5_2/constant,pylibkriging,nlpd,18.4322 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern5_2/constant,pylibkriging BFGS10,rmse,1.36208 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern5_2/constant,pylibkriging BFGS10,q2,0.997605 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern5_2/constant,pylibkriging BFGS10,nlpd,1.03215 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern5_2/linear,pylibkriging,fit_s,0.00207353 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern5_2/linear,pylibkriging,rmse,18.7127 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern5_2/linear,pylibkriging,q2,0.547901 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern5_2/linear,pylibkriging,nlpd,24.5771 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern5_2/interactive,pylibkriging,fit_s,0.0020268 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern5_2/interactive,pylibkriging,rmse,18.7194 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern5_2/interactive,pylibkriging,q2,0.547577 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern5_2/interactive,pylibkriging,nlpd,20.4255 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern5_2/quadratic,pylibkriging,fit_s,0.00144219 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern5_2/quadratic,pylibkriging,rmse,10.2878 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern5_2/quadratic,pylibkriging,q2,0.863351 -python_Kriging_kernels-trends-interpolation,python,six_hump_camel/matern5_2/quadratic,pylibkriging,nlpd,4.46344 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/gauss/constant,pylibkriging,fit_s,0.00526643 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/gauss/constant,pylibkriging,rmse,0.295934 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/gauss/constant,pylibkriging,q2,0.920537 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/gauss/constant,pylibkriging,nlpd,1.58666 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/gauss/linear,pylibkriging,fit_s,0.00606179 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/gauss/linear,pylibkriging,rmse,0.296355 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/gauss/linear,pylibkriging,q2,0.92031 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/gauss/linear,pylibkriging,nlpd,1.2962 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/gauss/interactive,pylibkriging,fit_s,0.00633311 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/gauss/interactive,pylibkriging,rmse,0.285136 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/gauss/interactive,pylibkriging,q2,0.926229 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/gauss/interactive,pylibkriging,nlpd,1.04122 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/gauss/quadratic,pylibkriging,fit_s,0.00615239 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/gauss/quadratic,pylibkriging,rmse,0.237308 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/gauss/quadratic,pylibkriging,q2,0.948902 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/gauss/quadratic,pylibkriging,nlpd,0.146484 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/exp/constant,pylibkriging,fit_s,0.00271893 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/exp/constant,pylibkriging,rmse,0.334876 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/exp/constant,pylibkriging,q2,0.898247 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/exp/constant,pylibkriging,nlpd,0.339235 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/exp/linear,pylibkriging,fit_s,0.00223827 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/exp/linear,pylibkriging,rmse,0.333447 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/exp/linear,pylibkriging,q2,0.899114 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/exp/linear,pylibkriging,nlpd,0.336515 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/exp/interactive,pylibkriging,fit_s,0.00192547 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/exp/interactive,pylibkriging,rmse,0.336819 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/exp/interactive,pylibkriging,q2,0.897063 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/exp/interactive,pylibkriging,nlpd,0.329566 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/exp/quadratic,pylibkriging,fit_s,0.00333118 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/exp/quadratic,pylibkriging,rmse,0.327816 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/exp/quadratic,pylibkriging,q2,0.902492 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/exp/quadratic,pylibkriging,nlpd,0.322056 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern3_2/constant,pylibkriging,fit_s,0.00606227 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern3_2/constant,pylibkriging,rmse,0.684362 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern3_2/constant,pylibkriging,q2,0.575038 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern3_2/constant,pylibkriging,nlpd,0.972803 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern3_2/linear,pylibkriging,fit_s,0.00382805 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern3_2/linear,pylibkriging,rmse,0.650201 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern3_2/linear,pylibkriging,q2,0.616404 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern3_2/linear,pylibkriging,nlpd,0.921455 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern3_2/interactive,pylibkriging,fit_s,0.00707507 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern3_2/interactive,pylibkriging,rmse,0.274326 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern3_2/interactive,pylibkriging,q2,0.931717 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern3_2/interactive,pylibkriging,nlpd,0.0549679 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern3_2/quadratic,pylibkriging,fit_s,0.00617123 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern3_2/quadratic,pylibkriging,rmse,0.263745 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern3_2/quadratic,pylibkriging,q2,0.936883 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern3_2/quadratic,pylibkriging,nlpd,0.0446267 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern5_2/constant,pylibkriging,fit_s,0.00426936 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern5_2/constant,pylibkriging,rmse,0.674391 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern5_2/constant,pylibkriging,q2,0.58733 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern5_2/constant,pylibkriging,nlpd,0.944765 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern5_2/constant,pylibkriging BFGS10,rmse,0.268216 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern5_2/constant,pylibkriging BFGS10,q2,0.934725 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern5_2/constant,pylibkriging BFGS10,nlpd,0.169585 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern5_2/linear,pylibkriging,fit_s,0.00730562 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern5_2/linear,pylibkriging,rmse,0.268808 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern5_2/linear,pylibkriging,q2,0.934436 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern5_2/linear,pylibkriging,nlpd,0.142382 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern5_2/interactive,pylibkriging,fit_s,0.00520349 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern5_2/interactive,pylibkriging,rmse,0.271088 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern5_2/interactive,pylibkriging,q2,0.933319 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern5_2/interactive,pylibkriging,nlpd,0.128788 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern5_2/quadratic,pylibkriging,fit_s,0.00757623 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern5_2/quadratic,pylibkriging,rmse,0.253093 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern5_2/quadratic,pylibkriging,q2,0.941878 -python_Kriging_kernels-trends-interpolation,python,log_goldstein_price/matern5_2/quadratic,pylibkriging,nlpd,0.025469 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/gauss/constant,pylibkriging,fit_s,0.000713825 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/gauss/constant,pylibkriging,rmse,0.026127 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/gauss/constant,pylibkriging,q2,0.999737 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/gauss/constant,pylibkriging,nlpd,-2.69817 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/gauss/linear,pylibkriging,fit_s,0.000786066 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/gauss/linear,pylibkriging,rmse,1.06243 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/gauss/linear,pylibkriging,q2,0.564691 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/gauss/linear,pylibkriging,nlpd,2.7394 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/gauss/interactive,pylibkriging,fit_s,0.000851393 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/gauss/interactive,pylibkriging,rmse,1.06504 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/gauss/interactive,pylibkriging,q2,0.562551 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/gauss/interactive,pylibkriging,nlpd,2.33575 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/gauss/quadratic,pylibkriging,fit_s,0.0012238 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/gauss/quadratic,pylibkriging,rmse,0.281816 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/gauss/quadratic,pylibkriging,q2,0.969371 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/gauss/quadratic,pylibkriging,nlpd,0.71682 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/exp/constant,pylibkriging,fit_s,0.000771999 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/exp/constant,pylibkriging,rmse,0.249023 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/exp/constant,pylibkriging,q2,0.976085 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/exp/constant,pylibkriging,nlpd,0.394159 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/exp/linear,pylibkriging,fit_s,0.0006814 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/exp/linear,pylibkriging,rmse,0.245558 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/exp/linear,pylibkriging,q2,0.976746 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/exp/linear,pylibkriging,nlpd,0.224593 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/exp/interactive,pylibkriging,fit_s,0.000779867 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/exp/interactive,pylibkriging,rmse,0.239032 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/exp/interactive,pylibkriging,q2,0.977965 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/exp/interactive,pylibkriging,nlpd,0.226133 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/exp/quadratic,pylibkriging,fit_s,0.00164485 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/exp/quadratic,pylibkriging,rmse,0.237713 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/exp/quadratic,pylibkriging,q2,0.978208 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/exp/quadratic,pylibkriging,nlpd,-0.139582 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern3_2/constant,pylibkriging,fit_s,0.000581741 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern3_2/constant,pylibkriging,rmse,0.0600911 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern3_2/constant,pylibkriging,q2,0.998607 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern3_2/constant,pylibkriging,nlpd,-1.34658 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern3_2/linear,pylibkriging,fit_s,0.000698566 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern3_2/linear,pylibkriging,rmse,0.0584376 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern3_2/linear,pylibkriging,q2,0.998683 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern3_2/linear,pylibkriging,nlpd,-1.34912 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern3_2/interactive,pylibkriging,fit_s,0.000763416 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern3_2/interactive,pylibkriging,rmse,0.0576096 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern3_2/interactive,pylibkriging,q2,0.99872 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern3_2/interactive,pylibkriging,nlpd,-1.35376 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern3_2/quadratic,pylibkriging,fit_s,0.00313497 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern3_2/quadratic,pylibkriging,rmse,0.215336 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern3_2/quadratic,pylibkriging,q2,0.982117 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern3_2/quadratic,pylibkriging,nlpd,-0.350837 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern5_2/constant,pylibkriging,fit_s,0.000642776 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern5_2/constant,pylibkriging,rmse,0.0246348 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern5_2/constant,pylibkriging,q2,0.999766 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern5_2/constant,pylibkriging,nlpd,-2.59598 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern5_2/constant,pylibkriging BFGS10,rmse,0.0246338 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern5_2/constant,pylibkriging BFGS10,q2,0.999766 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern5_2/constant,pylibkriging BFGS10,nlpd,-2.59598 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern5_2/linear,pylibkriging,fit_s,0.000878334 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern5_2/linear,pylibkriging,rmse,0.0232483 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern5_2/linear,pylibkriging,q2,0.999792 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern5_2/linear,pylibkriging,nlpd,-2.58843 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern5_2/interactive,pylibkriging,fit_s,0.000845909 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern5_2/interactive,pylibkriging,rmse,0.0230248 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern5_2/interactive,pylibkriging,q2,0.999796 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern5_2/interactive,pylibkriging,nlpd,-2.58839 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern5_2/quadratic,pylibkriging,fit_s,0.00176644 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern5_2/quadratic,pylibkriging,rmse,0.209802 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern5_2/quadratic,pylibkriging,q2,0.983025 -python_Kriging_kernels-trends-interpolation,python,lim_polynomial/matern5_2/quadratic,pylibkriging,nlpd,-0.403998 -python_Kriging_kernels-trends-interpolation,python,ishigami/gauss/constant,pylibkriging,fit_s,0.0820026 -python_Kriging_kernels-trends-interpolation,python,ishigami/gauss/constant,pylibkriging,rmse,1.41975 -python_Kriging_kernels-trends-interpolation,python,ishigami/gauss/constant,pylibkriging,q2,0.844642 -python_Kriging_kernels-trends-interpolation,python,ishigami/gauss/constant,pylibkriging,nlpd,2.25894 -python_Kriging_kernels-trends-interpolation,python,ishigami/gauss/linear,pylibkriging,fit_s,0.0247335 -python_Kriging_kernels-trends-interpolation,python,ishigami/gauss/linear,pylibkriging,rmse,2.84579 -python_Kriging_kernels-trends-interpolation,python,ishigami/gauss/linear,pylibkriging,q2,0.375811 -python_Kriging_kernels-trends-interpolation,python,ishigami/gauss/linear,pylibkriging,nlpd,3.41805 -python_Kriging_kernels-trends-interpolation,python,ishigami/gauss/interactive,pylibkriging,fit_s,0.0169995 -python_Kriging_kernels-trends-interpolation,python,ishigami/gauss/interactive,pylibkriging,rmse,2.85366 -python_Kriging_kernels-trends-interpolation,python,ishigami/gauss/interactive,pylibkriging,q2,0.372353 -python_Kriging_kernels-trends-interpolation,python,ishigami/gauss/interactive,pylibkriging,nlpd,3.28643 -python_Kriging_kernels-trends-interpolation,python,ishigami/gauss/quadratic,pylibkriging,fit_s,0.00751019 -python_Kriging_kernels-trends-interpolation,python,ishigami/gauss/quadratic,pylibkriging,rmse,2.08982 -python_Kriging_kernels-trends-interpolation,python,ishigami/gauss/quadratic,pylibkriging,q2,0.66339 -python_Kriging_kernels-trends-interpolation,python,ishigami/gauss/quadratic,pylibkriging,nlpd,2.80099 -python_Kriging_kernels-trends-interpolation,python,ishigami/exp/constant,pylibkriging,fit_s,0.00825953 -python_Kriging_kernels-trends-interpolation,python,ishigami/exp/constant,pylibkriging,rmse,1.41632 -python_Kriging_kernels-trends-interpolation,python,ishigami/exp/constant,pylibkriging,q2,0.845391 -python_Kriging_kernels-trends-interpolation,python,ishigami/exp/constant,pylibkriging,nlpd,1.75239 -python_Kriging_kernels-trends-interpolation,python,ishigami/exp/linear,pylibkriging,fit_s,0.00724149 -python_Kriging_kernels-trends-interpolation,python,ishigami/exp/linear,pylibkriging,rmse,1.4095 -python_Kriging_kernels-trends-interpolation,python,ishigami/exp/linear,pylibkriging,q2,0.846877 -python_Kriging_kernels-trends-interpolation,python,ishigami/exp/linear,pylibkriging,nlpd,1.74779 -python_Kriging_kernels-trends-interpolation,python,ishigami/exp/interactive,pylibkriging,fit_s,0.00750446 -python_Kriging_kernels-trends-interpolation,python,ishigami/exp/interactive,pylibkriging,rmse,1.41511 -python_Kriging_kernels-trends-interpolation,python,ishigami/exp/interactive,pylibkriging,q2,0.845657 -python_Kriging_kernels-trends-interpolation,python,ishigami/exp/interactive,pylibkriging,nlpd,1.75024 -python_Kriging_kernels-trends-interpolation,python,ishigami/exp/quadratic,pylibkriging,fit_s,0.00750661 -python_Kriging_kernels-trends-interpolation,python,ishigami/exp/quadratic,pylibkriging,rmse,1.40329 -python_Kriging_kernels-trends-interpolation,python,ishigami/exp/quadratic,pylibkriging,q2,0.848223 -python_Kriging_kernels-trends-interpolation,python,ishigami/exp/quadratic,pylibkriging,nlpd,1.73254 -python_Kriging_kernels-trends-interpolation,python,ishigami/matern3_2/constant,pylibkriging,fit_s,0.0483243 -python_Kriging_kernels-trends-interpolation,python,ishigami/matern3_2/constant,pylibkriging,rmse,1.32974 -python_Kriging_kernels-trends-interpolation,python,ishigami/matern3_2/constant,pylibkriging,q2,0.863717 -python_Kriging_kernels-trends-interpolation,python,ishigami/matern3_2/constant,pylibkriging,nlpd,1.50266 -python_Kriging_kernels-trends-interpolation,python,ishigami/matern3_2/linear,pylibkriging,fit_s,0.0462224 -python_Kriging_kernels-trends-interpolation,python,ishigami/matern3_2/linear,pylibkriging,rmse,1.30244 -python_Kriging_kernels-trends-interpolation,python,ishigami/matern3_2/linear,pylibkriging,q2,0.869255 -python_Kriging_kernels-trends-interpolation,python,ishigami/matern3_2/linear,pylibkriging,nlpd,1.47906 -python_Kriging_kernels-trends-interpolation,python,ishigami/matern3_2/interactive,pylibkriging,fit_s,0.0462408 -python_Kriging_kernels-trends-interpolation,python,ishigami/matern3_2/interactive,pylibkriging,rmse,1.30591 -python_Kriging_kernels-trends-interpolation,python,ishigami/matern3_2/interactive,pylibkriging,q2,0.868558 -python_Kriging_kernels-trends-interpolation,python,ishigami/matern3_2/interactive,pylibkriging,nlpd,1.47384 -python_Kriging_kernels-trends-interpolation,python,ishigami/matern3_2/quadratic,pylibkriging,fit_s,0.0412879 -python_Kriging_kernels-trends-interpolation,python,ishigami/matern3_2/quadratic,pylibkriging,rmse,1.32197 -python_Kriging_kernels-trends-interpolation,python,ishigami/matern3_2/quadratic,pylibkriging,q2,0.865305 -python_Kriging_kernels-trends-interpolation,python,ishigami/matern3_2/quadratic,pylibkriging,nlpd,1.46827 -python_Kriging_kernels-trends-interpolation,python,ishigami/matern5_2/constant,pylibkriging,fit_s,0.02212 -python_Kriging_kernels-trends-interpolation,python,ishigami/matern5_2/constant,pylibkriging,rmse,3.00566 -python_Kriging_kernels-trends-interpolation,python,ishigami/matern5_2/constant,pylibkriging,q2,0.303709 -python_Kriging_kernels-trends-interpolation,python,ishigami/matern5_2/constant,pylibkriging,nlpd,3.42085 -python_Kriging_kernels-trends-interpolation,python,ishigami/matern5_2/constant,pylibkriging BFGS10,rmse,1.33903 -python_Kriging_kernels-trends-interpolation,python,ishigami/matern5_2/constant,pylibkriging BFGS10,q2,0.861806 -python_Kriging_kernels-trends-interpolation,python,ishigami/matern5_2/constant,pylibkriging BFGS10,nlpd,1.66132 -python_Kriging_kernels-trends-interpolation,python,ishigami/matern5_2/linear,pylibkriging,fit_s,0.0472796 -python_Kriging_kernels-trends-interpolation,python,ishigami/matern5_2/linear,pylibkriging,rmse,1.3434 -python_Kriging_kernels-trends-interpolation,python,ishigami/matern5_2/linear,pylibkriging,q2,0.860901 -python_Kriging_kernels-trends-interpolation,python,ishigami/matern5_2/linear,pylibkriging,nlpd,1.65103 -python_Kriging_kernels-trends-interpolation,python,ishigami/matern5_2/interactive,pylibkriging,fit_s,0.0477624 -python_Kriging_kernels-trends-interpolation,python,ishigami/matern5_2/interactive,pylibkriging,rmse,1.34479 -python_Kriging_kernels-trends-interpolation,python,ishigami/matern5_2/interactive,pylibkriging,q2,0.860615 -python_Kriging_kernels-trends-interpolation,python,ishigami/matern5_2/interactive,pylibkriging,nlpd,1.64153 -python_Kriging_kernels-trends-interpolation,python,ishigami/matern5_2/quadratic,pylibkriging,fit_s,0.0483804 -python_Kriging_kernels-trends-interpolation,python,ishigami/matern5_2/quadratic,pylibkriging,rmse,1.37082 -python_Kriging_kernels-trends-interpolation,python,ishigami/matern5_2/quadratic,pylibkriging,q2,0.855166 -python_Kriging_kernels-trends-interpolation,python,ishigami/matern5_2/quadratic,pylibkriging,nlpd,1.62228 -python_Kriging_kernels-trends-interpolation,python,hartmann3/gauss/constant,pylibkriging,fit_s,0.00703788 -python_Kriging_kernels-trends-interpolation,python,hartmann3/gauss/constant,pylibkriging,rmse,0.156226 -python_Kriging_kernels-trends-interpolation,python,hartmann3/gauss/constant,pylibkriging,q2,0.971116 -python_Kriging_kernels-trends-interpolation,python,hartmann3/gauss/constant,pylibkriging,nlpd,-0.555623 -python_Kriging_kernels-trends-interpolation,python,hartmann3/gauss/linear,pylibkriging,fit_s,0.0116901 -python_Kriging_kernels-trends-interpolation,python,hartmann3/gauss/linear,pylibkriging,rmse,0.153297 -python_Kriging_kernels-trends-interpolation,python,hartmann3/gauss/linear,pylibkriging,q2,0.972189 -python_Kriging_kernels-trends-interpolation,python,hartmann3/gauss/linear,pylibkriging,nlpd,-0.568404 -python_Kriging_kernels-trends-interpolation,python,hartmann3/gauss/interactive,pylibkriging,fit_s,0.00962472 -python_Kriging_kernels-trends-interpolation,python,hartmann3/gauss/interactive,pylibkriging,rmse,0.712947 -python_Kriging_kernels-trends-interpolation,python,hartmann3/gauss/interactive,pylibkriging,q2,0.398455 -python_Kriging_kernels-trends-interpolation,python,hartmann3/gauss/interactive,pylibkriging,nlpd,1.74554 -python_Kriging_kernels-trends-interpolation,python,hartmann3/gauss/quadratic,pylibkriging,fit_s,0.00845146 -python_Kriging_kernels-trends-interpolation,python,hartmann3/gauss/quadratic,pylibkriging,rmse,0.73238 -python_Kriging_kernels-trends-interpolation,python,hartmann3/gauss/quadratic,pylibkriging,q2,0.365216 -python_Kriging_kernels-trends-interpolation,python,hartmann3/gauss/quadratic,pylibkriging,nlpd,2.14588 -python_Kriging_kernels-trends-interpolation,python,hartmann3/exp/constant,pylibkriging,fit_s,0.00484872 -python_Kriging_kernels-trends-interpolation,python,hartmann3/exp/constant,pylibkriging,rmse,0.347348 -python_Kriging_kernels-trends-interpolation,python,hartmann3/exp/constant,pylibkriging,q2,0.857215 -python_Kriging_kernels-trends-interpolation,python,hartmann3/exp/constant,pylibkriging,nlpd,0.352683 -python_Kriging_kernels-trends-interpolation,python,hartmann3/exp/linear,pylibkriging,fit_s,0.00318956 -python_Kriging_kernels-trends-interpolation,python,hartmann3/exp/linear,pylibkriging,rmse,0.347273 -python_Kriging_kernels-trends-interpolation,python,hartmann3/exp/linear,pylibkriging,q2,0.857276 -python_Kriging_kernels-trends-interpolation,python,hartmann3/exp/linear,pylibkriging,nlpd,0.343631 -python_Kriging_kernels-trends-interpolation,python,hartmann3/exp/interactive,pylibkriging,fit_s,0.00330567 -python_Kriging_kernels-trends-interpolation,python,hartmann3/exp/interactive,pylibkriging,rmse,0.378349 -python_Kriging_kernels-trends-interpolation,python,hartmann3/exp/interactive,pylibkriging,q2,0.83059 -python_Kriging_kernels-trends-interpolation,python,hartmann3/exp/interactive,pylibkriging,nlpd,0.385842 -python_Kriging_kernels-trends-interpolation,python,hartmann3/exp/quadratic,pylibkriging,fit_s,0.00355411 -python_Kriging_kernels-trends-interpolation,python,hartmann3/exp/quadratic,pylibkriging,rmse,0.355119 -python_Kriging_kernels-trends-interpolation,python,hartmann3/exp/quadratic,pylibkriging,q2,0.850755 -python_Kriging_kernels-trends-interpolation,python,hartmann3/exp/quadratic,pylibkriging,nlpd,0.334371 -python_Kriging_kernels-trends-interpolation,python,hartmann3/matern3_2/constant,pylibkriging,fit_s,0.0107734 -python_Kriging_kernels-trends-interpolation,python,hartmann3/matern3_2/constant,pylibkriging,rmse,0.227521 -python_Kriging_kernels-trends-interpolation,python,hartmann3/matern3_2/constant,pylibkriging,q2,0.938737 -python_Kriging_kernels-trends-interpolation,python,hartmann3/matern3_2/constant,pylibkriging,nlpd,-0.181207 -python_Kriging_kernels-trends-interpolation,python,hartmann3/matern3_2/linear,pylibkriging,fit_s,0.0118749 -python_Kriging_kernels-trends-interpolation,python,hartmann3/matern3_2/linear,pylibkriging,rmse,0.23038 -python_Kriging_kernels-trends-interpolation,python,hartmann3/matern3_2/linear,pylibkriging,q2,0.937188 -python_Kriging_kernels-trends-interpolation,python,hartmann3/matern3_2/linear,pylibkriging,nlpd,-0.162203 -python_Kriging_kernels-trends-interpolation,python,hartmann3/matern3_2/interactive,pylibkriging,fit_s,0.00637007 -python_Kriging_kernels-trends-interpolation,python,hartmann3/matern3_2/interactive,pylibkriging,rmse,0.315752 -python_Kriging_kernels-trends-interpolation,python,hartmann3/matern3_2/interactive,pylibkriging,q2,0.88201 -python_Kriging_kernels-trends-interpolation,python,hartmann3/matern3_2/interactive,pylibkriging,nlpd,0.898253 -python_Kriging_kernels-trends-interpolation,python,hartmann3/matern3_2/quadratic,pylibkriging,fit_s,0.00598764 -python_Kriging_kernels-trends-interpolation,python,hartmann3/matern3_2/quadratic,pylibkriging,rmse,0.291393 -python_Kriging_kernels-trends-interpolation,python,hartmann3/matern3_2/quadratic,pylibkriging,q2,0.899513 -python_Kriging_kernels-trends-interpolation,python,hartmann3/matern3_2/quadratic,pylibkriging,nlpd,0.53743 -python_Kriging_kernels-trends-interpolation,python,hartmann3/matern5_2/constant,pylibkriging,fit_s,0.0100911 -python_Kriging_kernels-trends-interpolation,python,hartmann3/matern5_2/constant,pylibkriging,rmse,0.190287 -python_Kriging_kernels-trends-interpolation,python,hartmann3/matern5_2/constant,pylibkriging,q2,0.957148 -python_Kriging_kernels-trends-interpolation,python,hartmann3/matern5_2/constant,pylibkriging,nlpd,-0.403054 -python_Kriging_kernels-trends-interpolation,python,hartmann3/matern5_2/constant,pylibkriging BFGS10,rmse,0.190286 -python_Kriging_kernels-trends-interpolation,python,hartmann3/matern5_2/constant,pylibkriging BFGS10,q2,0.957148 -python_Kriging_kernels-trends-interpolation,python,hartmann3/matern5_2/constant,pylibkriging BFGS10,nlpd,-0.40306 -python_Kriging_kernels-trends-interpolation,python,hartmann3/matern5_2/linear,pylibkriging,fit_s,0.0106812 -python_Kriging_kernels-trends-interpolation,python,hartmann3/matern5_2/linear,pylibkriging,rmse,0.189633 -python_Kriging_kernels-trends-interpolation,python,hartmann3/matern5_2/linear,pylibkriging,q2,0.957442 -python_Kriging_kernels-trends-interpolation,python,hartmann3/matern5_2/linear,pylibkriging,nlpd,-0.410075 -python_Kriging_kernels-trends-interpolation,python,hartmann3/matern5_2/interactive,pylibkriging,fit_s,0.00839925 -python_Kriging_kernels-trends-interpolation,python,hartmann3/matern5_2/interactive,pylibkriging,rmse,0.307114 -python_Kriging_kernels-trends-interpolation,python,hartmann3/matern5_2/interactive,pylibkriging,q2,0.888377 -python_Kriging_kernels-trends-interpolation,python,hartmann3/matern5_2/interactive,pylibkriging,nlpd,1.50063 -python_Kriging_kernels-trends-interpolation,python,hartmann3/matern5_2/quadratic,pylibkriging,fit_s,0.00724792 -python_Kriging_kernels-trends-interpolation,python,hartmann3/matern5_2/quadratic,pylibkriging,rmse,0.285355 -python_Kriging_kernels-trends-interpolation,python,hartmann3/matern5_2/quadratic,pylibkriging,q2,0.903634 -python_Kriging_kernels-trends-interpolation,python,hartmann3/matern5_2/quadratic,pylibkriging,nlpd,0.898128 -python_Kriging_objectives-LL-LOO-LMP-gradients,python,branin/LL,pylibkriging,rmse,71.4801 -python_Kriging_objectives-LL-LOO-LMP-gradients,python,branin/LL,pylibkriging,q2,-1991.62 -python_Kriging_objectives-LL-LOO-LMP-gradients,python,branin/LL,pylibkriging,nlpd,13603.3 -python_Kriging_objectives-LL-LOO-LMP-gradients,python,branin/LOO,pylibkriging,rmse,71.5631 -python_Kriging_objectives-LL-LOO-LMP-gradients,python,branin/LOO,pylibkriging,q2,-1996.25 -python_Kriging_objectives-LL-LOO-LMP-gradients,python,branin/LOO,pylibkriging,nlpd,18836.2 -python_Kriging_objectives-LL-LOO-LMP-gradients,python,branin/LMP,pylibkriging,rmse,71.5295 -python_Kriging_objectives-LL-LOO-LMP-gradients,python,branin/LMP,pylibkriging,q2,-1994.38 -python_Kriging_objectives-LL-LOO-LMP-gradients,python,branin/LMP,pylibkriging,nlpd,14603.3 -python_Kriging_objectives-LL-LOO-LMP-gradients,python,hartmann3/LL,pylibkriging,rmse,1.30952 -python_Kriging_objectives-LL-LOO-LMP-gradients,python,hartmann3/LL,pylibkriging,q2,-2013.29 -python_Kriging_objectives-LL-LOO-LMP-gradients,python,hartmann3/LL,pylibkriging,nlpd,41.7128 -python_Kriging_objectives-LL-LOO-LMP-gradients,python,hartmann3/LOO,pylibkriging,rmse,1.31436 -python_Kriging_objectives-LL-LOO-LMP-gradients,python,hartmann3/LOO,pylibkriging,q2,-2028.22 -python_Kriging_objectives-LL-LOO-LMP-gradients,python,hartmann3/LOO,pylibkriging,nlpd,49.7135 -python_Kriging_objectives-LL-LOO-LMP-gradients,python,hartmann3/LMP,pylibkriging,rmse,1.31148 -python_Kriging_objectives-LL-LOO-LMP-gradients,python,hartmann3/LMP,pylibkriging,q2,-2019.34 -python_Kriging_objectives-LL-LOO-LMP-gradients,python,hartmann3/LMP,pylibkriging,nlpd,43.397 -python_Kriging_objectives-LL-LOO-LMP-gradients,python,ishigami/LL,pylibkriging,rmse,5.11103 -python_Kriging_objectives-LL-LOO-LMP-gradients,python,ishigami/LL,pylibkriging,q2,-1783.42 -python_Kriging_objectives-LL-LOO-LMP-gradients,python,ishigami/LL,pylibkriging,nlpd,10.4546 -python_Kriging_objectives-LL-LOO-LMP-gradients,python,ishigami/LOO,pylibkriging,rmse,5.81789 -python_Kriging_objectives-LL-LOO-LMP-gradients,python,ishigami/LOO,pylibkriging,q2,-2311.12 -python_Kriging_objectives-LL-LOO-LMP-gradients,python,ishigami/LOO,pylibkriging,nlpd,40.8323 -python_Kriging_objectives-LL-LOO-LMP-gradients,python,ishigami/LMP,pylibkriging,rmse,5.18495 -python_Kriging_objectives-LL-LOO-LMP-gradients,python,ishigami/LMP,pylibkriging,q2,-1835.41 -python_Kriging_objectives-LL-LOO-LMP-gradients,python,ishigami/LMP,pylibkriging,nlpd,11.5894 -python_Kriging_objectives-LL-LOO-LMP-gradients,python,friedman/LL,pylibkriging,rmse,6.87533 -python_Kriging_objectives-LL-LOO-LMP-gradients,python,friedman/LL,pylibkriging,q2,-1991.33 -python_Kriging_objectives-LL-LOO-LMP-gradients,python,friedman/LL,pylibkriging,nlpd,651.706 -python_Kriging_objectives-LL-LOO-LMP-gradients,python,friedman/LOO,pylibkriging,rmse,6.87323 -python_Kriging_objectives-LL-LOO-LMP-gradients,python,friedman/LOO,pylibkriging,q2,-1990.12 -python_Kriging_objectives-LL-LOO-LMP-gradients,python,friedman/LOO,pylibkriging,nlpd,484.115 -python_Kriging_objectives-LL-LOO-LMP-gradients,python,friedman/LMP,pylibkriging,rmse,6.87647 -python_Kriging_objectives-LL-LOO-LMP-gradients,python,friedman/LMP,pylibkriging,q2,-1991.99 -python_Kriging_objectives-LL-LOO-LMP-gradients,python,friedman/LMP,pylibkriging,nlpd,658.808 -python_Kriging-nugget-noise_noisy-data,python,forrester,nugget,rmse,0.198091 -python_Kriging-nugget-noise_noisy-data,python,forrester,nugget,q2,0.998087 -python_Kriging-nugget-noise_noisy-data,python,forrester,nugget,nlpd,0.286631 -python_Kriging-nugget-noise_noisy-data,python,forrester,noise-free fit,rmse,0.487916 -python_Kriging-nugget-noise_noisy-data,python,forrester,noise-free fit,q2,0.988392 -python_Kriging-nugget-noise_noisy-data,python,forrester,noise-free fit,nlpd,862.687 -python_Kriging-nugget-noise_noisy-data,python,friedman,nugget,rmse,0.467335 -python_Kriging-nugget-noise_noisy-data,python,friedman,nugget,q2,0.99089 -python_Kriging-nugget-noise_noisy-data,python,friedman,nugget,nlpd,1.07073 -python_Kriging-nugget-noise_noisy-data,python,friedman,noise-free fit,rmse,1.27156 -python_Kriging-nugget-noise_noisy-data,python,friedman,noise-free fit,q2,0.932559 -python_Kriging-nugget-noise_noisy-data,python,friedman,noise-free fit,nlpd,1.64495 -python_Kriging-nugget-noise_noisy-data,python,gramacy_lee_2012/hetero,known variances,rmse,0.168741 -python_Kriging-nugget-noise_noisy-data,python,gramacy_lee_2012/hetero,known variances,q2,0.982515 -python_Kriging-nugget-noise_noisy-data,python,gramacy_lee_2012/hetero,known variances,nlpd,-0.570574 -python_Kriging-nugget-noise_noisy-data,python,gramacy_lee_2012/hetero,nugget,rmse,0.170945 -python_Kriging-nugget-noise_noisy-data,python,gramacy_lee_2012/hetero,nugget,q2,0.982055 -python_Kriging-nugget-noise_noisy-data,python,gramacy_lee_2012/hetero,nugget,nlpd,-0.221568 -python_WarpKriging_warpings-categorical-ordinal,python,gl2012,Kriging,rmse,0.167263 -python_WarpKriging_warpings-categorical-ordinal,python,gl2012,Kriging,q2,0.981176 -python_WarpKriging_warpings-categorical-ordinal,python,gl2012,Kriging,nlpd,-1.08917 -python_WarpKriging_warpings-categorical-ordinal,python,gl2012,affine,rmse,0.167263 -python_WarpKriging_warpings-categorical-ordinal,python,gl2012,affine,q2,0.981176 -python_WarpKriging_warpings-categorical-ordinal,python,gl2012,affine,nlpd,-1.08917 -python_WarpKriging_warpings-categorical-ordinal,python,gl2012,boxcox,rmse,0.16759 -python_WarpKriging_warpings-categorical-ordinal,python,gl2012,boxcox,q2,0.981103 -python_WarpKriging_warpings-categorical-ordinal,python,gl2012,boxcox,nlpd,282.422 -python_WarpKriging_warpings-categorical-ordinal,python,gl2012,kumaraswamy,rmse,0.168286 -python_WarpKriging_warpings-categorical-ordinal,python,gl2012,kumaraswamy,q2,0.980945 -python_WarpKriging_warpings-categorical-ordinal,python,gl2012,kumaraswamy,nlpd,2.04577e+06 -python_WarpKriging_warpings-categorical-ordinal,python,gl2012,knots(5),rmse,0.16791 -python_WarpKriging_warpings-categorical-ordinal,python,gl2012,knots(5),q2,0.98103 -python_WarpKriging_warpings-categorical-ordinal,python,gl2012,knots(5),nlpd,2.04376e+06 -python_WarpKriging_warpings-categorical-ordinal,python,gl2012,neural_mono(8),rmse,0.170824 -python_WarpKriging_warpings-categorical-ordinal,python,gl2012,neural_mono(8),q2,0.980366 -python_WarpKriging_warpings-categorical-ordinal,python,gl2012,neural_mono(8),nlpd,-0.265141 -python_WarpKriging_warpings-categorical-ordinal,python,gl2008,Kriging,rmse,0.0534649 -python_WarpKriging_warpings-categorical-ordinal,python,gl2008,Kriging,q2,0.585636 -python_WarpKriging_warpings-categorical-ordinal,python,gl2008,Kriging,nlpd,-1.74569 -python_WarpKriging_warpings-categorical-ordinal,python,gl2008,kumaraswamy,rmse,0.0535298 -python_WarpKriging_warpings-categorical-ordinal,python,gl2008,kumaraswamy,q2,0.58463 -python_WarpKriging_warpings-categorical-ordinal,python,gl2008,kumaraswamy,nlpd,-1.74552 -python_WarpKriging_warpings-categorical-ordinal,python,mixed,Kriging (level as number),rmse,16.7693 -python_WarpKriging_warpings-categorical-ordinal,python,mixed,Kriging (level as number),q2,0.838453 -python_WarpKriging_warpings-categorical-ordinal,python,mixed,Kriging (level as number),nlpd,3.81063 -python_WarpKriging_warpings-categorical-ordinal,python,mixed,"categorical(5,2)",rmse,16.1438 -python_WarpKriging_warpings-categorical-ordinal,python,mixed,"categorical(5,2)",q2,0.85028 -python_WarpKriging_warpings-categorical-ordinal,python,mixed,"categorical(5,2)",nlpd,3.3206 -python_WarpKriging_warpings-categorical-ordinal,python,ordinal,Kriging (rank as number),rmse,0.213845 -python_WarpKriging_warpings-categorical-ordinal,python,ordinal,Kriging (rank as number),q2,0.953177 -python_WarpKriging_warpings-categorical-ordinal,python,ordinal,Kriging (rank as number),nlpd,0.810969 -python_WarpKriging_warpings-categorical-ordinal,python,ordinal,ordinal(6),rmse,0.213043 -python_WarpKriging_warpings-categorical-ordinal,python,ordinal,ordinal(6),q2,0.953528 -python_WarpKriging_warpings-categorical-ordinal,python,ordinal,ordinal(6),nlpd,0.78888 -python_Kriging_high-dimension-normalize-screening,python,otl_circuit,unit cube,fit_s,0.00796986 -python_Kriging_high-dimension-normalize-screening,python,otl_circuit,unit cube,rmse,0.0150514 -python_Kriging_high-dimension-normalize-screening,python,otl_circuit,unit cube,q2,0.999836 -python_Kriging_high-dimension-normalize-screening,python,otl_circuit,unit cube,nlpd,-2.90452 -python_Kriging_high-dimension-normalize-screening,python,otl_circuit,native + normalize,rmse,0.0150514 -python_Kriging_high-dimension-normalize-screening,python,otl_circuit,native + normalize,q2,0.999836 -python_Kriging_high-dimension-normalize-screening,python,otl_circuit,native + normalize,nlpd,-2.90452 -python_Kriging_high-dimension-normalize-screening,python,piston,unit cube,fit_s,0.023371 -python_Kriging_high-dimension-normalize-screening,python,piston,unit cube,rmse,0.0104258 -python_Kriging_high-dimension-normalize-screening,python,piston,unit cube,q2,0.994462 -python_Kriging_high-dimension-normalize-screening,python,piston,unit cube,nlpd,-3.49038 -python_Kriging_high-dimension-normalize-screening,python,piston,native + normalize,rmse,0.0104258 -python_Kriging_high-dimension-normalize-screening,python,piston,native + normalize,q2,0.994462 -python_Kriging_high-dimension-normalize-screening,python,piston,native + normalize,nlpd,-3.49038 -python_Kriging_high-dimension-normalize-screening,python,borehole,unit cube,fit_s,0.0214174 -python_Kriging_high-dimension-normalize-screening,python,borehole,unit cube,rmse,0.893341 -python_Kriging_high-dimension-normalize-screening,python,borehole,unit cube,q2,0.999599 -python_Kriging_high-dimension-normalize-screening,python,borehole,unit cube,nlpd,1.08433 -python_Kriging_high-dimension-normalize-screening,python,borehole,native + normalize,rmse,0.893341 -python_Kriging_high-dimension-normalize-screening,python,borehole,native + normalize,q2,0.999599 -python_Kriging_high-dimension-normalize-screening,python,borehole,native + normalize,nlpd,1.08433 -python_Kriging_high-dimension-normalize-screening,python,robot_arm,unit cube,fit_s,0.187081 -python_Kriging_high-dimension-normalize-screening,python,robot_arm,unit cube,rmse,0.358507 -python_Kriging_high-dimension-normalize-screening,python,robot_arm,unit cube,q2,0.571154 -python_Kriging_high-dimension-normalize-screening,python,robot_arm,unit cube,nlpd,0.355106 -python_Kriging_high-dimension-normalize-screening,python,robot_arm,native + normalize,rmse,0.358507 -python_Kriging_high-dimension-normalize-screening,python,robot_arm,native + normalize,q2,0.571154 -python_Kriging_high-dimension-normalize-screening,python,robot_arm,native + normalize,nlpd,0.355106 -python_Kriging_high-dimension-normalize-screening,python,dette_pepelyshev,unit cube,fit_s,0.0181198 -python_Kriging_high-dimension-normalize-screening,python,dette_pepelyshev,unit cube,rmse,0.424908 -python_Kriging_high-dimension-normalize-screening,python,dette_pepelyshev,unit cube,q2,0.997757 -python_Kriging_high-dimension-normalize-screening,python,dette_pepelyshev,unit cube,nlpd,0.510276 -python_Kriging_high-dimension-normalize-screening,python,dette_pepelyshev,native + normalize,rmse,0.424908 -python_Kriging_high-dimension-normalize-screening,python,dette_pepelyshev,native + normalize,q2,0.997757 -python_Kriging_high-dimension-normalize-screening,python,dette_pepelyshev,native + normalize,nlpd,0.510276 -python_Kriging_high-dimension-normalize-screening,python,wing_weight,unit cube,fit_s,0.0303686 -python_Kriging_high-dimension-normalize-screening,python,wing_weight,unit cube,rmse,0.809743 -python_Kriging_high-dimension-normalize-screening,python,wing_weight,unit cube,q2,0.999716 -python_Kriging_high-dimension-normalize-screening,python,wing_weight,unit cube,nlpd,1.07161 -python_Kriging_high-dimension-normalize-screening,python,wing_weight,native + normalize,rmse,0.809743 -python_Kriging_high-dimension-normalize-screening,python,wing_weight,native + normalize,q2,0.999716 -python_Kriging_high-dimension-normalize-screening,python,wing_weight,native + normalize,nlpd,1.07161 -python_Kriging_high-dimension-normalize-screening,python,welch,unit cube,fit_s,0.500499 -python_Kriging_high-dimension-normalize-screening,python,welch,unit cube,rmse,0.167729 -python_Kriging_high-dimension-normalize-screening,python,welch,unit cube,q2,0.993452 -python_Kriging_high-dimension-normalize-screening,python,welch,unit cube,nlpd,-0.429138 -python_Kriging_high-dimension-normalize-screening,python,welch,native + normalize,rmse,0.167692 -python_Kriging_high-dimension-normalize-screening,python,welch,native + normalize,q2,0.993454 -python_Kriging_high-dimension-normalize-screening,python,welch,native + normalize,nlpd,-0.429291 -python_MLPKriging_deep-kernel-projected-branin,python,projected_branin_10d,Kriging,rmse,4.42284 -python_MLPKriging_deep-kernel-projected-branin,python,projected_branin_10d,Kriging,q2,0.959122 -python_MLPKriging_deep-kernel-projected-branin,python,projected_branin_10d,Kriging,nlpd,2.89997 -python_MLPKriging_deep-kernel-projected-branin,python,projected_branin_10d,MLPKriging tanh BFGS,fit_s,0.171914 -python_MLPKriging_deep-kernel-projected-branin,python,projected_branin_10d,MLPKriging tanh BFGS,rmse,59.5068 -python_MLPKriging_deep-kernel-projected-branin,python,projected_branin_10d,MLPKriging tanh BFGS,q2,-6.39973 -python_MLPKriging_deep-kernel-projected-branin,python,projected_branin_10d,MLPKriging tanh BFGS,nlpd,100.745 -python_MLPKriging_deep-kernel-projected-branin,python,projected_branin_10d,MLPKriging tanh BFGS10,fit_s,0.53444 -python_MLPKriging_deep-kernel-projected-branin,python,projected_branin_10d,MLPKriging tanh BFGS10,rmse,4.41161 -python_MLPKriging_deep-kernel-projected-branin,python,projected_branin_10d,MLPKriging tanh BFGS10,q2,0.95933 -python_MLPKriging_deep-kernel-projected-branin,python,projected_branin_10d,MLPKriging tanh BFGS10,nlpd,28.4236 -python_MLPKriging_deep-kernel-projected-branin,python,projected_branin_10d,MLPKriging selu BFGS,fit_s,0.145419 -python_MLPKriging_deep-kernel-projected-branin,python,projected_branin_10d,MLPKriging selu BFGS,rmse,58.9844 -python_MLPKriging_deep-kernel-projected-branin,python,projected_branin_10d,MLPKriging selu BFGS,q2,-6.27038 -python_MLPKriging_deep-kernel-projected-branin,python,projected_branin_10d,MLPKriging selu BFGS,nlpd,46.9347 -python_MLPKriging_deep-kernel-projected-branin,python,projected_branin_10d,MLPKriging selu BFGS10,fit_s,0.518803 -python_MLPKriging_deep-kernel-projected-branin,python,projected_branin_10d,MLPKriging selu BFGS10,rmse,3.79093 -python_MLPKriging_deep-kernel-projected-branin,python,projected_branin_10d,MLPKriging selu BFGS10,q2,0.969969 -python_MLPKriging_deep-kernel-projected-branin,python,projected_branin_10d,MLPKriging selu BFGS10,nlpd,12.6814 -python_Kriging-vecchia-nystrom_large-n,python,hartmann3/n=1500,exact,fit_s,11.2644 -python_NestedKriging_aggregations-large-n,python,hartmann3/n=1500,exact,fit_s,11.2644 -python_Kriging-vecchia-nystrom_large-n,python,hartmann3/n=1500,exact,rmse,0.000259233 -python_NestedKriging_aggregations-large-n,python,hartmann3/n=1500,exact,rmse,0.000259233 -python_Kriging-vecchia-nystrom_large-n,python,hartmann3/n=1500,exact,q2,1 -python_NestedKriging_aggregations-large-n,python,hartmann3/n=1500,exact,q2,1 -python_Kriging-vecchia-nystrom_large-n,python,hartmann3/n=1500,exact,nlpd,-6.92184 -python_NestedKriging_aggregations-large-n,python,hartmann3/n=1500,exact,nlpd,-6.92184 -python_NestedKriging_aggregations-large-n,python,hartmann3/n=1500,nested NK (20 groups),fit_s,0.324342 -python_NestedKriging_aggregations-large-n,python,hartmann3/n=1500,nested NK (20 groups),rmse,0.0029908 -python_NestedKriging_aggregations-large-n,python,hartmann3/n=1500,nested NK (20 groups),q2,0.999991 -python_NestedKriging_aggregations-large-n,python,hartmann3/n=1500,nested NK (20 groups),nlpd,-4.85946 -python_NestedKriging_aggregations-large-n,python,hartmann3/n=5000,nested NK (20 groups),fit_s,3.25297 -python_NestedKriging_aggregations-large-n,python,hartmann3/n=5000,nested NK (20 groups),rmse,0.000222655 -python_NestedKriging_aggregations-large-n,python,hartmann3/n=5000,nested NK (20 groups),q2,1 -python_NestedKriging_aggregations-large-n,python,hartmann3/n=5000,nested NK (20 groups),nlpd,-7.44453 -python_NestedKriging_aggregations-large-n,python,hartmann3/n=1500,nested rBCM (20 groups),fit_s,0.273419 -python_NestedKriging_aggregations-large-n,python,hartmann3/n=1500,nested rBCM (20 groups),rmse,0.00304431 -python_NestedKriging_aggregations-large-n,python,hartmann3/n=1500,nested rBCM (20 groups),q2,0.99999 -python_NestedKriging_aggregations-large-n,python,hartmann3/n=1500,nested rBCM (20 groups),nlpd,-4.50201 -python_NestedKriging_aggregations-large-n,python,hartmann3/n=5000,nested rBCM (20 groups),fit_s,2.67811 -python_NestedKriging_aggregations-large-n,python,hartmann3/n=5000,nested rBCM (20 groups),rmse,0.000224517 -python_NestedKriging_aggregations-large-n,python,hartmann3/n=5000,nested rBCM (20 groups),q2,1 -python_NestedKriging_aggregations-large-n,python,hartmann3/n=5000,nested rBCM (20 groups),nlpd,-7.18182 -python_Kriging-vecchia-nystrom_large-n,python,hartmann3/n=1500,subset of data (500),fit_s,0.685586 -python_Kriging-vecchia-nystrom_large-n,python,hartmann3/n=1500,subset of data (500),rmse,0.00269153 -python_Kriging-vecchia-nystrom_large-n,python,hartmann3/n=1500,subset of data (500),q2,0.999992 -python_Kriging-vecchia-nystrom_large-n,python,hartmann3/n=1500,subset of data (500),nlpd,-4.68641 -python_Kriging-vecchia-nystrom_large-n,python,hartmann3/n=5000,subset of data (500),fit_s,0.906366 -python_Kriging-vecchia-nystrom_large-n,python,hartmann3/n=5000,subset of data (500),rmse,0.00247552 -python_Kriging-vecchia-nystrom_large-n,python,hartmann3/n=5000,subset of data (500),q2,0.999994 -python_Kriging-vecchia-nystrom_large-n,python,hartmann3/n=5000,subset of data (500),nlpd,-4.71359 -python_Kriging-vecchia-nystrom_large-n,python,otl_circuit/n=1500,exact,fit_s,10.9405 -python_NestedKriging_aggregations-large-n,python,otl_circuit/n=1500,exact,fit_s,10.9405 -python_Kriging-vecchia-nystrom_large-n,python,otl_circuit/n=1500,exact,rmse,9.81598e-05 -python_NestedKriging_aggregations-large-n,python,otl_circuit/n=1500,exact,rmse,9.81598e-05 -python_Kriging-vecchia-nystrom_large-n,python,otl_circuit/n=1500,exact,q2,1 -python_NestedKriging_aggregations-large-n,python,otl_circuit/n=1500,exact,q2,1 -python_Kriging-vecchia-nystrom_large-n,python,otl_circuit/n=1500,exact,nlpd,-8.21962 -python_NestedKriging_aggregations-large-n,python,otl_circuit/n=1500,exact,nlpd,-8.21962 -python_NestedKriging_aggregations-large-n,python,otl_circuit/n=1500,nested NK (20 groups),fit_s,0.279359 -python_NestedKriging_aggregations-large-n,python,otl_circuit/n=1500,nested NK (20 groups),rmse,0.00226873 -python_NestedKriging_aggregations-large-n,python,otl_circuit/n=1500,nested NK (20 groups),q2,0.999996 -python_NestedKriging_aggregations-large-n,python,otl_circuit/n=1500,nested NK (20 groups),nlpd,-4.90542 -python_NestedKriging_aggregations-large-n,python,otl_circuit/n=5000,nested NK (20 groups),fit_s,3.08712 -python_NestedKriging_aggregations-large-n,python,otl_circuit/n=5000,nested NK (20 groups),rmse,0.000261686 -python_NestedKriging_aggregations-large-n,python,otl_circuit/n=5000,nested NK (20 groups),q2,1 -python_NestedKriging_aggregations-large-n,python,otl_circuit/n=5000,nested NK (20 groups),nlpd,-7.20623 -python_NestedKriging_aggregations-large-n,python,otl_circuit/n=1500,nested rBCM (20 groups),fit_s,0.251989 -python_NestedKriging_aggregations-large-n,python,otl_circuit/n=1500,nested rBCM (20 groups),rmse,0.00291869 -python_NestedKriging_aggregations-large-n,python,otl_circuit/n=1500,nested rBCM (20 groups),q2,0.999993 -python_NestedKriging_aggregations-large-n,python,otl_circuit/n=1500,nested rBCM (20 groups),nlpd,-2.55512 -python_NestedKriging_aggregations-large-n,python,otl_circuit/n=5000,nested rBCM (20 groups),fit_s,3.67768 -python_NestedKriging_aggregations-large-n,python,otl_circuit/n=5000,nested rBCM (20 groups),rmse,0.000278149 -python_NestedKriging_aggregations-large-n,python,otl_circuit/n=5000,nested rBCM (20 groups),q2,1 -python_NestedKriging_aggregations-large-n,python,otl_circuit/n=5000,nested rBCM (20 groups),nlpd,-4.34927 -python_Kriging-vecchia-nystrom_large-n,python,otl_circuit/n=1500,subset of data (500),fit_s,0.709236 -python_Kriging-vecchia-nystrom_large-n,python,otl_circuit/n=1500,subset of data (500),rmse,0.0004286 -python_Kriging-vecchia-nystrom_large-n,python,otl_circuit/n=1500,subset of data (500),q2,1 -python_Kriging-vecchia-nystrom_large-n,python,otl_circuit/n=1500,subset of data (500),nlpd,-6.72678 -python_Kriging-vecchia-nystrom_large-n,python,otl_circuit/n=5000,subset of data (500),fit_s,0.905534 -python_Kriging-vecchia-nystrom_large-n,python,otl_circuit/n=5000,subset of data (500),rmse,0.000446879 -python_Kriging-vecchia-nystrom_large-n,python,otl_circuit/n=5000,subset of data (500),q2,1 -python_Kriging-vecchia-nystrom_large-n,python,otl_circuit/n=5000,subset of data (500),nlpd,-6.78844 -python_NestedKriging_aggregations-large-n,python,branin/nested,NK,rmse,0.0343984 -python_NestedKriging_aggregations-large-n,python,branin/nested,NK,q2,1 -python_NestedKriging_aggregations-large-n,python,branin/nested,NK,nlpd,2.00083e+14 -python_NestedKriging_aggregations-large-n,python,branin/nested,PoE,rmse,0.0354265 -python_NestedKriging_aggregations-large-n,python,branin/nested,PoE,q2,1 -python_NestedKriging_aggregations-large-n,python,branin/nested,PoE,nlpd,-1.64533 -python_NestedKriging_aggregations-large-n,python,branin/nested,gPoE,rmse,0.0354265 -python_NestedKriging_aggregations-large-n,python,branin/nested,gPoE,q2,1 -python_NestedKriging_aggregations-large-n,python,branin/nested,gPoE,nlpd,-0.621836 -python_NestedKriging_aggregations-large-n,python,branin/nested,BCM,rmse,0.0354196 -python_NestedKriging_aggregations-large-n,python,branin/nested,BCM,q2,1 -python_NestedKriging_aggregations-large-n,python,branin/nested,BCM,nlpd,-1.64533 -python_NestedKriging_aggregations-large-n,python,branin/nested,rBCM,rmse,0.035287 -python_NestedKriging_aggregations-large-n,python,branin/nested,rBCM,q2,1 -python_NestedKriging_aggregations-large-n,python,branin/nested,rBCM,nlpd,-2.61369 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,branin,pylibkriging,q2,0.999944 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,branin,pylibkriging,nlpd,0.137225 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,branin,pylibkriging,fit_s,0.00151396 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,branin,pylibkriging BFGS10,q2,0.999945 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,branin,pylibkriging BFGS10,nlpd,0.137254 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,branin,pylibkriging BFGS10,fit_s,0.0161211 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,branin,scikit-learn,q2,0.999597 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,branin,scikit-learn,nlpd,0.928508 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,branin,scikit-learn,fit_s,0.147773 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,branin,SMT,q2,0.999944 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,branin,SMT,nlpd,0.137259 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,branin,SMT,fit_s,0.792991 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,branin,OpenTURNS,q2,0.998221 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,branin,OpenTURNS,nlpd,1.55599 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,branin,OpenTURNS,fit_s,0.00512815 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,six_hump_camel,pylibkriging,q2,0.999418 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,six_hump_camel,pylibkriging,nlpd,0.0429346 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,six_hump_camel,pylibkriging,fit_s,0.00358009 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,six_hump_camel,pylibkriging BFGS10,q2,0.999418 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,six_hump_camel,pylibkriging BFGS10,nlpd,0.0429347 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,six_hump_camel,pylibkriging BFGS10,fit_s,0.0500369 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,six_hump_camel,scikit-learn,q2,0.0163311 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,six_hump_camel,scikit-learn,nlpd,4.63763 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,six_hump_camel,scikit-learn,fit_s,0.156482 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,six_hump_camel,SMT,q2,0.999594 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,six_hump_camel,SMT,nlpd,-0.121007 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,six_hump_camel,SMT,fit_s,0.941661 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,six_hump_camel,OpenTURNS,q2,0.956713 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,six_hump_camel,OpenTURNS,nlpd,2.55446 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,six_hump_camel,OpenTURNS,fit_s,0.00930476 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann3,pylibkriging,q2,0.986148 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann3,pylibkriging,nlpd,-0.847247 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann3,pylibkriging,fit_s,0.023098 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann3,pylibkriging BFGS10,q2,0.986148 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann3,pylibkriging BFGS10,nlpd,-0.847249 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann3,pylibkriging BFGS10,fit_s,0.113887 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann3,scikit-learn,q2,0.983889 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann3,scikit-learn,nlpd,-0.782697 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann3,scikit-learn,fit_s,0.208279 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann3,SMT,q2,0.986148 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann3,SMT,nlpd,-0.847249 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann3,SMT,fit_s,1.35887 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann3,OpenTURNS,q2,0.984712 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann3,OpenTURNS,nlpd,-0.809851 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann3,OpenTURNS,fit_s,0.0169773 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,ishigami,pylibkriging,q2,0.259556 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,ishigami,pylibkriging,nlpd,4.38991 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,ishigami,pylibkriging,fit_s,0.0264924 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,ishigami,pylibkriging BFGS10,q2,0.935722 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,ishigami,pylibkriging BFGS10,nlpd,0.857443 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,ishigami,pylibkriging BFGS10,fit_s,0.322963 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,ishigami,scikit-learn,q2,0.831142 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,ishigami,scikit-learn,nlpd,1.5455 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,ishigami,scikit-learn,fit_s,0.512728 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,ishigami,SMT,q2,0.51196 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,ishigami,SMT,nlpd,3.31349 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,ishigami,SMT,fit_s,2.70852 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,ishigami,OpenTURNS,q2,0.887251 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,ishigami,OpenTURNS,nlpd,1.39348 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,ishigami,OpenTURNS,fit_s,0.0510182 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,friedman,pylibkriging,q2,0.999771 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,friedman,pylibkriging,nlpd,-1.29726 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,friedman,pylibkriging,fit_s,0.0165708 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,friedman,pylibkriging BFGS10,q2,0.999771 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,friedman,pylibkriging BFGS10,nlpd,-1.29727 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,friedman,pylibkriging BFGS10,fit_s,0.279774 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,friedman,scikit-learn,q2,0.999754 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,friedman,scikit-learn,nlpd,-1.37702 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,friedman,scikit-learn,fit_s,0.308127 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,friedman,SMT,q2,0.999899 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,friedman,SMT,nlpd,-1.871 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,friedman,SMT,fit_s,4.71948 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,friedman,OpenTURNS,q2,0.990862 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,friedman,OpenTURNS,nlpd,0.469828 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,friedman,OpenTURNS,fit_s,0.0511642 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,otl_circuit,pylibkriging,q2,0.999922 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,otl_circuit,pylibkriging,nlpd,-3.46228 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,otl_circuit,pylibkriging,fit_s,0.0136361 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,otl_circuit,pylibkriging BFGS10,q2,0.999922 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,otl_circuit,pylibkriging BFGS10,nlpd,-3.46227 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,otl_circuit,pylibkriging BFGS10,fit_s,0.168652 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,otl_circuit,scikit-learn,q2,0.999906 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,otl_circuit,scikit-learn,nlpd,-3.43062 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,otl_circuit,scikit-learn,fit_s,0.64982 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,otl_circuit,SMT,q2,0.999954 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,otl_circuit,SMT,nlpd,-3.35016 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,otl_circuit,SMT,fit_s,6.837 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,otl_circuit,OpenTURNS,q2,0.99777 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,otl_circuit,OpenTURNS,nlpd,-1.61723 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,otl_circuit,OpenTURNS,fit_s,0.0662544 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,piston,pylibkriging,q2,0.997498 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,piston,pylibkriging,nlpd,-3.80667 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,piston,pylibkriging,fit_s,0.03055 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,piston,pylibkriging BFGS10,q2,0.997498 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,piston,pylibkriging BFGS10,nlpd,-3.8067 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,piston,pylibkriging BFGS10,fit_s,0.326413 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,piston,scikit-learn,q2,0.998684 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,piston,scikit-learn,nlpd,-4.10497 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,piston,scikit-learn,fit_s,0.856336 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,piston,SMT,q2,0.997511 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,piston,SMT,nlpd,-3.86618 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,piston,SMT,fit_s,12.0067 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,piston,OpenTURNS,q2,0.990631 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,piston,OpenTURNS,nlpd,-2.9906 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,piston,OpenTURNS,fit_s,0.137117 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,borehole,pylibkriging,q2,0.999694 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,borehole,pylibkriging,nlpd,1.00118 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,borehole,pylibkriging,fit_s,0.0166521 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,borehole,pylibkriging BFGS10,q2,0.999694 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,borehole,pylibkriging BFGS10,nlpd,1.00121 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,borehole,pylibkriging BFGS10,fit_s,0.200685 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,borehole,scikit-learn,q2,0.999929 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,borehole,scikit-learn,nlpd,0.194455 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,borehole,scikit-learn,fit_s,1.18352 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,borehole,SMT,q2,0.999925 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,borehole,SMT,nlpd,0.279122 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,borehole,SMT,fit_s,12.3314 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,borehole,OpenTURNS,q2,0.996206 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,borehole,OpenTURNS,nlpd,2.55431 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,borehole,OpenTURNS,fit_s,0.121907 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,wing_weight,pylibkriging,q2,0.99972 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,wing_weight,pylibkriging,nlpd,1.11517 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,wing_weight,pylibkriging,fit_s,0.0297599 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,wing_weight,pylibkriging BFGS10,q2,0.99972 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,wing_weight,pylibkriging BFGS10,nlpd,1.11512 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,wing_weight,pylibkriging BFGS10,fit_s,0.335854 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,wing_weight,scikit-learn,q2,0.998048 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,wing_weight,scikit-learn,nlpd,2.57304 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,wing_weight,scikit-learn,fit_s,0.833252 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,wing_weight,SMT,q2,0.999898 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,wing_weight,SMT,nlpd,0.604023 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,wing_weight,SMT,fit_s,26.7889 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,wing_weight,OpenTURNS,q2,0.993145 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,wing_weight,OpenTURNS,nlpd,2.83627 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,wing_weight,OpenTURNS,fit_s,0.253929 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann6,pylibkriging,q2,0.422939 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann6,pylibkriging,nlpd,0.0965158 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann6,pylibkriging,fit_s,0.068651 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann6,pylibkriging BFGS10,q2,0.72352 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann6,pylibkriging BFGS10,nlpd,-0.156898 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann6,pylibkriging BFGS10,fit_s,0.877097 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann6,scikit-learn,q2,0.69049 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann6,scikit-learn,nlpd,-0.137845 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann6,scikit-learn,fit_s,1.43153 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann6,SMT,q2,0.708463 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann6,SMT,nlpd,-0.0177081 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann6,SMT,fit_s,11.2226 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann6,OpenTURNS,q2,0.696177 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann6,OpenTURNS,nlpd,-0.132838 -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,python,hartmann6,OpenTURNS,fit_s,0.141383 +suite,lang,case,package,metric,value +kernels-trends-interpolation_Kriging_python,python,forrester/gauss/constant,pylibkriging,fit_s,0.0039537 +kernels-trends-interpolation_Kriging_python,python,forrester/gauss/constant,pylibkriging,rmse,0.0848271 +kernels-trends-interpolation_Kriging_python,python,forrester/gauss/constant,pylibkriging,q2,0.999634 +kernels-trends-interpolation_Kriging_python,python,forrester/gauss/constant,pylibkriging,nlpd,-3.06486 +kernels-trends-interpolation_Kriging_python,python,forrester/gauss/linear,pylibkriging,fit_s,0.0008564 +kernels-trends-interpolation_Kriging_python,python,forrester/gauss/linear,pylibkriging,rmse,0.0855468 +kernels-trends-interpolation_Kriging_python,python,forrester/gauss/linear,pylibkriging,q2,0.999628 +kernels-trends-interpolation_Kriging_python,python,forrester/gauss/linear,pylibkriging,nlpd,-3.11029 +kernels-trends-interpolation_Kriging_python,python,forrester/gauss/interactive,pylibkriging,fit_s,0.000818729 +kernels-trends-interpolation_Kriging_python,python,forrester/gauss/interactive,pylibkriging,rmse,0.0855468 +kernels-trends-interpolation_Kriging_python,python,forrester/gauss/interactive,pylibkriging,q2,0.999628 +kernels-trends-interpolation_Kriging_python,python,forrester/gauss/interactive,pylibkriging,nlpd,-3.11029 +kernels-trends-interpolation_Kriging_python,python,forrester/gauss/quadratic,pylibkriging,fit_s,0.000473261 +kernels-trends-interpolation_Kriging_python,python,forrester/gauss/quadratic,pylibkriging,rmse,1.57249 +kernels-trends-interpolation_Kriging_python,python,forrester/gauss/quadratic,pylibkriging,q2,0.874394 +kernels-trends-interpolation_Kriging_python,python,forrester/gauss/quadratic,pylibkriging,nlpd,5.33879 +kernels-trends-interpolation_Kriging_python,python,forrester/exp/constant,pylibkriging,fit_s,0.00042963 +kernels-trends-interpolation_Kriging_python,python,forrester/exp/constant,pylibkriging,rmse,0.721582 +kernels-trends-interpolation_Kriging_python,python,forrester/exp/constant,pylibkriging,q2,0.973551 +kernels-trends-interpolation_Kriging_python,python,forrester/exp/constant,pylibkriging,nlpd,1.54551 +kernels-trends-interpolation_Kriging_python,python,forrester/exp/linear,pylibkriging,fit_s,0.000402451 +kernels-trends-interpolation_Kriging_python,python,forrester/exp/linear,pylibkriging,rmse,0.793413 +kernels-trends-interpolation_Kriging_python,python,forrester/exp/linear,pylibkriging,q2,0.968024 +kernels-trends-interpolation_Kriging_python,python,forrester/exp/linear,pylibkriging,nlpd,1.50639 +kernels-trends-interpolation_Kriging_python,python,forrester/exp/interactive,pylibkriging,fit_s,0.000428438 +kernels-trends-interpolation_Kriging_python,python,forrester/exp/interactive,pylibkriging,rmse,0.793413 +kernels-trends-interpolation_Kriging_python,python,forrester/exp/interactive,pylibkriging,q2,0.968024 +kernels-trends-interpolation_Kriging_python,python,forrester/exp/interactive,pylibkriging,nlpd,1.50639 +kernels-trends-interpolation_Kriging_python,python,forrester/exp/quadratic,pylibkriging,fit_s,0.000418425 +kernels-trends-interpolation_Kriging_python,python,forrester/exp/quadratic,pylibkriging,rmse,0.723267 +kernels-trends-interpolation_Kriging_python,python,forrester/exp/quadratic,pylibkriging,q2,0.973428 +kernels-trends-interpolation_Kriging_python,python,forrester/exp/quadratic,pylibkriging,nlpd,1.28782 +kernels-trends-interpolation_Kriging_python,python,forrester/matern3_2/constant,pylibkriging,fit_s,0.000519753 +kernels-trends-interpolation_Kriging_python,python,forrester/matern3_2/constant,pylibkriging,rmse,0.334007 +kernels-trends-interpolation_Kriging_python,python,forrester/matern3_2/constant,pylibkriging,q2,0.994333 +kernels-trends-interpolation_Kriging_python,python,forrester/matern3_2/constant,pylibkriging,nlpd,-0.227715 +kernels-trends-interpolation_Kriging_python,python,forrester/matern3_2/linear,pylibkriging,fit_s,0.000559092 +kernels-trends-interpolation_Kriging_python,python,forrester/matern3_2/linear,pylibkriging,rmse,0.364477 +kernels-trends-interpolation_Kriging_python,python,forrester/matern3_2/linear,pylibkriging,q2,0.993252 +kernels-trends-interpolation_Kriging_python,python,forrester/matern3_2/linear,pylibkriging,nlpd,-0.187262 +kernels-trends-interpolation_Kriging_python,python,forrester/matern3_2/interactive,pylibkriging,fit_s,0.00060463 +kernels-trends-interpolation_Kriging_python,python,forrester/matern3_2/interactive,pylibkriging,rmse,0.364477 +kernels-trends-interpolation_Kriging_python,python,forrester/matern3_2/interactive,pylibkriging,q2,0.993252 +kernels-trends-interpolation_Kriging_python,python,forrester/matern3_2/interactive,pylibkriging,nlpd,-0.187262 +kernels-trends-interpolation_Kriging_python,python,forrester/matern3_2/quadratic,pylibkriging,fit_s,0.000493765 +kernels-trends-interpolation_Kriging_python,python,forrester/matern3_2/quadratic,pylibkriging,rmse,0.435609 +kernels-trends-interpolation_Kriging_python,python,forrester/matern3_2/quadratic,pylibkriging,q2,0.990361 +kernels-trends-interpolation_Kriging_python,python,forrester/matern3_2/quadratic,pylibkriging,nlpd,-0.183076 +kernels-trends-interpolation_Kriging_python,python,forrester/matern5_2/constant,pylibkriging,fit_s,0.000758886 +kernels-trends-interpolation_Kriging_python,python,forrester/matern5_2/constant,pylibkriging,rmse,0.281945 +kernels-trends-interpolation_Kriging_python,python,forrester/matern5_2/constant,pylibkriging,q2,0.995962 +kernels-trends-interpolation_Kriging_python,python,forrester/matern5_2/constant,pylibkriging,nlpd,-1.01945 +kernels-trends-interpolation_Kriging_python,python,forrester/matern5_2/constant,pylibkriging BFGS10,rmse,0.281945 +kernels-trends-interpolation_Kriging_python,python,forrester/matern5_2/constant,pylibkriging BFGS10,q2,0.995962 +kernels-trends-interpolation_Kriging_python,python,forrester/matern5_2/constant,pylibkriging BFGS10,nlpd,-1.01945 +kernels-trends-interpolation_Kriging_python,python,forrester/matern5_2/linear,pylibkriging,fit_s,0.000576973 +kernels-trends-interpolation_Kriging_python,python,forrester/matern5_2/linear,pylibkriging,rmse,0.298959 +kernels-trends-interpolation_Kriging_python,python,forrester/matern5_2/linear,pylibkriging,q2,0.99546 +kernels-trends-interpolation_Kriging_python,python,forrester/matern5_2/linear,pylibkriging,nlpd,-0.95551 +kernels-trends-interpolation_Kriging_python,python,forrester/matern5_2/interactive,pylibkriging,fit_s,0.00054884 +kernels-trends-interpolation_Kriging_python,python,forrester/matern5_2/interactive,pylibkriging,rmse,0.298959 +kernels-trends-interpolation_Kriging_python,python,forrester/matern5_2/interactive,pylibkriging,q2,0.99546 +kernels-trends-interpolation_Kriging_python,python,forrester/matern5_2/interactive,pylibkriging,nlpd,-0.95551 +kernels-trends-interpolation_Kriging_python,python,forrester/matern5_2/quadratic,pylibkriging,fit_s,0.000477076 +kernels-trends-interpolation_Kriging_python,python,forrester/matern5_2/quadratic,pylibkriging,rmse,0.365039 +kernels-trends-interpolation_Kriging_python,python,forrester/matern5_2/quadratic,pylibkriging,q2,0.993231 +kernels-trends-interpolation_Kriging_python,python,forrester/matern5_2/quadratic,pylibkriging,nlpd,-0.790032 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/gauss/constant,pylibkriging,fit_s,0.0016222 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/gauss/constant,pylibkriging,rmse,0.191274 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/gauss/constant,pylibkriging,q2,0.978742 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/gauss/constant,pylibkriging,nlpd,-0.704948 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/gauss/linear,pylibkriging,fit_s,0.00240469 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/gauss/linear,pylibkriging,rmse,0.236028 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/gauss/linear,pylibkriging,q2,0.96763 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/gauss/linear,pylibkriging,nlpd,-0.580993 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/gauss/interactive,pylibkriging,fit_s,0.00205922 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/gauss/interactive,pylibkriging,rmse,0.236028 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/gauss/interactive,pylibkriging,q2,0.96763 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/gauss/interactive,pylibkriging,nlpd,-0.580993 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/gauss/quadratic,pylibkriging,fit_s,0.00200701 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/gauss/quadratic,pylibkriging,rmse,0.150914 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/gauss/quadratic,pylibkriging,q2,0.986767 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/gauss/quadratic,pylibkriging,nlpd,-0.458431 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/exp/constant,pylibkriging,fit_s,0.00123 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/exp/constant,pylibkriging,rmse,0.196586 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/exp/constant,pylibkriging,q2,0.977545 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/exp/constant,pylibkriging,nlpd,-0.0621192 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/exp/linear,pylibkriging,fit_s,0.00172782 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/exp/linear,pylibkriging,rmse,0.20979 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/exp/linear,pylibkriging,q2,0.974427 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/exp/linear,pylibkriging,nlpd,-0.0466568 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/exp/interactive,pylibkriging,fit_s,0.0017736 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/exp/interactive,pylibkriging,rmse,0.20979 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/exp/interactive,pylibkriging,q2,0.974427 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/exp/interactive,pylibkriging,nlpd,-0.0466568 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/exp/quadratic,pylibkriging,fit_s,0.00394821 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/exp/quadratic,pylibkriging,rmse,0.252565 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/exp/quadratic,pylibkriging,q2,0.962935 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/exp/quadratic,pylibkriging,nlpd,0.220609 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern3_2/constant,pylibkriging,fit_s,0.000825167 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern3_2/constant,pylibkriging,rmse,0.177253 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern3_2/constant,pylibkriging,q2,0.981744 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern3_2/constant,pylibkriging,nlpd,-0.482212 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern3_2/linear,pylibkriging,fit_s,0.000857115 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern3_2/linear,pylibkriging,rmse,0.211499 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern3_2/linear,pylibkriging,q2,0.974009 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern3_2/linear,pylibkriging,nlpd,-0.375844 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern3_2/interactive,pylibkriging,fit_s,0.00114012 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern3_2/interactive,pylibkriging,rmse,0.211499 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern3_2/interactive,pylibkriging,q2,0.974009 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern3_2/interactive,pylibkriging,nlpd,-0.375844 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern3_2/quadratic,pylibkriging,fit_s,0.00237894 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern3_2/quadratic,pylibkriging,rmse,0.184528 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern3_2/quadratic,pylibkriging,q2,0.980215 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern3_2/quadratic,pylibkriging,nlpd,-0.171795 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern5_2/constant,pylibkriging,fit_s,0.000836849 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern5_2/constant,pylibkriging,rmse,0.177638 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern5_2/constant,pylibkriging,q2,0.981665 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern5_2/constant,pylibkriging,nlpd,-0.575421 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern5_2/constant,pylibkriging BFGS10,rmse,0.177638 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern5_2/constant,pylibkriging BFGS10,q2,0.981665 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern5_2/constant,pylibkriging BFGS10,nlpd,-0.575422 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern5_2/linear,pylibkriging,fit_s,0.000859499 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern5_2/linear,pylibkriging,rmse,0.219759 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern5_2/linear,pylibkriging,q2,0.971939 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern5_2/linear,pylibkriging,nlpd,-0.458933 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern5_2/interactive,pylibkriging,fit_s,0.00080061 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern5_2/interactive,pylibkriging,rmse,0.219759 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern5_2/interactive,pylibkriging,q2,0.971939 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern5_2/interactive,pylibkriging,nlpd,-0.458933 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern5_2/quadratic,pylibkriging,fit_s,0.00249147 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern5_2/quadratic,pylibkriging,rmse,0.169941 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern5_2/quadratic,pylibkriging,q2,0.983219 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern5_2/quadratic,pylibkriging,nlpd,-0.28938 +kernels-trends-interpolation_Kriging_python,python,branin/gauss/constant,pylibkriging,fit_s,0.00172019 +kernels-trends-interpolation_Kriging_python,python,branin/gauss/constant,pylibkriging,rmse,7.2629 +kernels-trends-interpolation_Kriging_python,python,branin/gauss/constant,pylibkriging,q2,0.980655 +kernels-trends-interpolation_Kriging_python,python,branin/gauss/constant,pylibkriging,nlpd,4.3152 +kernels-trends-interpolation_Kriging_python,python,branin/gauss/linear,pylibkriging,fit_s,0.00188136 +kernels-trends-interpolation_Kriging_python,python,branin/gauss/linear,pylibkriging,rmse,7.25733 +kernels-trends-interpolation_Kriging_python,python,branin/gauss/linear,pylibkriging,q2,0.980685 +kernels-trends-interpolation_Kriging_python,python,branin/gauss/linear,pylibkriging,nlpd,4.30813 +kernels-trends-interpolation_Kriging_python,python,branin/gauss/interactive,pylibkriging,fit_s,0.00456238 +kernels-trends-interpolation_Kriging_python,python,branin/gauss/interactive,pylibkriging,rmse,7.2576 +kernels-trends-interpolation_Kriging_python,python,branin/gauss/interactive,pylibkriging,q2,0.980683 +kernels-trends-interpolation_Kriging_python,python,branin/gauss/interactive,pylibkriging,nlpd,4.30176 +kernels-trends-interpolation_Kriging_python,python,branin/gauss/quadratic,pylibkriging,fit_s,0.00182772 +kernels-trends-interpolation_Kriging_python,python,branin/gauss/quadratic,pylibkriging,rmse,7.27808 +kernels-trends-interpolation_Kriging_python,python,branin/gauss/quadratic,pylibkriging,q2,0.980574 +kernels-trends-interpolation_Kriging_python,python,branin/gauss/quadratic,pylibkriging,nlpd,4.29399 +kernels-trends-interpolation_Kriging_python,python,branin/exp/constant,pylibkriging,fit_s,0.00215006 +kernels-trends-interpolation_Kriging_python,python,branin/exp/constant,pylibkriging,rmse,14.5887 +kernels-trends-interpolation_Kriging_python,python,branin/exp/constant,pylibkriging,q2,0.921947 +kernels-trends-interpolation_Kriging_python,python,branin/exp/constant,pylibkriging,nlpd,4.09457 +kernels-trends-interpolation_Kriging_python,python,branin/exp/linear,pylibkriging,fit_s,0.00210071 +kernels-trends-interpolation_Kriging_python,python,branin/exp/linear,pylibkriging,rmse,14.9622 +kernels-trends-interpolation_Kriging_python,python,branin/exp/linear,pylibkriging,q2,0.9179 +kernels-trends-interpolation_Kriging_python,python,branin/exp/linear,pylibkriging,nlpd,4.0909 +kernels-trends-interpolation_Kriging_python,python,branin/exp/interactive,pylibkriging,fit_s,0.00220799 +kernels-trends-interpolation_Kriging_python,python,branin/exp/interactive,pylibkriging,rmse,11.6963 +kernels-trends-interpolation_Kriging_python,python,branin/exp/interactive,pylibkriging,q2,0.94983 +kernels-trends-interpolation_Kriging_python,python,branin/exp/interactive,pylibkriging,nlpd,3.79671 +kernels-trends-interpolation_Kriging_python,python,branin/exp/quadratic,pylibkriging,fit_s,0.00237751 +kernels-trends-interpolation_Kriging_python,python,branin/exp/quadratic,pylibkriging,rmse,9.33909 +kernels-trends-interpolation_Kriging_python,python,branin/exp/quadratic,pylibkriging,q2,0.968014 +kernels-trends-interpolation_Kriging_python,python,branin/exp/quadratic,pylibkriging,nlpd,3.62362 +kernels-trends-interpolation_Kriging_python,python,branin/matern3_2/constant,pylibkriging,fit_s,0.00252843 +kernels-trends-interpolation_Kriging_python,python,branin/matern3_2/constant,pylibkriging,rmse,3.24796 +kernels-trends-interpolation_Kriging_python,python,branin/matern3_2/constant,pylibkriging,q2,0.996131 +kernels-trends-interpolation_Kriging_python,python,branin/matern3_2/constant,pylibkriging,nlpd,2.30444 +kernels-trends-interpolation_Kriging_python,python,branin/matern3_2/linear,pylibkriging,fit_s,0.00200725 +kernels-trends-interpolation_Kriging_python,python,branin/matern3_2/linear,pylibkriging,rmse,3.14402 +kernels-trends-interpolation_Kriging_python,python,branin/matern3_2/linear,pylibkriging,q2,0.996375 +kernels-trends-interpolation_Kriging_python,python,branin/matern3_2/linear,pylibkriging,nlpd,2.28067 +kernels-trends-interpolation_Kriging_python,python,branin/matern3_2/interactive,pylibkriging,fit_s,0.00414944 +kernels-trends-interpolation_Kriging_python,python,branin/matern3_2/interactive,pylibkriging,rmse,3.19226 +kernels-trends-interpolation_Kriging_python,python,branin/matern3_2/interactive,pylibkriging,q2,0.996263 +kernels-trends-interpolation_Kriging_python,python,branin/matern3_2/interactive,pylibkriging,nlpd,2.20359 +kernels-trends-interpolation_Kriging_python,python,branin/matern3_2/quadratic,pylibkriging,fit_s,0.00253224 +kernels-trends-interpolation_Kriging_python,python,branin/matern3_2/quadratic,pylibkriging,rmse,3.01232 +kernels-trends-interpolation_Kriging_python,python,branin/matern3_2/quadratic,pylibkriging,q2,0.996672 +kernels-trends-interpolation_Kriging_python,python,branin/matern3_2/quadratic,pylibkriging,nlpd,2.11627 +kernels-trends-interpolation_Kriging_python,python,branin/matern5_2/constant,pylibkriging,fit_s,0.00171685 +kernels-trends-interpolation_Kriging_python,python,branin/matern5_2/constant,pylibkriging,rmse,0.52012 +kernels-trends-interpolation_Kriging_python,python,branin/matern5_2/constant,pylibkriging,q2,0.999901 +kernels-trends-interpolation_Kriging_python,python,branin/matern5_2/constant,pylibkriging,nlpd,0.283399 +kernels-trends-interpolation_Kriging_python,python,branin/matern5_2/constant,pylibkriging BFGS10,rmse,0.52012 +kernels-trends-interpolation_Kriging_python,python,branin/matern5_2/constant,pylibkriging BFGS10,q2,0.999901 +kernels-trends-interpolation_Kriging_python,python,branin/matern5_2/constant,pylibkriging BFGS10,nlpd,0.283395 +kernels-trends-interpolation_Kriging_python,python,branin/matern5_2/linear,pylibkriging,fit_s,0.0021255 +kernels-trends-interpolation_Kriging_python,python,branin/matern5_2/linear,pylibkriging,rmse,0.460868 +kernels-trends-interpolation_Kriging_python,python,branin/matern5_2/linear,pylibkriging,q2,0.999922 +kernels-trends-interpolation_Kriging_python,python,branin/matern5_2/linear,pylibkriging,nlpd,0.209741 +kernels-trends-interpolation_Kriging_python,python,branin/matern5_2/interactive,pylibkriging,fit_s,0.00307012 +kernels-trends-interpolation_Kriging_python,python,branin/matern5_2/interactive,pylibkriging,rmse,0.498484 +kernels-trends-interpolation_Kriging_python,python,branin/matern5_2/interactive,pylibkriging,q2,0.999909 +kernels-trends-interpolation_Kriging_python,python,branin/matern5_2/interactive,pylibkriging,nlpd,0.239079 +kernels-trends-interpolation_Kriging_python,python,branin/matern5_2/quadratic,pylibkriging,fit_s,0.0092895 +kernels-trends-interpolation_Kriging_python,python,branin/matern5_2/quadratic,pylibkriging,rmse,0.67472 +kernels-trends-interpolation_Kriging_python,python,branin/matern5_2/quadratic,pylibkriging,q2,0.999833 +kernels-trends-interpolation_Kriging_python,python,branin/matern5_2/quadratic,pylibkriging,nlpd,0.419266 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/gauss/constant,pylibkriging,fit_s,0.00194049 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/gauss/constant,pylibkriging,rmse,18.9501 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/gauss/constant,pylibkriging,q2,0.536354 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/gauss/constant,pylibkriging,nlpd,84.151 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/gauss/linear,pylibkriging,fit_s,0.00170755 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/gauss/linear,pylibkriging,rmse,18.9579 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/gauss/linear,pylibkriging,q2,0.535976 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/gauss/linear,pylibkriging,nlpd,44.8696 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/gauss/interactive,pylibkriging,fit_s,0.00180864 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/gauss/interactive,pylibkriging,rmse,18.9819 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/gauss/interactive,pylibkriging,q2,0.5348 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/gauss/interactive,pylibkriging,nlpd,35.2765 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/gauss/quadratic,pylibkriging,fit_s,0.00178218 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/gauss/quadratic,pylibkriging,rmse,10.9058 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/gauss/quadratic,pylibkriging,q2,0.846441 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/gauss/quadratic,pylibkriging,nlpd,4.92017 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/exp/constant,pylibkriging,fit_s,0.000977039 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/exp/constant,pylibkriging,rmse,16.9653 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/exp/constant,pylibkriging,q2,0.628393 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/exp/constant,pylibkriging,nlpd,4.48595 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/exp/linear,pylibkriging,fit_s,0.00107837 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/exp/linear,pylibkriging,rmse,17.1781 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/exp/linear,pylibkriging,q2,0.619012 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/exp/linear,pylibkriging,nlpd,4.47061 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/exp/interactive,pylibkriging,fit_s,0.00123405 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/exp/interactive,pylibkriging,rmse,17.1856 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/exp/interactive,pylibkriging,q2,0.618679 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/exp/interactive,pylibkriging,nlpd,4.43133 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/exp/quadratic,pylibkriging,fit_s,0.00120115 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/exp/quadratic,pylibkriging,rmse,7.99484 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/exp/quadratic,pylibkriging,q2,0.917476 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/exp/quadratic,pylibkriging,nlpd,3.35612 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern3_2/constant,pylibkriging,fit_s,0.00264788 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern3_2/constant,pylibkriging,rmse,18.4958 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern3_2/constant,pylibkriging,q2,0.558321 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern3_2/constant,pylibkriging,nlpd,18.8349 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern3_2/linear,pylibkriging,fit_s,0.00251079 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern3_2/linear,pylibkriging,rmse,18.5562 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern3_2/linear,pylibkriging,q2,0.555433 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern3_2/linear,pylibkriging,nlpd,14.9494 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern3_2/interactive,pylibkriging,fit_s,0.00148797 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern3_2/interactive,pylibkriging,rmse,18.561 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern3_2/interactive,pylibkriging,q2,0.555202 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern3_2/interactive,pylibkriging,nlpd,12.9923 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern3_2/quadratic,pylibkriging,fit_s,0.00289869 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern3_2/quadratic,pylibkriging,rmse,8.91645 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern3_2/quadratic,pylibkriging,q2,0.897353 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern3_2/quadratic,pylibkriging,nlpd,3.94939 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern5_2/constant,pylibkriging,fit_s,0.00238967 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern5_2/constant,pylibkriging,rmse,18.3236 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern5_2/constant,pylibkriging,q2,0.566507 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern5_2/constant,pylibkriging,nlpd,18.4322 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern5_2/constant,pylibkriging BFGS10,rmse,1.36208 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern5_2/constant,pylibkriging BFGS10,q2,0.997605 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern5_2/constant,pylibkriging BFGS10,nlpd,1.03215 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern5_2/linear,pylibkriging,fit_s,0.00207353 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern5_2/linear,pylibkriging,rmse,18.7127 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern5_2/linear,pylibkriging,q2,0.547901 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern5_2/linear,pylibkriging,nlpd,24.5771 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern5_2/interactive,pylibkriging,fit_s,0.0020268 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern5_2/interactive,pylibkriging,rmse,18.7194 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern5_2/interactive,pylibkriging,q2,0.547577 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern5_2/interactive,pylibkriging,nlpd,20.4255 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern5_2/quadratic,pylibkriging,fit_s,0.00144219 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern5_2/quadratic,pylibkriging,rmse,10.2878 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern5_2/quadratic,pylibkriging,q2,0.863351 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern5_2/quadratic,pylibkriging,nlpd,4.46344 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/gauss/constant,pylibkriging,fit_s,0.00526643 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/gauss/constant,pylibkriging,rmse,0.295934 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/gauss/constant,pylibkriging,q2,0.920537 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/gauss/constant,pylibkriging,nlpd,1.58666 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/gauss/linear,pylibkriging,fit_s,0.00606179 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/gauss/linear,pylibkriging,rmse,0.296355 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/gauss/linear,pylibkriging,q2,0.92031 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/gauss/linear,pylibkriging,nlpd,1.2962 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/gauss/interactive,pylibkriging,fit_s,0.00633311 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/gauss/interactive,pylibkriging,rmse,0.285136 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/gauss/interactive,pylibkriging,q2,0.926229 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/gauss/interactive,pylibkriging,nlpd,1.04122 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/gauss/quadratic,pylibkriging,fit_s,0.00615239 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/gauss/quadratic,pylibkriging,rmse,0.237308 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/gauss/quadratic,pylibkriging,q2,0.948902 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/gauss/quadratic,pylibkriging,nlpd,0.146484 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/exp/constant,pylibkriging,fit_s,0.00271893 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/exp/constant,pylibkriging,rmse,0.334876 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/exp/constant,pylibkriging,q2,0.898247 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/exp/constant,pylibkriging,nlpd,0.339235 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/exp/linear,pylibkriging,fit_s,0.00223827 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/exp/linear,pylibkriging,rmse,0.333447 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/exp/linear,pylibkriging,q2,0.899114 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/exp/linear,pylibkriging,nlpd,0.336515 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/exp/interactive,pylibkriging,fit_s,0.00192547 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/exp/interactive,pylibkriging,rmse,0.336819 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/exp/interactive,pylibkriging,q2,0.897063 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/exp/interactive,pylibkriging,nlpd,0.329566 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/exp/quadratic,pylibkriging,fit_s,0.00333118 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/exp/quadratic,pylibkriging,rmse,0.327816 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/exp/quadratic,pylibkriging,q2,0.902492 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/exp/quadratic,pylibkriging,nlpd,0.322056 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern3_2/constant,pylibkriging,fit_s,0.00606227 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern3_2/constant,pylibkriging,rmse,0.684362 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern3_2/constant,pylibkriging,q2,0.575038 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern3_2/constant,pylibkriging,nlpd,0.972803 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern3_2/linear,pylibkriging,fit_s,0.00382805 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern3_2/linear,pylibkriging,rmse,0.650201 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern3_2/linear,pylibkriging,q2,0.616404 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern3_2/linear,pylibkriging,nlpd,0.921455 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern3_2/interactive,pylibkriging,fit_s,0.00707507 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern3_2/interactive,pylibkriging,rmse,0.274326 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern3_2/interactive,pylibkriging,q2,0.931717 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern3_2/interactive,pylibkriging,nlpd,0.0549679 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern3_2/quadratic,pylibkriging,fit_s,0.00617123 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern3_2/quadratic,pylibkriging,rmse,0.263745 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern3_2/quadratic,pylibkriging,q2,0.936883 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern3_2/quadratic,pylibkriging,nlpd,0.0446267 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern5_2/constant,pylibkriging,fit_s,0.00426936 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern5_2/constant,pylibkriging,rmse,0.674391 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern5_2/constant,pylibkriging,q2,0.58733 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern5_2/constant,pylibkriging,nlpd,0.944765 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern5_2/constant,pylibkriging BFGS10,rmse,0.268216 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern5_2/constant,pylibkriging BFGS10,q2,0.934725 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern5_2/constant,pylibkriging BFGS10,nlpd,0.169585 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern5_2/linear,pylibkriging,fit_s,0.00730562 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern5_2/linear,pylibkriging,rmse,0.268808 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern5_2/linear,pylibkriging,q2,0.934436 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern5_2/linear,pylibkriging,nlpd,0.142382 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern5_2/interactive,pylibkriging,fit_s,0.00520349 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern5_2/interactive,pylibkriging,rmse,0.271088 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern5_2/interactive,pylibkriging,q2,0.933319 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern5_2/interactive,pylibkriging,nlpd,0.128788 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern5_2/quadratic,pylibkriging,fit_s,0.00757623 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern5_2/quadratic,pylibkriging,rmse,0.253093 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern5_2/quadratic,pylibkriging,q2,0.941878 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern5_2/quadratic,pylibkriging,nlpd,0.025469 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/gauss/constant,pylibkriging,fit_s,0.000713825 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/gauss/constant,pylibkriging,rmse,0.026127 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/gauss/constant,pylibkriging,q2,0.999737 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/gauss/constant,pylibkriging,nlpd,-2.69817 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/gauss/linear,pylibkriging,fit_s,0.000786066 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/gauss/linear,pylibkriging,rmse,1.06243 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/gauss/linear,pylibkriging,q2,0.564691 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/gauss/linear,pylibkriging,nlpd,2.7394 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/gauss/interactive,pylibkriging,fit_s,0.000851393 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/gauss/interactive,pylibkriging,rmse,1.06504 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/gauss/interactive,pylibkriging,q2,0.562551 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/gauss/interactive,pylibkriging,nlpd,2.33575 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/gauss/quadratic,pylibkriging,fit_s,0.0012238 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/gauss/quadratic,pylibkriging,rmse,0.281816 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/gauss/quadratic,pylibkriging,q2,0.969371 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/gauss/quadratic,pylibkriging,nlpd,0.71682 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/exp/constant,pylibkriging,fit_s,0.000771999 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/exp/constant,pylibkriging,rmse,0.249023 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/exp/constant,pylibkriging,q2,0.976085 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/exp/constant,pylibkriging,nlpd,0.394159 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/exp/linear,pylibkriging,fit_s,0.0006814 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/exp/linear,pylibkriging,rmse,0.245558 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/exp/linear,pylibkriging,q2,0.976746 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/exp/linear,pylibkriging,nlpd,0.224593 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/exp/interactive,pylibkriging,fit_s,0.000779867 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/exp/interactive,pylibkriging,rmse,0.239032 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/exp/interactive,pylibkriging,q2,0.977965 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/exp/interactive,pylibkriging,nlpd,0.226133 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/exp/quadratic,pylibkriging,fit_s,0.00164485 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/exp/quadratic,pylibkriging,rmse,0.237713 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/exp/quadratic,pylibkriging,q2,0.978208 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/exp/quadratic,pylibkriging,nlpd,-0.139582 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern3_2/constant,pylibkriging,fit_s,0.000581741 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern3_2/constant,pylibkriging,rmse,0.0600911 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern3_2/constant,pylibkriging,q2,0.998607 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern3_2/constant,pylibkriging,nlpd,-1.34658 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern3_2/linear,pylibkriging,fit_s,0.000698566 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern3_2/linear,pylibkriging,rmse,0.0584376 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern3_2/linear,pylibkriging,q2,0.998683 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern3_2/linear,pylibkriging,nlpd,-1.34912 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern3_2/interactive,pylibkriging,fit_s,0.000763416 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern3_2/interactive,pylibkriging,rmse,0.0576096 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern3_2/interactive,pylibkriging,q2,0.99872 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern3_2/interactive,pylibkriging,nlpd,-1.35376 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern3_2/quadratic,pylibkriging,fit_s,0.00313497 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern3_2/quadratic,pylibkriging,rmse,0.215336 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern3_2/quadratic,pylibkriging,q2,0.982117 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern3_2/quadratic,pylibkriging,nlpd,-0.350837 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern5_2/constant,pylibkriging,fit_s,0.000642776 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern5_2/constant,pylibkriging,rmse,0.0246348 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern5_2/constant,pylibkriging,q2,0.999766 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern5_2/constant,pylibkriging,nlpd,-2.59598 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern5_2/constant,pylibkriging BFGS10,rmse,0.0246338 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern5_2/constant,pylibkriging BFGS10,q2,0.999766 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern5_2/constant,pylibkriging BFGS10,nlpd,-2.59598 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern5_2/linear,pylibkriging,fit_s,0.000878334 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern5_2/linear,pylibkriging,rmse,0.0232483 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern5_2/linear,pylibkriging,q2,0.999792 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern5_2/linear,pylibkriging,nlpd,-2.58843 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern5_2/interactive,pylibkriging,fit_s,0.000845909 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern5_2/interactive,pylibkriging,rmse,0.0230248 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern5_2/interactive,pylibkriging,q2,0.999796 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern5_2/interactive,pylibkriging,nlpd,-2.58839 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern5_2/quadratic,pylibkriging,fit_s,0.00176644 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern5_2/quadratic,pylibkriging,rmse,0.209802 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern5_2/quadratic,pylibkriging,q2,0.983025 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern5_2/quadratic,pylibkriging,nlpd,-0.403998 +kernels-trends-interpolation_Kriging_python,python,ishigami/gauss/constant,pylibkriging,fit_s,0.0820026 +kernels-trends-interpolation_Kriging_python,python,ishigami/gauss/constant,pylibkriging,rmse,1.41975 +kernels-trends-interpolation_Kriging_python,python,ishigami/gauss/constant,pylibkriging,q2,0.844642 +kernels-trends-interpolation_Kriging_python,python,ishigami/gauss/constant,pylibkriging,nlpd,2.25894 +kernels-trends-interpolation_Kriging_python,python,ishigami/gauss/linear,pylibkriging,fit_s,0.0247335 +kernels-trends-interpolation_Kriging_python,python,ishigami/gauss/linear,pylibkriging,rmse,2.84579 +kernels-trends-interpolation_Kriging_python,python,ishigami/gauss/linear,pylibkriging,q2,0.375811 +kernels-trends-interpolation_Kriging_python,python,ishigami/gauss/linear,pylibkriging,nlpd,3.41805 +kernels-trends-interpolation_Kriging_python,python,ishigami/gauss/interactive,pylibkriging,fit_s,0.0169995 +kernels-trends-interpolation_Kriging_python,python,ishigami/gauss/interactive,pylibkriging,rmse,2.85366 +kernels-trends-interpolation_Kriging_python,python,ishigami/gauss/interactive,pylibkriging,q2,0.372353 +kernels-trends-interpolation_Kriging_python,python,ishigami/gauss/interactive,pylibkriging,nlpd,3.28643 +kernels-trends-interpolation_Kriging_python,python,ishigami/gauss/quadratic,pylibkriging,fit_s,0.00751019 +kernels-trends-interpolation_Kriging_python,python,ishigami/gauss/quadratic,pylibkriging,rmse,2.08982 +kernels-trends-interpolation_Kriging_python,python,ishigami/gauss/quadratic,pylibkriging,q2,0.66339 +kernels-trends-interpolation_Kriging_python,python,ishigami/gauss/quadratic,pylibkriging,nlpd,2.80099 +kernels-trends-interpolation_Kriging_python,python,ishigami/exp/constant,pylibkriging,fit_s,0.00825953 +kernels-trends-interpolation_Kriging_python,python,ishigami/exp/constant,pylibkriging,rmse,1.41632 +kernels-trends-interpolation_Kriging_python,python,ishigami/exp/constant,pylibkriging,q2,0.845391 +kernels-trends-interpolation_Kriging_python,python,ishigami/exp/constant,pylibkriging,nlpd,1.75239 +kernels-trends-interpolation_Kriging_python,python,ishigami/exp/linear,pylibkriging,fit_s,0.00724149 +kernels-trends-interpolation_Kriging_python,python,ishigami/exp/linear,pylibkriging,rmse,1.4095 +kernels-trends-interpolation_Kriging_python,python,ishigami/exp/linear,pylibkriging,q2,0.846877 +kernels-trends-interpolation_Kriging_python,python,ishigami/exp/linear,pylibkriging,nlpd,1.74779 +kernels-trends-interpolation_Kriging_python,python,ishigami/exp/interactive,pylibkriging,fit_s,0.00750446 +kernels-trends-interpolation_Kriging_python,python,ishigami/exp/interactive,pylibkriging,rmse,1.41511 +kernels-trends-interpolation_Kriging_python,python,ishigami/exp/interactive,pylibkriging,q2,0.845657 +kernels-trends-interpolation_Kriging_python,python,ishigami/exp/interactive,pylibkriging,nlpd,1.75024 +kernels-trends-interpolation_Kriging_python,python,ishigami/exp/quadratic,pylibkriging,fit_s,0.00750661 +kernels-trends-interpolation_Kriging_python,python,ishigami/exp/quadratic,pylibkriging,rmse,1.40329 +kernels-trends-interpolation_Kriging_python,python,ishigami/exp/quadratic,pylibkriging,q2,0.848223 +kernels-trends-interpolation_Kriging_python,python,ishigami/exp/quadratic,pylibkriging,nlpd,1.73254 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern3_2/constant,pylibkriging,fit_s,0.0483243 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern3_2/constant,pylibkriging,rmse,1.32974 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern3_2/constant,pylibkriging,q2,0.863717 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern3_2/constant,pylibkriging,nlpd,1.50266 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern3_2/linear,pylibkriging,fit_s,0.0462224 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern3_2/linear,pylibkriging,rmse,1.30244 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern3_2/linear,pylibkriging,q2,0.869255 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern3_2/linear,pylibkriging,nlpd,1.47906 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern3_2/interactive,pylibkriging,fit_s,0.0462408 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern3_2/interactive,pylibkriging,rmse,1.30591 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern3_2/interactive,pylibkriging,q2,0.868558 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern3_2/interactive,pylibkriging,nlpd,1.47384 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern3_2/quadratic,pylibkriging,fit_s,0.0412879 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern3_2/quadratic,pylibkriging,rmse,1.32197 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern3_2/quadratic,pylibkriging,q2,0.865305 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern3_2/quadratic,pylibkriging,nlpd,1.46827 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern5_2/constant,pylibkriging,fit_s,0.02212 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern5_2/constant,pylibkriging,rmse,3.00566 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern5_2/constant,pylibkriging,q2,0.303709 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern5_2/constant,pylibkriging,nlpd,3.42085 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern5_2/constant,pylibkriging BFGS10,rmse,1.33903 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern5_2/constant,pylibkriging BFGS10,q2,0.861806 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern5_2/constant,pylibkriging BFGS10,nlpd,1.66132 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern5_2/linear,pylibkriging,fit_s,0.0472796 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern5_2/linear,pylibkriging,rmse,1.3434 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern5_2/linear,pylibkriging,q2,0.860901 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern5_2/linear,pylibkriging,nlpd,1.65103 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern5_2/interactive,pylibkriging,fit_s,0.0477624 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern5_2/interactive,pylibkriging,rmse,1.34479 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern5_2/interactive,pylibkriging,q2,0.860615 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern5_2/interactive,pylibkriging,nlpd,1.64153 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern5_2/quadratic,pylibkriging,fit_s,0.0483804 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern5_2/quadratic,pylibkriging,rmse,1.37082 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern5_2/quadratic,pylibkriging,q2,0.855166 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern5_2/quadratic,pylibkriging,nlpd,1.62228 +kernels-trends-interpolation_Kriging_python,python,hartmann3/gauss/constant,pylibkriging,fit_s,0.00703788 +kernels-trends-interpolation_Kriging_python,python,hartmann3/gauss/constant,pylibkriging,rmse,0.156226 +kernels-trends-interpolation_Kriging_python,python,hartmann3/gauss/constant,pylibkriging,q2,0.971116 +kernels-trends-interpolation_Kriging_python,python,hartmann3/gauss/constant,pylibkriging,nlpd,-0.555623 +kernels-trends-interpolation_Kriging_python,python,hartmann3/gauss/linear,pylibkriging,fit_s,0.0116901 +kernels-trends-interpolation_Kriging_python,python,hartmann3/gauss/linear,pylibkriging,rmse,0.153297 +kernels-trends-interpolation_Kriging_python,python,hartmann3/gauss/linear,pylibkriging,q2,0.972189 +kernels-trends-interpolation_Kriging_python,python,hartmann3/gauss/linear,pylibkriging,nlpd,-0.568404 +kernels-trends-interpolation_Kriging_python,python,hartmann3/gauss/interactive,pylibkriging,fit_s,0.00962472 +kernels-trends-interpolation_Kriging_python,python,hartmann3/gauss/interactive,pylibkriging,rmse,0.712947 +kernels-trends-interpolation_Kriging_python,python,hartmann3/gauss/interactive,pylibkriging,q2,0.398455 +kernels-trends-interpolation_Kriging_python,python,hartmann3/gauss/interactive,pylibkriging,nlpd,1.74554 +kernels-trends-interpolation_Kriging_python,python,hartmann3/gauss/quadratic,pylibkriging,fit_s,0.00845146 +kernels-trends-interpolation_Kriging_python,python,hartmann3/gauss/quadratic,pylibkriging,rmse,0.73238 +kernels-trends-interpolation_Kriging_python,python,hartmann3/gauss/quadratic,pylibkriging,q2,0.365216 +kernels-trends-interpolation_Kriging_python,python,hartmann3/gauss/quadratic,pylibkriging,nlpd,2.14588 +kernels-trends-interpolation_Kriging_python,python,hartmann3/exp/constant,pylibkriging,fit_s,0.00484872 +kernels-trends-interpolation_Kriging_python,python,hartmann3/exp/constant,pylibkriging,rmse,0.347348 +kernels-trends-interpolation_Kriging_python,python,hartmann3/exp/constant,pylibkriging,q2,0.857215 +kernels-trends-interpolation_Kriging_python,python,hartmann3/exp/constant,pylibkriging,nlpd,0.352683 +kernels-trends-interpolation_Kriging_python,python,hartmann3/exp/linear,pylibkriging,fit_s,0.00318956 +kernels-trends-interpolation_Kriging_python,python,hartmann3/exp/linear,pylibkriging,rmse,0.347273 +kernels-trends-interpolation_Kriging_python,python,hartmann3/exp/linear,pylibkriging,q2,0.857276 +kernels-trends-interpolation_Kriging_python,python,hartmann3/exp/linear,pylibkriging,nlpd,0.343631 +kernels-trends-interpolation_Kriging_python,python,hartmann3/exp/interactive,pylibkriging,fit_s,0.00330567 +kernels-trends-interpolation_Kriging_python,python,hartmann3/exp/interactive,pylibkriging,rmse,0.378349 +kernels-trends-interpolation_Kriging_python,python,hartmann3/exp/interactive,pylibkriging,q2,0.83059 +kernels-trends-interpolation_Kriging_python,python,hartmann3/exp/interactive,pylibkriging,nlpd,0.385842 +kernels-trends-interpolation_Kriging_python,python,hartmann3/exp/quadratic,pylibkriging,fit_s,0.00355411 +kernels-trends-interpolation_Kriging_python,python,hartmann3/exp/quadratic,pylibkriging,rmse,0.355119 +kernels-trends-interpolation_Kriging_python,python,hartmann3/exp/quadratic,pylibkriging,q2,0.850755 +kernels-trends-interpolation_Kriging_python,python,hartmann3/exp/quadratic,pylibkriging,nlpd,0.334371 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern3_2/constant,pylibkriging,fit_s,0.0107734 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern3_2/constant,pylibkriging,rmse,0.227521 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern3_2/constant,pylibkriging,q2,0.938737 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern3_2/constant,pylibkriging,nlpd,-0.181207 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern3_2/linear,pylibkriging,fit_s,0.0118749 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern3_2/linear,pylibkriging,rmse,0.23038 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern3_2/linear,pylibkriging,q2,0.937188 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern3_2/linear,pylibkriging,nlpd,-0.162203 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern3_2/interactive,pylibkriging,fit_s,0.00637007 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern3_2/interactive,pylibkriging,rmse,0.315752 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern3_2/interactive,pylibkriging,q2,0.88201 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern3_2/interactive,pylibkriging,nlpd,0.898253 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern3_2/quadratic,pylibkriging,fit_s,0.00598764 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern3_2/quadratic,pylibkriging,rmse,0.291393 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern3_2/quadratic,pylibkriging,q2,0.899513 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern3_2/quadratic,pylibkriging,nlpd,0.53743 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern5_2/constant,pylibkriging,fit_s,0.0100911 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern5_2/constant,pylibkriging,rmse,0.190287 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern5_2/constant,pylibkriging,q2,0.957148 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern5_2/constant,pylibkriging,nlpd,-0.403054 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern5_2/constant,pylibkriging BFGS10,rmse,0.190286 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern5_2/constant,pylibkriging BFGS10,q2,0.957148 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern5_2/constant,pylibkriging BFGS10,nlpd,-0.40306 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern5_2/linear,pylibkriging,fit_s,0.0106812 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern5_2/linear,pylibkriging,rmse,0.189633 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern5_2/linear,pylibkriging,q2,0.957442 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern5_2/linear,pylibkriging,nlpd,-0.410075 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern5_2/interactive,pylibkriging,fit_s,0.00839925 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern5_2/interactive,pylibkriging,rmse,0.307114 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern5_2/interactive,pylibkriging,q2,0.888377 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern5_2/interactive,pylibkriging,nlpd,1.50063 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern5_2/quadratic,pylibkriging,fit_s,0.00724792 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern5_2/quadratic,pylibkriging,rmse,0.285355 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern5_2/quadratic,pylibkriging,q2,0.903634 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern5_2/quadratic,pylibkriging,nlpd,0.898128 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,branin/LL,pylibkriging,rmse,71.4801 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,branin/LL,pylibkriging,q2,-1991.62 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,branin/LL,pylibkriging,nlpd,13603.3 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,branin/LOO,pylibkriging,rmse,71.5631 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,branin/LOO,pylibkriging,q2,-1996.25 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,branin/LOO,pylibkriging,nlpd,18836.2 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,branin/LMP,pylibkriging,rmse,71.5295 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,branin/LMP,pylibkriging,q2,-1994.38 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,branin/LMP,pylibkriging,nlpd,14603.3 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,hartmann3/LL,pylibkriging,rmse,1.30952 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,hartmann3/LL,pylibkriging,q2,-2013.29 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,hartmann3/LL,pylibkriging,nlpd,41.7128 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,hartmann3/LOO,pylibkriging,rmse,1.31436 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,hartmann3/LOO,pylibkriging,q2,-2028.22 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,hartmann3/LOO,pylibkriging,nlpd,49.7135 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,hartmann3/LMP,pylibkriging,rmse,1.31148 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,hartmann3/LMP,pylibkriging,q2,-2019.34 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,hartmann3/LMP,pylibkriging,nlpd,43.397 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,ishigami/LL,pylibkriging,rmse,5.11103 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,ishigami/LL,pylibkriging,q2,-1783.42 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,ishigami/LL,pylibkriging,nlpd,10.4546 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,ishigami/LOO,pylibkriging,rmse,5.81789 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,ishigami/LOO,pylibkriging,q2,-2311.12 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,ishigami/LOO,pylibkriging,nlpd,40.8323 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,ishigami/LMP,pylibkriging,rmse,5.18495 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,ishigami/LMP,pylibkriging,q2,-1835.41 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,ishigami/LMP,pylibkriging,nlpd,11.5894 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,friedman/LL,pylibkriging,rmse,6.87533 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,friedman/LL,pylibkriging,q2,-1991.33 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,friedman/LL,pylibkriging,nlpd,651.706 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,friedman/LOO,pylibkriging,rmse,6.87323 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,friedman/LOO,pylibkriging,q2,-1990.12 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,friedman/LOO,pylibkriging,nlpd,484.115 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,friedman/LMP,pylibkriging,rmse,6.87647 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,friedman/LMP,pylibkriging,q2,-1991.99 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,friedman/LMP,pylibkriging,nlpd,658.808 +noisy-data_Kriging-nugget-noise_python,python,forrester,nugget,rmse,0.198091 +noisy-data_Kriging-nugget-noise_python,python,forrester,nugget,q2,0.998087 +noisy-data_Kriging-nugget-noise_python,python,forrester,nugget,nlpd,0.286631 +noisy-data_Kriging-nugget-noise_python,python,forrester,noise-free fit,rmse,0.487916 +noisy-data_Kriging-nugget-noise_python,python,forrester,noise-free fit,q2,0.988392 +noisy-data_Kriging-nugget-noise_python,python,forrester,noise-free fit,nlpd,862.687 +noisy-data_Kriging-nugget-noise_python,python,friedman,nugget,rmse,0.467335 +noisy-data_Kriging-nugget-noise_python,python,friedman,nugget,q2,0.99089 +noisy-data_Kriging-nugget-noise_python,python,friedman,nugget,nlpd,1.07073 +noisy-data_Kriging-nugget-noise_python,python,friedman,noise-free fit,rmse,1.27156 +noisy-data_Kriging-nugget-noise_python,python,friedman,noise-free fit,q2,0.932559 +noisy-data_Kriging-nugget-noise_python,python,friedman,noise-free fit,nlpd,1.64495 +noisy-data_Kriging-nugget-noise_python,python,gramacy_lee_2012/hetero,known variances,rmse,0.168741 +noisy-data_Kriging-nugget-noise_python,python,gramacy_lee_2012/hetero,known variances,q2,0.982515 +noisy-data_Kriging-nugget-noise_python,python,gramacy_lee_2012/hetero,known variances,nlpd,-0.570574 +noisy-data_Kriging-nugget-noise_python,python,gramacy_lee_2012/hetero,nugget,rmse,0.170945 +noisy-data_Kriging-nugget-noise_python,python,gramacy_lee_2012/hetero,nugget,q2,0.982055 +noisy-data_Kriging-nugget-noise_python,python,gramacy_lee_2012/hetero,nugget,nlpd,-0.221568 +warpings-categorical-ordinal_WarpKriging_python,python,gl2012,Kriging,rmse,0.167263 +warpings-categorical-ordinal_WarpKriging_python,python,gl2012,Kriging,q2,0.981176 +warpings-categorical-ordinal_WarpKriging_python,python,gl2012,Kriging,nlpd,-1.08917 +warpings-categorical-ordinal_WarpKriging_python,python,gl2012,affine,rmse,0.167263 +warpings-categorical-ordinal_WarpKriging_python,python,gl2012,affine,q2,0.981176 +warpings-categorical-ordinal_WarpKriging_python,python,gl2012,affine,nlpd,-1.08917 +warpings-categorical-ordinal_WarpKriging_python,python,gl2012,boxcox,rmse,0.16759 +warpings-categorical-ordinal_WarpKriging_python,python,gl2012,boxcox,q2,0.981103 +warpings-categorical-ordinal_WarpKriging_python,python,gl2012,boxcox,nlpd,282.422 +warpings-categorical-ordinal_WarpKriging_python,python,gl2012,kumaraswamy,rmse,0.168286 +warpings-categorical-ordinal_WarpKriging_python,python,gl2012,kumaraswamy,q2,0.980945 +warpings-categorical-ordinal_WarpKriging_python,python,gl2012,kumaraswamy,nlpd,2.04577e+06 +warpings-categorical-ordinal_WarpKriging_python,python,gl2012,knots(5),rmse,0.16791 +warpings-categorical-ordinal_WarpKriging_python,python,gl2012,knots(5),q2,0.98103 +warpings-categorical-ordinal_WarpKriging_python,python,gl2012,knots(5),nlpd,2.04376e+06 +warpings-categorical-ordinal_WarpKriging_python,python,gl2012,neural_mono(8),rmse,0.170824 +warpings-categorical-ordinal_WarpKriging_python,python,gl2012,neural_mono(8),q2,0.980366 +warpings-categorical-ordinal_WarpKriging_python,python,gl2012,neural_mono(8),nlpd,-0.265141 +warpings-categorical-ordinal_WarpKriging_python,python,gl2008,Kriging,rmse,0.0534649 +warpings-categorical-ordinal_WarpKriging_python,python,gl2008,Kriging,q2,0.585636 +warpings-categorical-ordinal_WarpKriging_python,python,gl2008,Kriging,nlpd,-1.74569 +warpings-categorical-ordinal_WarpKriging_python,python,gl2008,kumaraswamy,rmse,0.0535298 +warpings-categorical-ordinal_WarpKriging_python,python,gl2008,kumaraswamy,q2,0.58463 +warpings-categorical-ordinal_WarpKriging_python,python,gl2008,kumaraswamy,nlpd,-1.74552 +warpings-categorical-ordinal_WarpKriging_python,python,mixed,Kriging (level as number),rmse,16.7693 +warpings-categorical-ordinal_WarpKriging_python,python,mixed,Kriging (level as number),q2,0.838453 +warpings-categorical-ordinal_WarpKriging_python,python,mixed,Kriging (level as number),nlpd,3.81063 +warpings-categorical-ordinal_WarpKriging_python,python,mixed,"categorical(5,2)",rmse,16.1438 +warpings-categorical-ordinal_WarpKriging_python,python,mixed,"categorical(5,2)",q2,0.85028 +warpings-categorical-ordinal_WarpKriging_python,python,mixed,"categorical(5,2)",nlpd,3.3206 +warpings-categorical-ordinal_WarpKriging_python,python,ordinal,Kriging (rank as number),rmse,0.213845 +warpings-categorical-ordinal_WarpKriging_python,python,ordinal,Kriging (rank as number),q2,0.953177 +warpings-categorical-ordinal_WarpKriging_python,python,ordinal,Kriging (rank as number),nlpd,0.810969 +warpings-categorical-ordinal_WarpKriging_python,python,ordinal,ordinal(6),rmse,0.213043 +warpings-categorical-ordinal_WarpKriging_python,python,ordinal,ordinal(6),q2,0.953528 +warpings-categorical-ordinal_WarpKriging_python,python,ordinal,ordinal(6),nlpd,0.78888 +high-dimension-normalize-screening_Kriging_python,python,otl_circuit,unit cube,fit_s,0.00796986 +high-dimension-normalize-screening_Kriging_python,python,otl_circuit,unit cube,rmse,0.0150514 +high-dimension-normalize-screening_Kriging_python,python,otl_circuit,unit cube,q2,0.999836 +high-dimension-normalize-screening_Kriging_python,python,otl_circuit,unit cube,nlpd,-2.90452 +high-dimension-normalize-screening_Kriging_python,python,otl_circuit,native + normalize,rmse,0.0150514 +high-dimension-normalize-screening_Kriging_python,python,otl_circuit,native + normalize,q2,0.999836 +high-dimension-normalize-screening_Kriging_python,python,otl_circuit,native + normalize,nlpd,-2.90452 +high-dimension-normalize-screening_Kriging_python,python,piston,unit cube,fit_s,0.023371 +high-dimension-normalize-screening_Kriging_python,python,piston,unit cube,rmse,0.0104258 +high-dimension-normalize-screening_Kriging_python,python,piston,unit cube,q2,0.994462 +high-dimension-normalize-screening_Kriging_python,python,piston,unit cube,nlpd,-3.49038 +high-dimension-normalize-screening_Kriging_python,python,piston,native + normalize,rmse,0.0104258 +high-dimension-normalize-screening_Kriging_python,python,piston,native + normalize,q2,0.994462 +high-dimension-normalize-screening_Kriging_python,python,piston,native + normalize,nlpd,-3.49038 +high-dimension-normalize-screening_Kriging_python,python,borehole,unit cube,fit_s,0.0214174 +high-dimension-normalize-screening_Kriging_python,python,borehole,unit cube,rmse,0.893341 +high-dimension-normalize-screening_Kriging_python,python,borehole,unit cube,q2,0.999599 +high-dimension-normalize-screening_Kriging_python,python,borehole,unit cube,nlpd,1.08433 +high-dimension-normalize-screening_Kriging_python,python,borehole,native + normalize,rmse,0.893341 +high-dimension-normalize-screening_Kriging_python,python,borehole,native + normalize,q2,0.999599 +high-dimension-normalize-screening_Kriging_python,python,borehole,native + normalize,nlpd,1.08433 +high-dimension-normalize-screening_Kriging_python,python,robot_arm,unit cube,fit_s,0.187081 +high-dimension-normalize-screening_Kriging_python,python,robot_arm,unit cube,rmse,0.358507 +high-dimension-normalize-screening_Kriging_python,python,robot_arm,unit cube,q2,0.571154 +high-dimension-normalize-screening_Kriging_python,python,robot_arm,unit cube,nlpd,0.355106 +high-dimension-normalize-screening_Kriging_python,python,robot_arm,native + normalize,rmse,0.358507 +high-dimension-normalize-screening_Kriging_python,python,robot_arm,native + normalize,q2,0.571154 +high-dimension-normalize-screening_Kriging_python,python,robot_arm,native + normalize,nlpd,0.355106 +high-dimension-normalize-screening_Kriging_python,python,dette_pepelyshev,unit cube,fit_s,0.0181198 +high-dimension-normalize-screening_Kriging_python,python,dette_pepelyshev,unit cube,rmse,0.424908 +high-dimension-normalize-screening_Kriging_python,python,dette_pepelyshev,unit cube,q2,0.997757 +high-dimension-normalize-screening_Kriging_python,python,dette_pepelyshev,unit cube,nlpd,0.510276 +high-dimension-normalize-screening_Kriging_python,python,dette_pepelyshev,native + normalize,rmse,0.424908 +high-dimension-normalize-screening_Kriging_python,python,dette_pepelyshev,native + normalize,q2,0.997757 +high-dimension-normalize-screening_Kriging_python,python,dette_pepelyshev,native + normalize,nlpd,0.510276 +high-dimension-normalize-screening_Kriging_python,python,wing_weight,unit cube,fit_s,0.0303686 +high-dimension-normalize-screening_Kriging_python,python,wing_weight,unit cube,rmse,0.809743 +high-dimension-normalize-screening_Kriging_python,python,wing_weight,unit cube,q2,0.999716 +high-dimension-normalize-screening_Kriging_python,python,wing_weight,unit cube,nlpd,1.07161 +high-dimension-normalize-screening_Kriging_python,python,wing_weight,native + normalize,rmse,0.809743 +high-dimension-normalize-screening_Kriging_python,python,wing_weight,native + normalize,q2,0.999716 +high-dimension-normalize-screening_Kriging_python,python,wing_weight,native + normalize,nlpd,1.07161 +high-dimension-normalize-screening_Kriging_python,python,welch,unit cube,fit_s,0.500499 +high-dimension-normalize-screening_Kriging_python,python,welch,unit cube,rmse,0.167729 +high-dimension-normalize-screening_Kriging_python,python,welch,unit cube,q2,0.993452 +high-dimension-normalize-screening_Kriging_python,python,welch,unit cube,nlpd,-0.429138 +high-dimension-normalize-screening_Kriging_python,python,welch,native + normalize,rmse,0.167692 +high-dimension-normalize-screening_Kriging_python,python,welch,native + normalize,q2,0.993454 +high-dimension-normalize-screening_Kriging_python,python,welch,native + normalize,nlpd,-0.429291 +deep-kernel-projected-branin_MLPKriging_python,python,projected_branin_10d,Kriging,rmse,4.42284 +deep-kernel-projected-branin_MLPKriging_python,python,projected_branin_10d,Kriging,q2,0.959122 +deep-kernel-projected-branin_MLPKriging_python,python,projected_branin_10d,Kriging,nlpd,2.89997 +deep-kernel-projected-branin_MLPKriging_python,python,projected_branin_10d,MLPKriging tanh BFGS,fit_s,0.171914 +deep-kernel-projected-branin_MLPKriging_python,python,projected_branin_10d,MLPKriging tanh BFGS,rmse,59.5068 +deep-kernel-projected-branin_MLPKriging_python,python,projected_branin_10d,MLPKriging tanh BFGS,q2,-6.39973 +deep-kernel-projected-branin_MLPKriging_python,python,projected_branin_10d,MLPKriging tanh BFGS,nlpd,100.745 +deep-kernel-projected-branin_MLPKriging_python,python,projected_branin_10d,MLPKriging tanh BFGS10,fit_s,0.53444 +deep-kernel-projected-branin_MLPKriging_python,python,projected_branin_10d,MLPKriging tanh BFGS10,rmse,4.41161 +deep-kernel-projected-branin_MLPKriging_python,python,projected_branin_10d,MLPKriging tanh BFGS10,q2,0.95933 +deep-kernel-projected-branin_MLPKriging_python,python,projected_branin_10d,MLPKriging tanh BFGS10,nlpd,28.4236 +deep-kernel-projected-branin_MLPKriging_python,python,projected_branin_10d,MLPKriging selu BFGS,fit_s,0.145419 +deep-kernel-projected-branin_MLPKriging_python,python,projected_branin_10d,MLPKriging selu BFGS,rmse,58.9844 +deep-kernel-projected-branin_MLPKriging_python,python,projected_branin_10d,MLPKriging selu BFGS,q2,-6.27038 +deep-kernel-projected-branin_MLPKriging_python,python,projected_branin_10d,MLPKriging selu BFGS,nlpd,46.9347 +deep-kernel-projected-branin_MLPKriging_python,python,projected_branin_10d,MLPKriging selu BFGS10,fit_s,0.518803 +deep-kernel-projected-branin_MLPKriging_python,python,projected_branin_10d,MLPKriging selu BFGS10,rmse,3.79093 +deep-kernel-projected-branin_MLPKriging_python,python,projected_branin_10d,MLPKriging selu BFGS10,q2,0.969969 +deep-kernel-projected-branin_MLPKriging_python,python,projected_branin_10d,MLPKriging selu BFGS10,nlpd,12.6814 +large-n_Kriging-vecchia-nystrom_python,python,hartmann3/n=1500,exact,fit_s,11.2644 +aggregations-large-n_NestedKriging_python,python,hartmann3/n=1500,exact,fit_s,11.2644 +large-n_Kriging-vecchia-nystrom_python,python,hartmann3/n=1500,exact,rmse,0.000259233 +aggregations-large-n_NestedKriging_python,python,hartmann3/n=1500,exact,rmse,0.000259233 +large-n_Kriging-vecchia-nystrom_python,python,hartmann3/n=1500,exact,q2,1 +aggregations-large-n_NestedKriging_python,python,hartmann3/n=1500,exact,q2,1 +large-n_Kriging-vecchia-nystrom_python,python,hartmann3/n=1500,exact,nlpd,-6.92184 +aggregations-large-n_NestedKriging_python,python,hartmann3/n=1500,exact,nlpd,-6.92184 +aggregations-large-n_NestedKriging_python,python,hartmann3/n=1500,nested NK (20 groups),fit_s,0.324342 +aggregations-large-n_NestedKriging_python,python,hartmann3/n=1500,nested NK (20 groups),rmse,0.0029908 +aggregations-large-n_NestedKriging_python,python,hartmann3/n=1500,nested NK (20 groups),q2,0.999991 +aggregations-large-n_NestedKriging_python,python,hartmann3/n=1500,nested NK (20 groups),nlpd,-4.85946 +aggregations-large-n_NestedKriging_python,python,hartmann3/n=5000,nested NK (20 groups),fit_s,3.25297 +aggregations-large-n_NestedKriging_python,python,hartmann3/n=5000,nested NK (20 groups),rmse,0.000222655 +aggregations-large-n_NestedKriging_python,python,hartmann3/n=5000,nested NK (20 groups),q2,1 +aggregations-large-n_NestedKriging_python,python,hartmann3/n=5000,nested NK (20 groups),nlpd,-7.44453 +aggregations-large-n_NestedKriging_python,python,hartmann3/n=1500,nested rBCM (20 groups),fit_s,0.273419 +aggregations-large-n_NestedKriging_python,python,hartmann3/n=1500,nested rBCM (20 groups),rmse,0.00304431 +aggregations-large-n_NestedKriging_python,python,hartmann3/n=1500,nested rBCM (20 groups),q2,0.99999 +aggregations-large-n_NestedKriging_python,python,hartmann3/n=1500,nested rBCM (20 groups),nlpd,-4.50201 +aggregations-large-n_NestedKriging_python,python,hartmann3/n=5000,nested rBCM (20 groups),fit_s,2.67811 +aggregations-large-n_NestedKriging_python,python,hartmann3/n=5000,nested rBCM (20 groups),rmse,0.000224517 +aggregations-large-n_NestedKriging_python,python,hartmann3/n=5000,nested rBCM (20 groups),q2,1 +aggregations-large-n_NestedKriging_python,python,hartmann3/n=5000,nested rBCM (20 groups),nlpd,-7.18182 +large-n_Kriging-vecchia-nystrom_python,python,hartmann3/n=1500,subset of data (500),fit_s,0.685586 +large-n_Kriging-vecchia-nystrom_python,python,hartmann3/n=1500,subset of data (500),rmse,0.00269153 +large-n_Kriging-vecchia-nystrom_python,python,hartmann3/n=1500,subset of data (500),q2,0.999992 +large-n_Kriging-vecchia-nystrom_python,python,hartmann3/n=1500,subset of data (500),nlpd,-4.68641 +large-n_Kriging-vecchia-nystrom_python,python,hartmann3/n=5000,subset of data (500),fit_s,0.906366 +large-n_Kriging-vecchia-nystrom_python,python,hartmann3/n=5000,subset of data (500),rmse,0.00247552 +large-n_Kriging-vecchia-nystrom_python,python,hartmann3/n=5000,subset of data (500),q2,0.999994 +large-n_Kriging-vecchia-nystrom_python,python,hartmann3/n=5000,subset of data (500),nlpd,-4.71359 +large-n_Kriging-vecchia-nystrom_python,python,otl_circuit/n=1500,exact,fit_s,10.9405 +aggregations-large-n_NestedKriging_python,python,otl_circuit/n=1500,exact,fit_s,10.9405 +large-n_Kriging-vecchia-nystrom_python,python,otl_circuit/n=1500,exact,rmse,9.81598e-05 +aggregations-large-n_NestedKriging_python,python,otl_circuit/n=1500,exact,rmse,9.81598e-05 +large-n_Kriging-vecchia-nystrom_python,python,otl_circuit/n=1500,exact,q2,1 +aggregations-large-n_NestedKriging_python,python,otl_circuit/n=1500,exact,q2,1 +large-n_Kriging-vecchia-nystrom_python,python,otl_circuit/n=1500,exact,nlpd,-8.21962 +aggregations-large-n_NestedKriging_python,python,otl_circuit/n=1500,exact,nlpd,-8.21962 +aggregations-large-n_NestedKriging_python,python,otl_circuit/n=1500,nested NK (20 groups),fit_s,0.279359 +aggregations-large-n_NestedKriging_python,python,otl_circuit/n=1500,nested NK (20 groups),rmse,0.00226873 +aggregations-large-n_NestedKriging_python,python,otl_circuit/n=1500,nested NK (20 groups),q2,0.999996 +aggregations-large-n_NestedKriging_python,python,otl_circuit/n=1500,nested NK (20 groups),nlpd,-4.90542 +aggregations-large-n_NestedKriging_python,python,otl_circuit/n=5000,nested NK (20 groups),fit_s,3.08712 +aggregations-large-n_NestedKriging_python,python,otl_circuit/n=5000,nested NK (20 groups),rmse,0.000261686 +aggregations-large-n_NestedKriging_python,python,otl_circuit/n=5000,nested NK (20 groups),q2,1 +aggregations-large-n_NestedKriging_python,python,otl_circuit/n=5000,nested NK (20 groups),nlpd,-7.20623 +aggregations-large-n_NestedKriging_python,python,otl_circuit/n=1500,nested rBCM (20 groups),fit_s,0.251989 +aggregations-large-n_NestedKriging_python,python,otl_circuit/n=1500,nested rBCM (20 groups),rmse,0.00291869 +aggregations-large-n_NestedKriging_python,python,otl_circuit/n=1500,nested rBCM (20 groups),q2,0.999993 +aggregations-large-n_NestedKriging_python,python,otl_circuit/n=1500,nested rBCM (20 groups),nlpd,-2.55512 +aggregations-large-n_NestedKriging_python,python,otl_circuit/n=5000,nested rBCM (20 groups),fit_s,3.67768 +aggregations-large-n_NestedKriging_python,python,otl_circuit/n=5000,nested rBCM (20 groups),rmse,0.000278149 +aggregations-large-n_NestedKriging_python,python,otl_circuit/n=5000,nested rBCM (20 groups),q2,1 +aggregations-large-n_NestedKriging_python,python,otl_circuit/n=5000,nested rBCM (20 groups),nlpd,-4.34927 +large-n_Kriging-vecchia-nystrom_python,python,otl_circuit/n=1500,subset of data (500),fit_s,0.709236 +large-n_Kriging-vecchia-nystrom_python,python,otl_circuit/n=1500,subset of data (500),rmse,0.0004286 +large-n_Kriging-vecchia-nystrom_python,python,otl_circuit/n=1500,subset of data (500),q2,1 +large-n_Kriging-vecchia-nystrom_python,python,otl_circuit/n=1500,subset of data (500),nlpd,-6.72678 +large-n_Kriging-vecchia-nystrom_python,python,otl_circuit/n=5000,subset of data (500),fit_s,0.905534 +large-n_Kriging-vecchia-nystrom_python,python,otl_circuit/n=5000,subset of data (500),rmse,0.000446879 +large-n_Kriging-vecchia-nystrom_python,python,otl_circuit/n=5000,subset of data (500),q2,1 +large-n_Kriging-vecchia-nystrom_python,python,otl_circuit/n=5000,subset of data (500),nlpd,-6.78844 +aggregations-large-n_NestedKriging_python,python,branin/nested,NK,rmse,0.0343984 +aggregations-large-n_NestedKriging_python,python,branin/nested,NK,q2,1 +aggregations-large-n_NestedKriging_python,python,branin/nested,NK,nlpd,2.00083e+14 +aggregations-large-n_NestedKriging_python,python,branin/nested,PoE,rmse,0.0354265 +aggregations-large-n_NestedKriging_python,python,branin/nested,PoE,q2,1 +aggregations-large-n_NestedKriging_python,python,branin/nested,PoE,nlpd,-1.64533 +aggregations-large-n_NestedKriging_python,python,branin/nested,gPoE,rmse,0.0354265 +aggregations-large-n_NestedKriging_python,python,branin/nested,gPoE,q2,1 +aggregations-large-n_NestedKriging_python,python,branin/nested,gPoE,nlpd,-0.621836 +aggregations-large-n_NestedKriging_python,python,branin/nested,BCM,rmse,0.0354196 +aggregations-large-n_NestedKriging_python,python,branin/nested,BCM,q2,1 +aggregations-large-n_NestedKriging_python,python,branin/nested,BCM,nlpd,-1.64533 +aggregations-large-n_NestedKriging_python,python,branin/nested,rBCM,rmse,0.035287 +aggregations-large-n_NestedKriging_python,python,branin/nested,rBCM,q2,1 +aggregations-large-n_NestedKriging_python,python,branin/nested,rBCM,nlpd,-2.61369 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,branin,pylibkriging,q2,0.999944 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,branin,pylibkriging,nlpd,0.137225 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,branin,pylibkriging,fit_s,0.00151396 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,branin,pylibkriging BFGS10,q2,0.999945 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,branin,pylibkriging BFGS10,nlpd,0.137254 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,branin,pylibkriging BFGS10,fit_s,0.0161211 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,branin,scikit-learn,q2,0.999597 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,branin,scikit-learn,nlpd,0.928508 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,branin,scikit-learn,fit_s,0.147773 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,branin,SMT,q2,0.999944 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,branin,SMT,nlpd,0.137259 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,branin,SMT,fit_s,0.792991 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,branin,OpenTURNS,q2,0.998221 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,branin,OpenTURNS,nlpd,1.55599 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,branin,OpenTURNS,fit_s,0.00512815 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,six_hump_camel,pylibkriging,q2,0.999418 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,six_hump_camel,pylibkriging,nlpd,0.0429346 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,six_hump_camel,pylibkriging,fit_s,0.00358009 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,six_hump_camel,pylibkriging BFGS10,q2,0.999418 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,six_hump_camel,pylibkriging BFGS10,nlpd,0.0429347 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,six_hump_camel,pylibkriging BFGS10,fit_s,0.0500369 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,six_hump_camel,scikit-learn,q2,0.0163311 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,six_hump_camel,scikit-learn,nlpd,4.63763 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,six_hump_camel,scikit-learn,fit_s,0.156482 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,six_hump_camel,SMT,q2,0.999594 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,six_hump_camel,SMT,nlpd,-0.121007 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,six_hump_camel,SMT,fit_s,0.941661 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,six_hump_camel,OpenTURNS,q2,0.956713 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,six_hump_camel,OpenTURNS,nlpd,2.55446 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,six_hump_camel,OpenTURNS,fit_s,0.00930476 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann3,pylibkriging,q2,0.986148 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann3,pylibkriging,nlpd,-0.847247 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann3,pylibkriging,fit_s,0.023098 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann3,pylibkriging BFGS10,q2,0.986148 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann3,pylibkriging BFGS10,nlpd,-0.847249 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann3,pylibkriging BFGS10,fit_s,0.113887 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann3,scikit-learn,q2,0.983889 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann3,scikit-learn,nlpd,-0.782697 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann3,scikit-learn,fit_s,0.208279 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann3,SMT,q2,0.986148 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann3,SMT,nlpd,-0.847249 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann3,SMT,fit_s,1.35887 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann3,OpenTURNS,q2,0.984712 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann3,OpenTURNS,nlpd,-0.809851 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann3,OpenTURNS,fit_s,0.0169773 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,ishigami,pylibkriging,q2,0.259556 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,ishigami,pylibkriging,nlpd,4.38991 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,ishigami,pylibkriging,fit_s,0.0264924 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,ishigami,pylibkriging BFGS10,q2,0.935722 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,ishigami,pylibkriging BFGS10,nlpd,0.857443 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,ishigami,pylibkriging BFGS10,fit_s,0.322963 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,ishigami,scikit-learn,q2,0.831142 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,ishigami,scikit-learn,nlpd,1.5455 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,ishigami,scikit-learn,fit_s,0.512728 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,ishigami,SMT,q2,0.51196 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,ishigami,SMT,nlpd,3.31349 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,ishigami,SMT,fit_s,2.70852 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,ishigami,OpenTURNS,q2,0.887251 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,ishigami,OpenTURNS,nlpd,1.39348 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,ishigami,OpenTURNS,fit_s,0.0510182 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,friedman,pylibkriging,q2,0.999771 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,friedman,pylibkriging,nlpd,-1.29726 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,friedman,pylibkriging,fit_s,0.0165708 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,friedman,pylibkriging BFGS10,q2,0.999771 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,friedman,pylibkriging BFGS10,nlpd,-1.29727 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,friedman,pylibkriging BFGS10,fit_s,0.279774 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,friedman,scikit-learn,q2,0.999754 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,friedman,scikit-learn,nlpd,-1.37702 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,friedman,scikit-learn,fit_s,0.308127 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,friedman,SMT,q2,0.999899 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,friedman,SMT,nlpd,-1.871 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,friedman,SMT,fit_s,4.71948 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,friedman,OpenTURNS,q2,0.990862 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,friedman,OpenTURNS,nlpd,0.469828 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,friedman,OpenTURNS,fit_s,0.0511642 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,otl_circuit,pylibkriging,q2,0.999922 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,otl_circuit,pylibkriging,nlpd,-3.46228 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,otl_circuit,pylibkriging,fit_s,0.0136361 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,otl_circuit,pylibkriging BFGS10,q2,0.999922 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,otl_circuit,pylibkriging BFGS10,nlpd,-3.46227 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,otl_circuit,pylibkriging BFGS10,fit_s,0.168652 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,otl_circuit,scikit-learn,q2,0.999906 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,otl_circuit,scikit-learn,nlpd,-3.43062 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,otl_circuit,scikit-learn,fit_s,0.64982 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,otl_circuit,SMT,q2,0.999954 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,otl_circuit,SMT,nlpd,-3.35016 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,otl_circuit,SMT,fit_s,6.837 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,otl_circuit,OpenTURNS,q2,0.99777 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,otl_circuit,OpenTURNS,nlpd,-1.61723 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,otl_circuit,OpenTURNS,fit_s,0.0662544 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,piston,pylibkriging,q2,0.997498 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,piston,pylibkriging,nlpd,-3.80667 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,piston,pylibkriging,fit_s,0.03055 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,piston,pylibkriging BFGS10,q2,0.997498 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,piston,pylibkriging BFGS10,nlpd,-3.8067 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,piston,pylibkriging BFGS10,fit_s,0.326413 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,piston,scikit-learn,q2,0.998684 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,piston,scikit-learn,nlpd,-4.10497 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,piston,scikit-learn,fit_s,0.856336 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,piston,SMT,q2,0.997511 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,piston,SMT,nlpd,-3.86618 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,piston,SMT,fit_s,12.0067 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,piston,OpenTURNS,q2,0.990631 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,piston,OpenTURNS,nlpd,-2.9906 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,piston,OpenTURNS,fit_s,0.137117 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,borehole,pylibkriging,q2,0.999694 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,borehole,pylibkriging,nlpd,1.00118 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,borehole,pylibkriging,fit_s,0.0166521 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,borehole,pylibkriging BFGS10,q2,0.999694 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,borehole,pylibkriging BFGS10,nlpd,1.00121 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,borehole,pylibkriging BFGS10,fit_s,0.200685 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,borehole,scikit-learn,q2,0.999929 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,borehole,scikit-learn,nlpd,0.194455 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,borehole,scikit-learn,fit_s,1.18352 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,borehole,SMT,q2,0.999925 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,borehole,SMT,nlpd,0.279122 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,borehole,SMT,fit_s,12.3314 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,borehole,OpenTURNS,q2,0.996206 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,borehole,OpenTURNS,nlpd,2.55431 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,borehole,OpenTURNS,fit_s,0.121907 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,wing_weight,pylibkriging,q2,0.99972 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,wing_weight,pylibkriging,nlpd,1.11517 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,wing_weight,pylibkriging,fit_s,0.0297599 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,wing_weight,pylibkriging BFGS10,q2,0.99972 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,wing_weight,pylibkriging BFGS10,nlpd,1.11512 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,wing_weight,pylibkriging BFGS10,fit_s,0.335854 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,wing_weight,scikit-learn,q2,0.998048 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,wing_weight,scikit-learn,nlpd,2.57304 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,wing_weight,scikit-learn,fit_s,0.833252 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,wing_weight,SMT,q2,0.999898 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,wing_weight,SMT,nlpd,0.604023 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,wing_weight,SMT,fit_s,26.7889 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,wing_weight,OpenTURNS,q2,0.993145 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,wing_weight,OpenTURNS,nlpd,2.83627 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,wing_weight,OpenTURNS,fit_s,0.253929 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann6,pylibkriging,q2,0.422939 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann6,pylibkriging,nlpd,0.0965158 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann6,pylibkriging,fit_s,0.068651 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann6,pylibkriging BFGS10,q2,0.72352 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann6,pylibkriging BFGS10,nlpd,-0.156898 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann6,pylibkriging BFGS10,fit_s,0.877097 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann6,scikit-learn,q2,0.69049 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann6,scikit-learn,nlpd,-0.137845 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann6,scikit-learn,fit_s,1.43153 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann6,SMT,q2,0.708463 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann6,SMT,nlpd,-0.0177081 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann6,SMT,fit_s,11.2226 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann6,OpenTURNS,q2,0.696177 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann6,OpenTURNS,nlpd,-0.132838 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann6,OpenTURNS,fit_s,0.141383 diff --git a/tools/known_issues.csv b/tools/known_issues.csv index c69a1b4..8281855 100644 --- a/tools/known_issues.csv +++ b/tools/known_issues.csv @@ -1,11 +1,17 @@ suite,check,issue -python_Kriging-vecchia-nystrom_large-n,vecchia(n-1) LL == exact LL (same theta),With m >= n-1 neighbours the Vecchia likelihood should be the exact one; it matches for n=20 but not for n=40 (-111.349 vs -110.834) -python_Kriging-vecchia-nystrom_large-n,hartmann3/n=1500/nystrom(100): Q2,Nystrom predictor degrades as n grows at fixed k (theta estimates are fine: the exact predictor at the Nystrom theta has Q2 ~ 1) -python_Kriging-vecchia-nystrom_large-n,hartmann3/n=5000/nystrom(100): Q2,Nystrom predictor degrades as n grows at fixed k (theta estimates are fine: the exact predictor at the Nystrom theta has Q2 ~ 1) -python_Kriging-vecchia-nystrom_large-n,otl_circuit/n=1500/nystrom(100): Q2,Nystrom predictor degrades as n grows at fixed k (theta estimates are fine: the exact predictor at the Nystrom theta has Q2 ~ 1) -python_Kriging-vecchia-nystrom_large-n,otl_circuit/n=5000/nystrom(100): Q2,Nystrom predictor degrades as n grows at fixed k (theta estimates are fine: the exact predictor at the Nystrom theta has Q2 ~ 1) -python_Kriging_robustness-invalid-inputs,NaN in y is rejected,Kriging accepts NaN in y without raising -python_Kriging_robustness-invalid-inputs,Inf in y is rejected,Kriging accepts Inf in y without raising -python_Kriging_robustness-invalid-inputs,WarpKriging with 1 spec for 2 columns is rejected,WarpKriging accepts fewer warping specs than columns of X -python_Kriging_robustness-invalid-inputs,predict(stdev) time growth exponent in q (8000 vs 4000),"predict(return_stdev=True, return_cov=False) is O(q^2) in time and memory (10 000 points: 1.6 GB; 30 000: out of memory)" -python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy,sklearn RBF == libKriging gauss: simple kriging stdev (beta fixed),With beta fixed (is_beta_estim=False) predict still adds the trend-estimation variance (universal kriging) instead of the simple kriging variance +large-n_Kriging-vecchia-nystrom_python,vecchia(n-1) LL == exact LL (same theta),With m >= n-1 neighbours the Vecchia likelihood should be the exact one; it matches for n=20 but not for n=40 (-111.349 vs -110.834) +large-n_Kriging-vecchia-nystrom_python,hartmann3/n=1500/nystrom(100): Q2,Nystrom predictor degrades as n grows at fixed k (theta estimates are fine: the exact predictor at the Nystrom theta has Q2 ~ 1) +large-n_Kriging-vecchia-nystrom_python,hartmann3/n=5000/nystrom(100): Q2,Nystrom predictor degrades as n grows at fixed k (theta estimates are fine: the exact predictor at the Nystrom theta has Q2 ~ 1) +large-n_Kriging-vecchia-nystrom_python,otl_circuit/n=1500/nystrom(100): Q2,Nystrom predictor degrades as n grows at fixed k (theta estimates are fine: the exact predictor at the Nystrom theta has Q2 ~ 1) +large-n_Kriging-vecchia-nystrom_python,otl_circuit/n=5000/nystrom(100): Q2,Nystrom predictor degrades as n grows at fixed k (theta estimates are fine: the exact predictor at the Nystrom theta has Q2 ~ 1) +robustness-invalid-inputs_Kriging_python,NaN in y is rejected,Kriging accepts NaN in y without raising +robustness-invalid-inputs_Kriging_python,Inf in y is rejected,Kriging accepts Inf in y without raising +robustness-invalid-inputs_Kriging_python,WarpKriging with 1 spec for 2 columns is rejected,WarpKriging accepts fewer warping specs than columns of X +robustness-invalid-inputs_Kriging_python,predict(stdev) time growth exponent in q (8000 vs 4000),"predict(return_stdev=True, return_cov=False) is O(q^2) in time and memory (10 000 points: 1.6 GB; 30 000: out of memory)" +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,sklearn RBF == libKriging gauss: simple kriging stdev (beta fixed),With beta fixed (is_beta_estim=False) predict still adds the trend-estimation variance (universal kriging) instead of the simple kriging variance +deep-kernel_MLPKriging_R,MLPKriging: Q2(default optim) - Q2(BFGS10),https://github.com/libKriging/libKriging/issues/382 single-start MLPKriging fit stops after a few ms at a poor optimum +noisy-data_Kriging-nugget-noise_R,Kriging(nugget): LL(single start) - LL(10 restarts),https://github.com/libKriging/libKriging/issues/383 default single-start nugget fit lands far from the optimum +deep-kernel_MLPKriging_octave,MLPKriging: Q2(default optim) - Q2(BFGS10),https://github.com/libKriging/libKriging/issues/382 single-start MLPKriging fit stops after a few ms at a poor optimum +noisy-data_Kriging-nugget-noise_octave,Kriging(nugget): LL(single start) - LL(10 restarts),https://github.com/libKriging/libKriging/issues/383 default single-start nugget fit lands far from the optimum +deep-kernel_MLPKriging_julia,MLPKriging: Q2(default optim) - Q2(BFGS10),https://github.com/libKriging/libKriging/issues/382 single-start MLPKriging fit stops after a few ms at a poor optimum +noisy-data_Kriging-nugget-noise_julia,Kriging(nugget): LL(single start) - LL(10 restarts),https://github.com/libKriging/libKriging/issues/383 default single-start nugget fit lands far from the optimum diff --git a/tools/lkbench/__init__.py b/tools/lkbench/__init__.py index bebe556..82fbb65 100644 --- a/tools/lkbench/__init__.py +++ b/tools/lkbench/__init__.py @@ -1,4 +1,4 @@ """Helpers shared by the libKriging benchmark notebooks.""" -from . import checks, compat, design, functions, metrics # noqa: F401 +from . import checks, compat, design, functions, metrics, persistence # noqa: F401 from .checks import Suite # noqa: F401 from .functions import TESTS # noqa: F401 diff --git a/tools/lkbench/functions.py b/tools/lkbench/functions.py index 87537b2..33e52f9 100644 --- a/tools/lkbench/functions.py +++ b/tools/lkbench/functions.py @@ -3,7 +3,7 @@ Every function takes ``X`` of shape (n, d) in its *native* domain and returns a length-n vector. ``TESTS`` gathers the metadata (dimension, domain, a reference -point with its known value) used by ``python_no-model_test-functions-reference-values.ipynb`` +point with its known value) used by ``test-functions-reference-values_no-model_python.ipynb`` to check the implementations themselves before anything is benchmarked on them. """ from dataclasses import dataclass, field diff --git a/tools/lkbench/persistence.py b/tools/lkbench/persistence.py new file mode 100644 index 0000000..7c46785 --- /dev/null +++ b/tools/lkbench/persistence.py @@ -0,0 +1,50 @@ +"""Persistence checks shared by the notebooks of every model class: a model saved and +loaded back (class-specific and generic ``pylibkriging.load``), pickled, or deep-copied +must predict exactly as the original.""" +import copy +import pickle +import tempfile +from pathlib import Path + +import numpy as np + +from . import compat + + +def check(suite, name, k, Xn, predict): + """Record the persistence checks of model ``k`` (label ``name``) on the points ``Xn``. + + ``predict(model, Xn)`` returns (mean, stdev) as 1-D arrays; it carries the + prediction method of an approximate fit (Vecchia / Nystrom).""" + import pylibkriging as lk + m0, s0 = predict(k, Xn) + cls = type(k) + path = Path(tempfile.mkdtemp()) / f"{name.replace(' ', '_')}.json" + if not hasattr(k, "save"): + suite.skip(f"{name}: save + load", "no save() in this pylibkriging") + else: + with suite.runs(f"{name}: save + load"): + k.save(str(path)) + k1 = cls.load(str(path)) + m1, s1 = predict(k1, Xn) + suite.close(f"{name}: load(save(k)) predicts the same mean", m1, m0, rtol=1e-12, atol=1e-12) + suite.close(f"{name}: load(save(k)) predicts the same stdev", s1, s0, rtol=1e-10, atol=1e-12) + if hasattr(k, "theta"): + suite.close(f"{name}: same theta after load", np.ravel(k1.theta()), np.ravel(k.theta()), + rtol=1e-14, atol=0) + if hasattr(lk, "load") and (compat.has("load(noise)") or "noise" not in name and "nugget" not in name): + with suite.runs(f"{name}: generic pylibkriging.load"): + k2 = lk.load(str(path)) + suite.true(f"{name}: generic load returns {cls.__name__}", type(k2).__name__ == cls.__name__, + detail=type(k2).__name__) + suite.close(f"{name}: generic load predicts the same", predict(k2, Xn)[0], m0, rtol=1e-12, + atol=1e-12) + if not compat.has("pickle"): + suite.skip(f"{name}: pickle / deepcopy", "models not picklable in this pylibkriging") + return + with suite.runs(f"{name}: pickle round trip"): + k3 = pickle.loads(pickle.dumps(k)) + suite.close(f"{name}: pickle predicts the same", predict(k3, Xn)[0], m0, rtol=1e-12, atol=1e-12) + with suite.runs(f"{name}: deepcopy"): + k4 = copy.deepcopy(k) + suite.close(f"{name}: deepcopy predicts the same", predict(k4, Xn)[0], m0, rtol=1e-12, atol=1e-12) diff --git a/tools/report.py b/tools/report.py index 874a082..72cbaf1 100644 --- a/tools/report.py +++ b/tools/report.py @@ -15,7 +15,7 @@ 3. known issues whose check now passes ("FIXED?": remove them from tools/known_issues.csv); 4. cross-binding consistency: each language against Python, per quantity, with - the tolerances of python_Kriging_cross-binding-reference.ipynb; + the tolerances of cross-binding-reference_Kriging_python.ipynb; 5. drifts of the accuracy metrics (q2, rmse, nlpd) and of the fit times against a baseline (tools/baseline/metrics-.csv), when one is given. @@ -23,8 +23,8 @@ differs, or if --expect lists a suite that produced no output (crashed before finalize). The Markdown goes to stdout and, with --summary, is appended to a file (e.g. $GITHUB_STEP_SUMMARY). The suite of a notebook is its file name without -extension; `--expect python,R` expects one suite per `python_*.ipynb` and -`R_*.ipynb` notebook of the repository root (names are also accepted). +extension; `--expect python,R` expects one suite per `*_python.ipynb` and +`*_R.ipynb` notebook of the repository root (names are also accepted). Usage: python tools/report.py [--results results] [--baseline tools/baseline/metrics-master.csv] @@ -140,7 +140,7 @@ def main(argv=None): p.add_argument("--write-baseline", help="write the current metrics as a new baseline") p.add_argument("--summary", help="also append the report to this file") p.add_argument("--expect", default="", - help="comma-separated suites, or binding prefixes (python, R, octave, julia), that must have output") + help="comma-separated suites, or bindings (python, R, octave, julia), that must have output") args = p.parse_args(argv) checks = read_all(args.results, "*.checks.csv") @@ -162,7 +162,7 @@ def main(argv=None): expected = [] for item in filter(None, args.expect.split(",")): - stems = sorted(nb.stem for nb in ROOT.glob(f"{item}_*.ipynb")) + stems = sorted(nb.stem for nb in ROOT.glob(f"*_{item}.ipynb")) expected += stems or [item] missing = sorted(set(expected) - set(checks.get("suite", pd.Series(dtype=str)))) if missing: diff --git a/tools/run_notebooks.py b/tools/run_notebooks.py index 7c835c6..707b830 100644 --- a/tools/run_notebooks.py +++ b/tools/run_notebooks.py @@ -13,8 +13,8 @@ kernel (Python: installed; R: library; Octave: `OCTAVE_PATH`; Julia: environment). Usage: - python3 tools/run_notebooks.py python_*.ipynb - python3 tools/run_notebooks.py --inplace --timeout 3600 --kernel julia-1.11 julia_Kriging_cross-binding.ipynb + python3 tools/run_notebooks.py *_python.ipynb + python3 tools/run_notebooks.py --inplace --timeout 3600 --kernel julia-1.11 cross-binding_Kriging_julia.ipynb python3 tools/run_notebooks.py --skip-cell '^repo_root = ' --prelude 'addpath(...)' nb.ipynb Requires nbclient, nbformat and ipykernel. diff --git a/tools/src/R_all-classes_api-tour.R b/tools/src/R_all-classes_api-tour.R deleted file mode 100644 index 5d7b48f..0000000 --- a/tools/src/R_all-classes_api-tour.R +++ /dev/null @@ -1,205 +0,0 @@ -# %% [markdown] -# # R · all classes — every libKriging model class and fit method through rlibkriging -# -# One problem (Branin on [0,1]², Q² on 500 test points), every class and every fit -# method of the R binding, each with the checks that hold for it whatever the -# data: -# -# | model | checks (error-level unless stated) | -# |---|---| -# | `Kriging` (noise-free) | interpolation, Q² floor, conditional simulations through the data, `update` = fit on all points (same θ), `save`/`load` | -# | `Kriging(noise = "nugget")` | nugget > 0, smoothing next to the design points, single-start optimum as good as the multistart one (warn), `save`/`load` | -# | `Kriging(noise = )` | the stored variances, finite predictions | -# | `Kriging(method = "vecchia(m)")`, `"nystrom(k)"` | `k$method()`, `predict(method = ...)` finite, Q² floor (warn) | -# | `WarpKriging` (continuous and categorical warpings) | finite predictions, Q² floor (warn), `save`/`load` | -# | `MLPKriging` | finite predictions, Q² floor (warn), default optimizer as good as a multistart BFGS (warn), `save`/`load` | -# | `NestedKriging` (`NK`, `PoE`) | `NK` interpolates, stdev ≥ 0, `save`/`load` | -# -# A class missing from the installed rlibkriging (release channel) is recorded as -# SKIP. - -# %% -suppressPackageStartupMessages(library(rlibkriging)) -source("tools/r/checks.R") -s <- new_suite("R_all-classes_api-tour") -cat("rlibkriging", as.character(packageVersion("rlibkriging")), "\n") - -branin <- function(U) { # on [0,1]^2 - x1 <- 15 * U[, 1] - 5; x2 <- 15 * U[, 2] - (x2 - 5.1 / (4 * pi^2) * x1^2 + 5 / pi * x1 - 6)^2 + 10 * (1 - 1 / (8 * pi)) * cos(x1) + 10 -} -set.seed(1) -lhs <- function(n, d) sapply(1:d, function(j) (sample(n) - runif(n)) / n) -X <- lhs(30, 2); y <- branin(X) -Xt <- matrix(runif(1000), ncol = 2); yt <- branin(Xt) -tmp <- tempdir() -has_class <- function(name) exists(name, mode = "function") -skip <- function(s, check, reason) { - s$checks[[length(s$checks) + 1]] <- data.frame(suite = s$name, lang = "R", check = check, severity = "info", - status = "SKIP", value = "", expected = "", detail = reason, stringsAsFactors = FALSE) - message("[SKIP] ", check, ": ", reason) -} -roundtrip_same_mean <- function(s, label, k, Xn) { - f <- file.path(tmp, paste0(gsub("[^A-Za-z0-9]", "_", label), ".json")) - check_runs(s, paste0(label, ": save + load"), { - save(k, f) - k2 <- rlibkriging::load(f) - check_close(s, paste0(label, ": load(save(k)) predicts the same mean"), - predict(k2, Xn, return_stdev = FALSE)$mean, predict(k, Xn, return_stdev = FALSE)$mean, rtol = 1e-10) - }) -} - -# %% [markdown] -# ## `Kriging`, noise-free - -# %% -k <- Kriging(y, X, "matern5_2") -print(k) -p <- predict(k, Xt, return_stdev = TRUE) -add_metric(s, "branin", "Kriging", q2 = q2(yt, p$mean), rmse = rmse(yt, p$mean)) -check_greater(s, "Kriging: Q2", q2(yt, p$mean), 0.95) -check_less(s, "Kriging: interpolates (max |m(X) - y| / sd(y))", - max(abs(predict(k, X, return_stdev = FALSE)$mean - y)) / sd(y), 1e-6) -sims <- simulate(k, nsim = 5, seed = 2, x = X) -check_less(s, "Kriging: conditional simulations go through the data (max |sim(X) - y| / sigma)", - max(abs(sims - y)) / sqrt(k$sigma2()), 1e-4) -check_greater(s, "Kriging: stdev > 0 off the design", min(p$stdev), 0) -# update with 5 new points (no refit) = a fit on the 35 points at the same theta -Xu <- lhs(5, 2); yu <- branin(Xu) -ku <- copy(k) -update(ku, yu, Xu, refit = FALSE) -k35 <- Kriging(c(y, yu), rbind(X, Xu), "matern5_2", optim = "none", - parameters = list(theta = matrix(k$theta(), nrow = 1), is_theta_estim = FALSE, - sigma2 = k$sigma2(), is_sigma2_estim = FALSE)) -check_close(s, "Kriging: update(refit = FALSE) == fit on all points (same theta, sigma2)", - predict(ku, Xt, return_stdev = FALSE)$mean, predict(k35, Xt, return_stdev = FALSE)$mean, - rtol = 1e-6, atol = 1e-8 * sd(y)) -roundtrip_same_mean(s, "Kriging", k, Xt) - -# %% [markdown] -# ## `Kriging` with noise: estimated nugget, known variances -# -# Noise of standard deviation 5% of sd(y). The nugget model is fitted with 10 -# restarts (`optim = "BFGS10"`); the default single start is compared with it -# (warn-level: a lower likelihood means a local optimum). As DiceKriging's -# nugget, the predictor returns y exactly *at* the design points (the nugget is -# a micro-scale variation): smoothing is checked next to them. - -# %% -sd_noise <- 0.05 * sd(y) -yn <- y + rnorm(length(y), sd = sd_noise) -kn <- Kriging(yn, X, "matern5_2", noise = "nugget", optim = "BFGS10") -pn <- predict(kn, Xt, return_stdev = TRUE) -add_metric(s, "branin noisy", "Kriging nugget BFGS10", q2 = q2(yt, pn$mean), nugget = kn$nugget(), - ll = logLikelihood(kn)) -check_greater(s, "Kriging(nugget): estimated nugget > 0", kn$nugget(), 0) -check_greater(s, "Kriging(nugget): Q2", q2(yt, pn$mean), 0.95) -near <- pmin(X + 1e-7, 1) -check_greater(s, "Kriging(nugget): smooths the noisy data next to the design points (rms residual / noise sd)", - rmse(yn, predict(kn, near, return_stdev = FALSE)$mean) / sd_noise, 0.3) -kn1 <- Kriging(yn, X, "matern5_2", noise = "nugget") -add_metric(s, "branin noisy", "Kriging nugget BFGS", q2 = q2(yt, predict(kn1, Xt, return_stdev = FALSE)$mean), - nugget = kn1$nugget(), ll = logLikelihood(kn1)) -check_greater(s, "Kriging(nugget): LL(single start) - LL(10 restarts)", logLikelihood(kn1) - logLikelihood(kn), -1, - severity = "warn") -roundtrip_same_mean(s, "Kriging(nugget)", kn, Xt) -kh <- Kriging(yn, X, "matern5_2", noise = rep(sd_noise^2, length(yn))) -check_close(s, "Kriging(noise = variances): noise() returns them", kh$noise(), rep(sd_noise^2, length(yn)), - rtol = 1e-12) -ph <- predict(kh, Xt, return_stdev = TRUE) -check_less(s, "Kriging(noise = variances): non-finite predictions", sum(!is.finite(c(ph$mean, ph$stdev))), 1) -add_metric(s, "branin noisy", "Kriging known noise", q2 = q2(yt, ph$mean)) - -# %% [markdown] -# ## `Kriging` with an approximate fit: Vecchia, Nyström (n = 400) - -# %% -Xl <- lhs(400, 2); yl <- branin(Xl) -for (m in c("vecchia(20)", "nystrom(60)")) { - short <- sub("\\(.*", "", m) - check_runs(s, paste0("Kriging(method = '", m, "'): fit + predict"), { - ka <- Kriging(yl, Xl, "matern5_2", method = m) - add_check(s, paste0("Kriging(method = '", m, "'): method()"), identical(ka$method(), m), ka$method(), m) - pa <- predict(ka, Xt, return_stdev = TRUE, method = short) - check_less(s, paste0("Kriging(method = '", m, "'): non-finite predictions"), - sum(!is.finite(c(pa$mean, pa$stdev))), 1) - add_metric(s, "branin n=400", paste("Kriging", m), q2 = q2(yt, pa$mean)) - check_greater(s, paste0("Kriging(method = '", m, "'): Q2"), q2(yt, pa$mean), 0.95, severity = "warn") - }) -} - -# %% [markdown] -# ## `WarpKriging`: continuous warpings, then a categorical input -# -# The categorical case: x₂ restricted to 5 levels (coded 0, …, 4), embedded in ℝ². - -# %% -if (has_class("WarpKriging")) { - check_runs(s, "WarpKriging(kumaraswamy, kumaraswamy): fit + predict", { - kw <- WarpKriging(y, X, warping = c("kumaraswamy", "kumaraswamy"), kernel = "matern5_2") - pw <- predict(kw, Xt, return_stdev = TRUE) - check_less(s, "WarpKriging(kumaraswamy): non-finite predictions", sum(!is.finite(c(pw$mean, pw$stdev))), 1) - add_metric(s, "branin", "WarpKriging kumaraswamy", q2 = q2(yt, pw$mean)) - check_greater(s, "WarpKriging(kumaraswamy): Q2", q2(yt, pw$mean), 0.9, severity = "warn") - roundtrip_same_mean(s, "WarpKriging(kumaraswamy)", kw, Xt) - }) - lev <- function(U) cbind(U[, 1], round(U[, 2] * 4)) # level codes 0..4 - unlev <- function(L) cbind(L[, 1], L[, 2] / 4) - Xc <- lev(lhs(40, 2)); yc <- branin(unlev(Xc)) - Xtc <- lev(Xt); ytc <- branin(unlev(Xtc)) - check_runs(s, "WarpKriging(none, categorical(5,2)): fit + predict", { - kc <- WarpKriging(yc, Xc, warping = c("none", "categorical(5,2)"), kernel = "matern5_2") - pc <- predict(kc, Xtc, return_stdev = TRUE) - check_less(s, "WarpKriging(categorical): non-finite predictions", sum(!is.finite(c(pc$mean, pc$stdev))), 1) - add_metric(s, "branin, x2 categorical", "WarpKriging categorical", q2 = q2(ytc, pc$mean)) - check_greater(s, "WarpKriging(categorical): Q2", q2(ytc, pc$mean), 0.8, severity = "warn") - roundtrip_same_mean(s, "WarpKriging(categorical)", kc, Xtc) - }) -} else skip(s, "WarpKriging", "not in this rlibkriging") - -# %% [markdown] -# ## `MLPKriging`: a joint feature map ℝ² → ℝ² before the kernel - -# %% -if (has_class("MLPKriging")) { - check_runs(s, "MLPKriging: fit + predict", { - km <- MLPKriging(y, X, hidden_dims = c(8), d_out = 2, activation = "tanh", kernel = "matern5_2", optim = "BFGS10") - pm <- predict(km, Xt, return_stdev = TRUE) - check_less(s, "MLPKriging: non-finite predictions", sum(!is.finite(c(pm$mean, pm$stdev))), 1) - add_metric(s, "branin", "MLPKriging tanh [8] -> 2, BFGS10", q2 = q2(yt, pm$mean)) - check_greater(s, "MLPKriging: Q2", q2(yt, pm$mean), 0.9, severity = "warn") - roundtrip_same_mean(s, "MLPKriging", km, Xt) - # the default optimizer ("BFGS+Adam") - kd <- MLPKriging(y, X, hidden_dims = c(8), d_out = 2, activation = "tanh", kernel = "matern5_2") - qd <- q2(yt, predict(kd, Xt, return_stdev = FALSE)$mean) - add_metric(s, "branin", "MLPKriging tanh [8] -> 2, default optim", q2 = qd) - check_greater(s, "MLPKriging: Q2(default optim) - Q2(BFGS10)", qd - q2(yt, pm$mean), -0.05, severity = "warn") - }) -} else skip(s, "MLPKriging", "not in this rlibkriging") - -# %% [markdown] -# ## `NestedKriging`: 4 groups of 100 points, `NK` and `PoE` aggregations - -# %% -if (has_class("NestedKriging")) { - for (agg in c("NK", "PoE")) { - check_runs(s, paste0("NestedKriging(", agg, "): fit + predict"), { - kk <- NestedKriging(yl, Xl, kernel = "matern5_2", nb_groups = 4, aggregation = agg, seed = 3) - pk <- predict(kk, Xt, return_stdev = TRUE) - check_less(s, paste0("NestedKriging(", agg, "): stdev < 0"), sum(pk$stdev < 0), 1) - add_metric(s, "branin n=400", paste("NestedKriging", agg), q2 = q2(yt, pk$mean)) - check_greater(s, paste0("NestedKriging(", agg, "): Q2"), q2(yt, pk$mean), 0.95, severity = "warn") - if (agg == "NK") - check_less(s, "NestedKriging(NK): interpolates (max |m(X) - y| / sd(y))", - max(abs(predict(kk, Xl, return_stdev = FALSE)$mean - yl)) / sd(yl), 1e-3) - roundtrip_same_mean(s, paste0("NestedKriging(", agg, ")"), kk, Xt) - }) - } -} else skip(s, "NestedKriging", "not in this rlibkriging") - -# %% -me <- do.call(rbind, s$metrics) -print(me[me$metric == "q2", c("case", "package", "value")], row.names = FALSE) - -# %% -finalize(s) diff --git a/tools/src/python_NestedKriging_aggregations-large-n.py b/tools/src/aggregations-large-n_NestedKriging_python.py similarity index 86% rename from tools/src/python_NestedKriging_aggregations-large-n.py rename to tools/src/aggregations-large-n_NestedKriging_python.py index f02862b..c114924 100644 --- a/tools/src/python_NestedKriging_aggregations-large-n.py +++ b/tools/src/aggregations-large-n_NestedKriging_python.py @@ -7,7 +7,7 @@ # kriging predictor), or the products of experts `PoE`, `gPoE`, `BCM`, `rBCM` # (cheaper, variances not fully consistent). # -# Problems: as in `python_Kriging-vecchia-nystrom_large-n.ipynb`, Hartmann 3 +# Problems: as in `large-n_Kriging-vecchia-nystrom_python.ipynb`, Hartmann 3 # and the OTL circuit with n = 1500 (exact fit as reference) and n = 5000; then # every aggregation on Branin. # @@ -28,7 +28,7 @@ import lkbench from lkbench import TESTS, design, metrics, compat -suite = lkbench.Suite("python_NestedKriging_aggregations-large-n") +suite = lkbench.Suite("aggregations-large-n_NestedKriging_python") print("pylibkriging", compat.version()) if not compat.has("NestedKriging"): suite.skip("NestedKriging", "not available in this pylibkriging") @@ -97,5 +97,18 @@ def timed(f): lambda: lk.NestedKriging(y, X, "matern5_2", nb_groups=4, aggregation="NK", regmodel="linear"), severity="warn") +# %% [markdown] +# ## save / load, pickle, copy +# +# On Hartmann 3 (n = 40): a model saved and loaded back (class-specific and +# generic `pylibkriging.load`), pickled, or deep-copied predicts exactly as the +# original. + +# %% +if compat.has("NestedKriging"): + X, y, Xt, yt = design.dataset(TESTS["hartmann3"], 40, n_test=100, seed=61) + with suite.runs("NestedKriging: fit (persistence)"): + lkbench.persistence.check(suite, "NestedKriging", lk.NestedKriging(y, X, "matern5_2", nb_groups=4, seed=1), Xt, pred) + # %% suite.finalize() diff --git a/tools/src/aggregations_NestedKriging_R.R b/tools/src/aggregations_NestedKriging_R.R new file mode 100644 index 0000000..1beb97d --- /dev/null +++ b/tools/src/aggregations_NestedKriging_R.R @@ -0,0 +1,70 @@ +# %% [markdown] +# # R · `NestedKriging` — `NK` and `PoE` aggregations, 4 groups of 100 points +# +# Branin on [0,1]² (n = 30 LHS points unless stated, Q² on 500 uniform test +# points), through rlibkriging. Checks (error-level unless stated): +# +# * stdev ≥ 0, Q² > 0.95 (warn-level) +# * `NK` interpolates the data (it is a proper kriging predictor) +# * `save` then `load`: the same predicted mean +# +# A class missing from the installed rlibkriging (release channel) is recorded as SKIP. + +# %% +suppressPackageStartupMessages(library(rlibkriging)) +source("tools/r/checks.R") +s <- new_suite("aggregations_NestedKriging_R") +cat("rlibkriging", as.character(packageVersion("rlibkriging")), "\n") + +branin <- function(U) { # on [0,1]^2 + x1 <- 15 * U[, 1] - 5; x2 <- 15 * U[, 2] + (x2 - 5.1 / (4 * pi^2) * x1^2 + 5 / pi * x1 - 6)^2 + 10 * (1 - 1 / (8 * pi)) * cos(x1) + 10 +} +set.seed(1) +lhs <- function(n, d) sapply(1:d, function(j) (sample(n) - runif(n)) / n) +X <- lhs(30, 2); y <- branin(X) +Xt <- matrix(runif(1000), ncol = 2); yt <- branin(Xt) +tmp <- tempdir() +has_class <- function(name) exists(name, mode = "function") +skip <- function(s, check, reason) { + s$checks[[length(s$checks) + 1]] <- data.frame(suite = s$name, lang = "R", check = check, severity = "info", + status = "SKIP", value = "", expected = "", detail = reason, stringsAsFactors = FALSE) + message("[SKIP] ", check, ": ", reason) +} +roundtrip_same_mean <- function(s, label, k, Xn) { + f <- file.path(tmp, paste0(gsub("[^A-Za-z0-9]", "_", label), ".json")) + check_runs(s, paste0(label, ": save + load"), { + save(k, f) + k2 <- rlibkriging::load(f) + check_close(s, paste0(label, ": load(save(k)) predicts the same mean"), + predict(k2, Xn, return_stdev = FALSE)$mean, predict(k, Xn, return_stdev = FALSE)$mean, rtol = 1e-10) + }) +} + +# %% [markdown] +# ## `NestedKriging`: 4 groups of 100 points, `NK` and `PoE` aggregations + +# %% +Xl <- lhs(400, 2); yl <- branin(Xl) +if (has_class("NestedKriging")) { + for (agg in c("NK", "PoE")) { + check_runs(s, paste0("NestedKriging(", agg, "): fit + predict"), { + kk <- NestedKriging(yl, Xl, kernel = "matern5_2", nb_groups = 4, aggregation = agg, seed = 3) + pk <- predict(kk, Xt, return_stdev = TRUE) + check_less(s, paste0("NestedKriging(", agg, "): stdev < 0"), sum(pk$stdev < 0), 1) + add_metric(s, "branin n=400", paste("NestedKriging", agg), q2 = q2(yt, pk$mean)) + check_greater(s, paste0("NestedKriging(", agg, "): Q2"), q2(yt, pk$mean), 0.95, severity = "warn") + if (agg == "NK") + check_less(s, "NestedKriging(NK): interpolates (max |m(X) - y| / sd(y))", + max(abs(predict(kk, Xl, return_stdev = FALSE)$mean - yl)) / sd(yl), 1e-3) + roundtrip_same_mean(s, paste0("NestedKriging(", agg, ")"), kk, Xt) + }) + } +} else skip(s, "NestedKriging", "not in this rlibkriging") + +# %% +me <- do.call(rbind, s$metrics) +print(me[me$metric == "q2", c("case", "package", "value")], row.names = FALSE) + +# %% +finalize(s) diff --git a/tools/src/aggregations_NestedKriging_julia.jl b/tools/src/aggregations_NestedKriging_julia.jl new file mode 100644 index 0000000..cd4cfe5 --- /dev/null +++ b/tools/src/aggregations_NestedKriging_julia.jl @@ -0,0 +1,73 @@ +# %% [markdown] +# # Julia · `NestedKriging` — `NK` and `PoE` aggregations, 4 groups of 100 points +# +# Branin on [0,1]² (n = 30 LHS points unless stated, Q² on 500 uniform test +# points), through JLibKriging. Checks (error-level unless stated): +# +# * stdev ≥ 0, Q² > 0.95 (warn-level) +# * `NK` interpolates the data (it is a proper kriging predictor) +# * `save` then `load`: the same predicted mean +# +# The same checks as `aggregations_NestedKriging_R.ipynb`, with the Julia binding. + +# %% +import JLibKriging as LK +using Random +include("tools/julia/checks.jl") +s = Suite("aggregations_NestedKriging_julia") + +branin(U) = begin + x1 = 15 .* U[:, 1] .- 5; x2 = 15 .* U[:, 2] + (x2 .- 5.1 / (4π^2) .* x1 .^ 2 .+ 5 / π .* x1 .- 6) .^ 2 .+ 10 * (1 - 1 / (8π)) .* cos.(x1) .+ 10 +end +rng = MersenneTwister(1) +lhs(n, d) = hcat([(randperm(rng, n) .- rand(rng, n)) ./ n for _ in 1:d]...) +X = lhs(30, 2); y = branin(X) +Xt = rand(rng, 500, 2); yt = branin(Xt) +sd(v) = sqrt(sum((v .- sum(v) / length(v)) .^ 2) / (length(v) - 1)) +finite_count(v...) = count(!isfinite, vcat(v...)) + +function check_runs!(f, s, check) + try + f() + catch e + add_check!(s, check, false; value=string(typeof(e)), expected="no error", detail=first(sprint(showerror, e), 200)) + end +end + +function roundtrip!(s, label, k, Xn) + check_runs!(s, "$label: save + load") do + f = joinpath(mktempdir(), "k.json") + LK.save(k, f) + k2 = LK.load(f) + check_close!(s, "$label: load(save(k)) predicts the same mean", + LK.predict(k2, Xn; return_stdev=false).mean, LK.predict(k, Xn; return_stdev=false).mean; rtol=1e-10) + end +end + +# %% [markdown] +# ## `NestedKriging`: 4 groups of 100 points, `NK` and `PoE` aggregations + +# %% +Xl = lhs(400, 2); yl = branin(Xl) +for agg in ("NK", "PoE") + label = "NestedKriging($agg)" + check_runs!(s, "$label: fit + predict") do + kk = LK.NestedKriging(yl, Xl, "matern5_2", 4; aggregation=agg, seed=3) + pk = LK.predict(kk, Xt; return_stdev=true) + add_check!(s, "$label: stdev >= 0", all(pk.stdev .>= 0); value=minimum(pk.stdev), expected=">= 0") + add_metric!(s, "branin n=400", "NestedKriging $agg"; q2=q2(yt, pk.mean)) + add_check!(s, "$label: Q2", q2(yt, pk.mean) > 0.95; value=q2(yt, pk.mean), expected="> 0.95", severity="warn") + if agg == "NK" + e = maximum(abs.(LK.predict(kk, Xl; return_stdev=false).mean .- yl)) / sd(yl) + add_check!(s, "NestedKriging(NK): interpolates (max |m(X) - y| / sd(y))", e < 1e-3; value=e, expected="< 1e-3") + end + roundtrip!(s, label, kk, Xt) + end +end + +# %% +foreach(r -> r[5] == "q2" && println(rpad(r[3], 24), rpad(r[4], 28), r[6]), s.metrics) + +# %% +finalize!(s) diff --git a/tools/src/aggregations_NestedKriging_octave.m b/tools/src/aggregations_NestedKriging_octave.m new file mode 100644 index 0000000..3553028 --- /dev/null +++ b/tools/src/aggregations_NestedKriging_octave.m @@ -0,0 +1,83 @@ +% %% [markdown] +% # Octave · `NestedKriging` — `NK` and `PoE` aggregations, 4 groups of 100 points +% +% Branin on [0,1]² (n = 30 LHS points unless stated, Q² on 500 uniform test +% points), through mLibKriging. Checks (error-level unless stated): +% +% * stdev ≥ 0, Q² > 0.95 (warn-level) +% * `NK` interpolates the data (it is a proper kriging predictor) +% * `save` then `load`: the same predicted mean +% +% The same checks as `aggregations_NestedKriging_R.ipynb`, with the Octave binding. +% +% A class missing from the installed mLibKriging (release channel) is recorded as SKIP. + +% %% +addpath(fullfile(pwd, "tools", "octave")); +s = lkb_suite("aggregations_NestedKriging_octave"); +q2 = @(y, m) 1 - sum((y - m).^2) / sum((y - mean(y)).^2); +branin = @(U) ((15 * U(:, 2)) - 5.1 / (4 * pi^2) * (15 * U(:, 1) - 5).^2 + 5 / pi * (15 * U(:, 1) - 5) - 6).^2 ... + + 10 * (1 - 1 / (8 * pi)) * cos(15 * U(:, 1) - 5) + 10; +rand("seed", 1); randn("seed", 1); +function U = lhs(n, d) + U = zeros(n, d); + for j = 1:d, U(:, j) = (randperm(n)' - rand(n, 1)) / n; end +end +X = lhs(30, 2); y = branin(X); +Xt = rand(500, 2); yt = branin(Xt); +tmp = tempname(); mkdir(tmp); +has_class = @(name) exist(name) == 2; % a class file on the path + +function s = roundtrip(s, label, k, Xn, loader) + f = fullfile(tempdir(), [regexprep(label, "[^A-Za-z0-9]", "_") ".json"]); + try + k.save(f); + k2 = loader(f); + m1 = k.predict(Xn, false, false, false); + m2 = k2.predict(Xn, false, false, false); + s = lkb_close(s, [label ": load(save(k)) predicts the same mean"], m2, m1, 1e-10); + catch err + s = lkb_check(s, [label ": save + load"], false, "error", "no error", err.message); + end +end + +function s = skip(s, check, reason) + s.checks(end + 1, :) = {s.name, "octave", check, "info", "SKIP", "", "", reason}; + printf("[SKIP] %s: %s\n", check, reason); +end + +% %% [markdown] +% ## `NestedKriging`: 4 groups of 100 points, `NK` and `PoE` aggregations + +% %% +Xl = lhs(400, 2); yl = branin(Xl); +if has_class("NestedKriging") + for agg = {"NK", "PoE"} + agg = agg{1}; label = ["NestedKriging(" agg ")"]; + try + kk = NestedKriging(yl, Xl, "matern5_2", 4, agg, "kmeans", 3); + [mk, sk] = kk.predict(Xt); % stdev returned when asked for (2 outputs) + s = lkb_check(s, [label ": stdev >= 0"], all(sk >= 0), min(sk), ">= 0"); + s = lkb_metric(s, "branin n=400", ["NestedKriging " agg], "q2", q2(yt, mk)); + s = lkb_check(s, [label ": Q2"], q2(yt, mk) > 0.95, q2(yt, mk), "> 0.95", "", "warn"); + if strcmp(agg, "NK") + e = max(abs(kk.predict(Xl) - yl)) / std(yl); + s = lkb_check(s, "NestedKriging(NK): interpolates (max |m(X) - y| / sd(y))", e < 1e-3, e, "< 1e-3"); + end + f = fullfile(tmp, ["nested_" agg ".json"]); + kk.save(f); k2 = load_kriging(f); + s = lkb_close(s, [label ": load(save(k)) predicts the same mean"], k2.predict(Xt), kk.predict(Xt), 1e-10); + catch err + s = lkb_check(s, [label ": fit + predict"], false, "error", "no error", err.message); + end + end +else + s = skip(s, "NestedKriging", "not in this mLibKriging"); +end + +% %% +q = strcmp(s.metrics(:, 5), "q2"); +disp(s.metrics(q, [3, 4, 6])) + +% %% +lkb_finalize(s); diff --git a/tools/src/python_Kriging_cross-binding-reference.py b/tools/src/cross-binding-reference_Kriging_python.py similarity index 98% rename from tools/src/python_Kriging_cross-binding-reference.py rename to tools/src/cross-binding-reference_Kriging_python.py index a61ac4f..b8098c0 100644 --- a/tools/src/python_Kriging_cross-binding-reference.py +++ b/tools/src/cross-binding-reference_Kriging_python.py @@ -29,7 +29,7 @@ from lkbench import compat from lkbench.checks import DATA_DIR as DATA, RESULTS_DIR -suite = lkbench.Suite("python_Kriging_cross-binding-reference") +suite = lkbench.Suite("cross-binding-reference_Kriging_python") print("pylibkriging", compat.version()) DATASETS = ["branin", "hartmann3", "borehole"] rows = [] diff --git a/tools/src/R_Kriging_cross-binding.R b/tools/src/cross-binding_Kriging_R.R similarity index 95% rename from tools/src/R_Kriging_cross-binding.R rename to tools/src/cross-binding_Kriging_R.R index 785c8a8..4496ac8 100644 --- a/tools/src/R_Kriging_cross-binding.R +++ b/tools/src/cross-binding_Kriging_R.R @@ -1,14 +1,14 @@ # %% [markdown] # # R · Kriging — cross-binding consistency: the R side # -# Same protocol as `python_Kriging_cross-binding-reference.ipynb` (see there), on the +# Same protocol as `cross-binding-reference_Kriging_python.ipynb` (see there), on the # shared datasets of `data/`; writes `results/xbind_R.csv`, compared with the # Python values by `tools/report.py`. # %% suppressPackageStartupMessages(library(rlibkriging)) source("tools/r/checks.R") -s <- new_suite("R_Kriging_cross-binding") +s <- new_suite("cross-binding_Kriging_R") cat("rlibkriging", as.character(packageVersion("rlibkriging")), "\n") rows <- list() put <- function(dataset, quantity, values) { diff --git a/tools/src/julia_Kriging_cross-binding.jl b/tools/src/cross-binding_Kriging_julia.jl similarity index 95% rename from tools/src/julia_Kriging_cross-binding.jl rename to tools/src/cross-binding_Kriging_julia.jl index 6e16558..4e41d5a 100644 --- a/tools/src/julia_Kriging_cross-binding.jl +++ b/tools/src/cross-binding_Kriging_julia.jl @@ -1,7 +1,7 @@ # %% [markdown] # # Julia · Kriging — cross-binding consistency: the Julia side # -# Same protocol as `python_Kriging_cross-binding-reference.ipynb` (see there), on +# Same protocol as `cross-binding-reference_Kriging_python.ipynb` (see there), on # the shared datasets of `data/`; writes `results/xbind_julia.csv`, compared with # the Python values by `tools/report.py`. # @@ -12,7 +12,7 @@ # %% import JLibKriging as LK include("tools/julia/checks.jl") -s = Suite("julia_Kriging_cross-binding") +s = Suite("cross-binding_Kriging_julia") rows = Vector{Vector{String}}() record!(dataset, quantity, values) = for (i, v) in enumerate(vec(Float64.(collect(values)))) diff --git a/tools/src/octave_Kriging_cross-binding.m b/tools/src/cross-binding_Kriging_octave.m similarity index 95% rename from tools/src/octave_Kriging_cross-binding.m rename to tools/src/cross-binding_Kriging_octave.m index ad4cf57..5c2d89e 100644 --- a/tools/src/octave_Kriging_cross-binding.m +++ b/tools/src/cross-binding_Kriging_octave.m @@ -1,14 +1,14 @@ % %% [markdown] % # Octave · Kriging — cross-binding consistency: the Octave side % -% Same protocol as `python_Kriging_cross-binding-reference.ipynb` (see there), on the +% Same protocol as `cross-binding-reference_Kriging_python.ipynb` (see there), on the % shared datasets of `data/`; writes `results/xbind_octave.csv`, compared with % the Python values by `tools/report.py`. mLibKriging must be on the path % (`OCTAVE_PATH` or `addpath`). % %% addpath(fullfile(pwd, "tools", "octave")); -s = lkb_suite("octave_Kriging_cross-binding"); +s = lkb_suite("cross-binding_Kriging_octave"); out = {}; function out = put(out, dataset, quantity, values) v = double(values(:)); diff --git a/tools/src/python_MLPKriging_deep-kernel-projected-branin.py b/tools/src/deep-kernel-projected-branin_MLPKriging_python.py similarity index 79% rename from tools/src/python_MLPKriging_deep-kernel-projected-branin.py rename to tools/src/deep-kernel-projected-branin_MLPKriging_python.py index 849a2f1..966a59e 100644 --- a/tools/src/python_MLPKriging_deep-kernel-projected-branin.py +++ b/tools/src/deep-kernel-projected-branin_MLPKriging_python.py @@ -20,7 +20,7 @@ import lkbench from lkbench import TESTS, design, metrics, compat -suite = lkbench.Suite("python_MLPKriging_deep-kernel-projected-branin") +suite = lkbench.Suite("deep-kernel-projected-branin_MLPKriging_python") print("pylibkriging", compat.version()) @@ -66,5 +66,19 @@ def projected(X): else: suite.skip("MLPKriging", "not available in this pylibkriging") +# %% [markdown] +# ## save / load, pickle, copy +# +# On Hartmann 3 (n = 40): a model saved and loaded back (class-specific and +# generic `pylibkriging.load`), pickled, or deep-copied predicts exactly as the +# original. + +# %% +if compat.has("MLPKriging"): + X, y, Xt, yt = design.dataset(TESTS["hartmann3"], 40, n_test=100, seed=61) + with suite.runs("MLPKriging: fit (persistence)"): + lkbench.persistence.check(suite, "MLPKriging", lk.MLPKriging(y, X, hidden_dims=[8], d_out=2, activation="tanh", + kernel="matern5_2"), Xt, pred) + # %% suite.finalize() diff --git a/tools/src/deep-kernel_MLPKriging_R.R b/tools/src/deep-kernel_MLPKriging_R.R new file mode 100644 index 0000000..b0184da --- /dev/null +++ b/tools/src/deep-kernel_MLPKriging_R.R @@ -0,0 +1,69 @@ +# %% [markdown] +# # R · `MLPKriging` — a joint feature map ℝ² → ℝ² before the kernel +# +# Branin on [0,1]² (n = 30 LHS points unless stated, Q² on 500 uniform test +# points), through rlibkriging. Checks (error-level unless stated): +# +# * 10 restarts (`BFGS10`): finite predictions, Q² > 0.9 (warn-level) +# * `save` then `load`: the same predicted mean +# * the default optimizer as good as `BFGS10` (warn-level; see libKriging/libKriging#382) +# +# A class missing from the installed rlibkriging (release channel) is recorded as SKIP. + +# %% +suppressPackageStartupMessages(library(rlibkriging)) +source("tools/r/checks.R") +s <- new_suite("deep-kernel_MLPKriging_R") +cat("rlibkriging", as.character(packageVersion("rlibkriging")), "\n") + +branin <- function(U) { # on [0,1]^2 + x1 <- 15 * U[, 1] - 5; x2 <- 15 * U[, 2] + (x2 - 5.1 / (4 * pi^2) * x1^2 + 5 / pi * x1 - 6)^2 + 10 * (1 - 1 / (8 * pi)) * cos(x1) + 10 +} +set.seed(1) +lhs <- function(n, d) sapply(1:d, function(j) (sample(n) - runif(n)) / n) +X <- lhs(30, 2); y <- branin(X) +Xt <- matrix(runif(1000), ncol = 2); yt <- branin(Xt) +tmp <- tempdir() +has_class <- function(name) exists(name, mode = "function") +skip <- function(s, check, reason) { + s$checks[[length(s$checks) + 1]] <- data.frame(suite = s$name, lang = "R", check = check, severity = "info", + status = "SKIP", value = "", expected = "", detail = reason, stringsAsFactors = FALSE) + message("[SKIP] ", check, ": ", reason) +} +roundtrip_same_mean <- function(s, label, k, Xn) { + f <- file.path(tmp, paste0(gsub("[^A-Za-z0-9]", "_", label), ".json")) + check_runs(s, paste0(label, ": save + load"), { + save(k, f) + k2 <- rlibkriging::load(f) + check_close(s, paste0(label, ": load(save(k)) predicts the same mean"), + predict(k2, Xn, return_stdev = FALSE)$mean, predict(k, Xn, return_stdev = FALSE)$mean, rtol = 1e-10) + }) +} + +# %% [markdown] +# ## `MLPKriging`: a joint feature map ℝ² → ℝ² before the kernel + +# %% +if (has_class("MLPKriging")) { + check_runs(s, "MLPKriging: fit + predict", { + km <- MLPKriging(y, X, hidden_dims = c(8), d_out = 2, activation = "tanh", kernel = "matern5_2", optim = "BFGS10") + pm <- predict(km, Xt, return_stdev = TRUE) + check_less(s, "MLPKriging: non-finite predictions", sum(!is.finite(c(pm$mean, pm$stdev))), 1) + add_metric(s, "branin", "MLPKriging tanh [8] -> 2, BFGS10", q2 = q2(yt, pm$mean)) + check_greater(s, "MLPKriging: Q2", q2(yt, pm$mean), 0.9, severity = "warn") + roundtrip_same_mean(s, "MLPKriging", km, Xt) + # the default optimizer ("BFGS+Adam") + kd <- MLPKriging(y, X, hidden_dims = c(8), d_out = 2, activation = "tanh", kernel = "matern5_2") + qd <- q2(yt, predict(kd, Xt, return_stdev = FALSE)$mean) + add_metric(s, "branin", "MLPKriging tanh [8] -> 2, default optim", q2 = qd) + check_greater(s, "MLPKriging: Q2(default optim) - Q2(BFGS10)", qd - q2(yt, pm$mean), -0.05, severity = "warn") + }) +} else skip(s, "MLPKriging", "not in this rlibkriging") + +# %% +me <- do.call(rbind, s$metrics) +print(me[me$metric == "q2", c("case", "package", "value")], row.names = FALSE) + +# %% +finalize(s) diff --git a/tools/src/deep-kernel_MLPKriging_julia.jl b/tools/src/deep-kernel_MLPKriging_julia.jl new file mode 100644 index 0000000..57b1d87 --- /dev/null +++ b/tools/src/deep-kernel_MLPKriging_julia.jl @@ -0,0 +1,71 @@ +# %% [markdown] +# # Julia · `MLPKriging` — a joint feature map ℝ² → ℝ² before the kernel +# +# Branin on [0,1]² (n = 30 LHS points unless stated, Q² on 500 uniform test +# points), through JLibKriging. Checks (error-level unless stated): +# +# * 10 restarts (`BFGS10`): finite predictions, Q² > 0.9 (warn-level) +# * `save` then `load`: the same predicted mean +# * the default optimizer as good as `BFGS10` (warn-level; see libKriging/libKriging#382) +# +# The same checks as `deep-kernel_MLPKriging_R.ipynb`, with the Julia binding. + +# %% +import JLibKriging as LK +using Random +include("tools/julia/checks.jl") +s = Suite("deep-kernel_MLPKriging_julia") + +branin(U) = begin + x1 = 15 .* U[:, 1] .- 5; x2 = 15 .* U[:, 2] + (x2 .- 5.1 / (4π^2) .* x1 .^ 2 .+ 5 / π .* x1 .- 6) .^ 2 .+ 10 * (1 - 1 / (8π)) .* cos.(x1) .+ 10 +end +rng = MersenneTwister(1) +lhs(n, d) = hcat([(randperm(rng, n) .- rand(rng, n)) ./ n for _ in 1:d]...) +X = lhs(30, 2); y = branin(X) +Xt = rand(rng, 500, 2); yt = branin(Xt) +sd(v) = sqrt(sum((v .- sum(v) / length(v)) .^ 2) / (length(v) - 1)) +finite_count(v...) = count(!isfinite, vcat(v...)) + +function check_runs!(f, s, check) + try + f() + catch e + add_check!(s, check, false; value=string(typeof(e)), expected="no error", detail=first(sprint(showerror, e), 200)) + end +end + +function roundtrip!(s, label, k, Xn) + check_runs!(s, "$label: save + load") do + f = joinpath(mktempdir(), "k.json") + LK.save(k, f) + k2 = LK.load(f) + check_close!(s, "$label: load(save(k)) predicts the same mean", + LK.predict(k2, Xn; return_stdev=false).mean, LK.predict(k, Xn; return_stdev=false).mean; rtol=1e-10) + end +end + +# %% [markdown] +# ## `MLPKriging`: a joint feature map ℝ² → ℝ² before the kernel + +# %% +check_runs!(s, "MLPKriging: fit + predict") do + km = LK.MLPKriging(y, X, [8], 2; activation="tanh", kernel="matern5_2", optim="BFGS10") + pm = LK.predict(km, Xt; return_stdev=true) + add_check!(s, "MLPKriging: finite predictions", finite_count(pm.mean, pm.stdev) == 0; + value=finite_count(pm.mean, pm.stdev), expected="0 non-finite") + add_metric!(s, "branin", "MLPKriging tanh [8] -> 2, BFGS10"; q2=q2(yt, pm.mean)) + add_check!(s, "MLPKriging: Q2", q2(yt, pm.mean) > 0.9; value=q2(yt, pm.mean), expected="> 0.9", severity="warn") + roundtrip!(s, "MLPKriging", km, Xt) + kd = LK.MLPKriging(y, X, [8], 2; activation="tanh", kernel="matern5_2") # default optimizer + qd = q2(yt, LK.predict(kd, Xt; return_stdev=false).mean) + add_metric!(s, "branin", "MLPKriging tanh [8] -> 2, default optim"; q2=qd) + add_check!(s, "MLPKriging: Q2(default optim) - Q2(BFGS10)", qd - q2(yt, pm.mean) > -0.05; + value=qd - q2(yt, pm.mean), expected="> -0.05", severity="warn") +end + +# %% +foreach(r -> r[5] == "q2" && println(rpad(r[3], 24), rpad(r[4], 28), r[6]), s.metrics) + +# %% +finalize!(s) diff --git a/tools/src/deep-kernel_MLPKriging_octave.m b/tools/src/deep-kernel_MLPKriging_octave.m new file mode 100644 index 0000000..46476ac --- /dev/null +++ b/tools/src/deep-kernel_MLPKriging_octave.m @@ -0,0 +1,77 @@ +% %% [markdown] +% # Octave · `MLPKriging` — a joint feature map ℝ² → ℝ² before the kernel +% +% Branin on [0,1]² (n = 30 LHS points unless stated, Q² on 500 uniform test +% points), through mLibKriging. Checks (error-level unless stated): +% +% * 10 restarts (`BFGS10`): finite predictions, Q² > 0.9 (warn-level) +% * `save` then `load`: the same predicted mean +% * the default optimizer as good as `BFGS10` (warn-level; see libKriging/libKriging#382) +% +% The same checks as `deep-kernel_MLPKriging_R.ipynb`, with the Octave binding. +% +% A class missing from the installed mLibKriging (release channel) is recorded as SKIP. + +% %% +addpath(fullfile(pwd, "tools", "octave")); +s = lkb_suite("deep-kernel_MLPKriging_octave"); +q2 = @(y, m) 1 - sum((y - m).^2) / sum((y - mean(y)).^2); +branin = @(U) ((15 * U(:, 2)) - 5.1 / (4 * pi^2) * (15 * U(:, 1) - 5).^2 + 5 / pi * (15 * U(:, 1) - 5) - 6).^2 ... + + 10 * (1 - 1 / (8 * pi)) * cos(15 * U(:, 1) - 5) + 10; +rand("seed", 1); randn("seed", 1); +function U = lhs(n, d) + U = zeros(n, d); + for j = 1:d, U(:, j) = (randperm(n)' - rand(n, 1)) / n; end +end +X = lhs(30, 2); y = branin(X); +Xt = rand(500, 2); yt = branin(Xt); +tmp = tempname(); mkdir(tmp); +has_class = @(name) exist(name) == 2; % a class file on the path + +function s = roundtrip(s, label, k, Xn, loader) + f = fullfile(tempdir(), [regexprep(label, "[^A-Za-z0-9]", "_") ".json"]); + try + k.save(f); + k2 = loader(f); + m1 = k.predict(Xn, false, false, false); + m2 = k2.predict(Xn, false, false, false); + s = lkb_close(s, [label ": load(save(k)) predicts the same mean"], m2, m1, 1e-10); + catch err + s = lkb_check(s, [label ": save + load"], false, "error", "no error", err.message); + end +end + +function s = skip(s, check, reason) + s.checks(end + 1, :) = {s.name, "octave", check, "info", "SKIP", "", "", reason}; + printf("[SKIP] %s: %s\n", check, reason); +end + +% %% [markdown] +% ## `MLPKriging`: a joint feature map ℝ² → ℝ² before the kernel + +% %% +if has_class("MLPKriging") + try + km = MLPKriging(y, X, [8], 2, "tanh", "matern5_2", "constant", false, "BFGS10"); + [mm, sm] = km.predict(Xt, true, false, false); + s = lkb_check(s, "MLPKriging: finite predictions", all(isfinite([mm; sm])), sum(~isfinite([mm; sm])), "0 non-finite"); + s = lkb_metric(s, "branin", "MLPKriging tanh [8] -> 2, BFGS10", "q2", q2(yt, mm)); + s = lkb_check(s, "MLPKriging: Q2", q2(yt, mm) > 0.9, q2(yt, mm), "> 0.9", "", "warn"); + s = roundtrip(s, "MLPKriging", km, Xt, @load_kriging); + kd = MLPKriging(y, X, [8], 2, "tanh", "matern5_2"); % default optimizer + qd = q2(yt, kd.predict(Xt, false, false, false)); + s = lkb_metric(s, "branin", "MLPKriging tanh [8] -> 2, default optim", "q2", qd); + s = lkb_check(s, "MLPKriging: Q2(default optim) - Q2(BFGS10)", qd - q2(yt, mm) > -0.05, qd - q2(yt, mm), "> -0.05", "", "warn"); + catch err + s = lkb_check(s, "MLPKriging: fit + predict", false, "error", "no error", err.message); + end +else + s = skip(s, "MLPKriging", "not in this mLibKriging"); +end + +% %% +q = strcmp(s.metrics(:, 5), "q2"); +disp(s.metrics(q, [3, 4, 6])) + +% %% +lkb_finalize(s); diff --git a/tools/src/python_Kriging_high-dimension-normalize-screening.py b/tools/src/high-dimension-normalize-screening_Kriging_python.py similarity index 96% rename from tools/src/python_Kriging_high-dimension-normalize-screening.py rename to tools/src/high-dimension-normalize-screening_Kriging_python.py index a127c64..0282456 100644 --- a/tools/src/python_Kriging_high-dimension-normalize-screening.py +++ b/tools/src/high-dimension-normalize-screening_Kriging_python.py @@ -20,7 +20,7 @@ # active ones (screening ability, warn-level). # # `MLPKriging` on a high-dimensional function with a low-dimensional structure: -# `python_MLPKriging_deep-kernel-projected-branin.ipynb`. +# `deep-kernel-projected-branin_MLPKriging_python.ipynb`. # %% import time @@ -32,7 +32,7 @@ import lkbench from lkbench import TESTS, design, metrics, compat -suite = lkbench.Suite("python_Kriging_high-dimension-normalize-screening") +suite = lkbench.Suite("high-dimension-normalize-screening_Kriging_python") print("pylibkriging", compat.version()) diff --git a/tools/src/interpolation-simulate-update-save_Kriging_R.R b/tools/src/interpolation-simulate-update-save_Kriging_R.R new file mode 100644 index 0000000..8a3726b --- /dev/null +++ b/tools/src/interpolation-simulate-update-save_Kriging_R.R @@ -0,0 +1,76 @@ +# %% [markdown] +# # R · `Kriging`, noise-free — interpolation, conditional simulation, update, save/load +# +# Branin on [0,1]² (n = 30 LHS points unless stated, Q² on 500 uniform test +# points), through rlibkriging. Checks (error-level unless stated): +# +# * interpolation (max |m(X) − y| / sd(y) < 1e-6) and Q² > 0.95 on 500 test points +# * conditional simulations through the data (relative to σ) +# * stdev > 0 off the design +# * `update` without refit = a fit on all the points at the same θ, σ² +# * `save` then `load`: the same predicted mean + +# %% +suppressPackageStartupMessages(library(rlibkriging)) +source("tools/r/checks.R") +s <- new_suite("interpolation-simulate-update-save_Kriging_R") +cat("rlibkriging", as.character(packageVersion("rlibkriging")), "\n") + +branin <- function(U) { # on [0,1]^2 + x1 <- 15 * U[, 1] - 5; x2 <- 15 * U[, 2] + (x2 - 5.1 / (4 * pi^2) * x1^2 + 5 / pi * x1 - 6)^2 + 10 * (1 - 1 / (8 * pi)) * cos(x1) + 10 +} +set.seed(1) +lhs <- function(n, d) sapply(1:d, function(j) (sample(n) - runif(n)) / n) +X <- lhs(30, 2); y <- branin(X) +Xt <- matrix(runif(1000), ncol = 2); yt <- branin(Xt) +tmp <- tempdir() +has_class <- function(name) exists(name, mode = "function") +skip <- function(s, check, reason) { + s$checks[[length(s$checks) + 1]] <- data.frame(suite = s$name, lang = "R", check = check, severity = "info", + status = "SKIP", value = "", expected = "", detail = reason, stringsAsFactors = FALSE) + message("[SKIP] ", check, ": ", reason) +} +roundtrip_same_mean <- function(s, label, k, Xn) { + f <- file.path(tmp, paste0(gsub("[^A-Za-z0-9]", "_", label), ".json")) + check_runs(s, paste0(label, ": save + load"), { + save(k, f) + k2 <- rlibkriging::load(f) + check_close(s, paste0(label, ": load(save(k)) predicts the same mean"), + predict(k2, Xn, return_stdev = FALSE)$mean, predict(k, Xn, return_stdev = FALSE)$mean, rtol = 1e-10) + }) +} + +# %% [markdown] +# ## `Kriging`, noise-free + +# %% +k <- Kriging(y, X, "matern5_2") +print(k) +p <- predict(k, Xt, return_stdev = TRUE) +add_metric(s, "branin", "Kriging", q2 = q2(yt, p$mean), rmse = rmse(yt, p$mean)) +check_greater(s, "Kriging: Q2", q2(yt, p$mean), 0.95) +check_less(s, "Kriging: interpolates (max |m(X) - y| / sd(y))", + max(abs(predict(k, X, return_stdev = FALSE)$mean - y)) / sd(y), 1e-6) +sims <- simulate(k, nsim = 5, seed = 2, x = X) +check_less(s, "Kriging: conditional simulations go through the data (max |sim(X) - y| / sigma)", + max(abs(sims - y)) / sqrt(k$sigma2()), 1e-4) +check_greater(s, "Kriging: stdev > 0 off the design", min(p$stdev), 0) +# update with 5 new points (no refit) = a fit on the 35 points at the same theta +Xu <- lhs(5, 2); yu <- branin(Xu) +ku <- copy(k) +update(ku, yu, Xu, refit = FALSE) +k35 <- Kriging(c(y, yu), rbind(X, Xu), "matern5_2", optim = "none", + parameters = list(theta = matrix(k$theta(), nrow = 1), is_theta_estim = FALSE, + sigma2 = k$sigma2(), is_sigma2_estim = FALSE)) +check_close(s, "Kriging: update(refit = FALSE) == fit on all points (same theta, sigma2)", + predict(ku, Xt, return_stdev = FALSE)$mean, predict(k35, Xt, return_stdev = FALSE)$mean, + rtol = 1e-6, atol = 1e-8 * sd(y)) +roundtrip_same_mean(s, "Kriging", k, Xt) + +# %% +me <- do.call(rbind, s$metrics) +print(me[me$metric == "q2", c("case", "package", "value")], row.names = FALSE) + +# %% +finalize(s) diff --git a/tools/src/interpolation-simulate-update-save_Kriging_julia.jl b/tools/src/interpolation-simulate-update-save_Kriging_julia.jl new file mode 100644 index 0000000..6ea1743 --- /dev/null +++ b/tools/src/interpolation-simulate-update-save_Kriging_julia.jl @@ -0,0 +1,79 @@ +# %% [markdown] +# # Julia · `Kriging`, noise-free — interpolation, conditional simulation, update, save/load +# +# Branin on [0,1]² (n = 30 LHS points unless stated, Q² on 500 uniform test +# points), through JLibKriging. Checks (error-level unless stated): +# +# * interpolation (max |m(X) − y| / sd(y) < 1e-6) and Q² > 0.95 on 500 test points +# * conditional simulations through the data (relative to σ) +# * stdev > 0 off the design +# * `update` without refit = a fit on all the points at the same θ, σ² +# * `save` then `load`: the same predicted mean +# +# The same checks as `interpolation-simulate-update-save_Kriging_R.ipynb`, with the Julia binding. + +# %% +import JLibKriging as LK +using Random +include("tools/julia/checks.jl") +s = Suite("interpolation-simulate-update-save_Kriging_julia") + +branin(U) = begin + x1 = 15 .* U[:, 1] .- 5; x2 = 15 .* U[:, 2] + (x2 .- 5.1 / (4π^2) .* x1 .^ 2 .+ 5 / π .* x1 .- 6) .^ 2 .+ 10 * (1 - 1 / (8π)) .* cos.(x1) .+ 10 +end +rng = MersenneTwister(1) +lhs(n, d) = hcat([(randperm(rng, n) .- rand(rng, n)) ./ n for _ in 1:d]...) +X = lhs(30, 2); y = branin(X) +Xt = rand(rng, 500, 2); yt = branin(Xt) +sd(v) = sqrt(sum((v .- sum(v) / length(v)) .^ 2) / (length(v) - 1)) +finite_count(v...) = count(!isfinite, vcat(v...)) + +function check_runs!(f, s, check) + try + f() + catch e + add_check!(s, check, false; value=string(typeof(e)), expected="no error", detail=first(sprint(showerror, e), 200)) + end +end + +function roundtrip!(s, label, k, Xn) + check_runs!(s, "$label: save + load") do + f = joinpath(mktempdir(), "k.json") + LK.save(k, f) + k2 = LK.load(f) + check_close!(s, "$label: load(save(k)) predicts the same mean", + LK.predict(k2, Xn; return_stdev=false).mean, LK.predict(k, Xn; return_stdev=false).mean; rtol=1e-10) + end +end + +# %% [markdown] +# ## `Kriging`, noise-free + +# %% +k = LK.Kriging(y, X, "matern5_2") +p = LK.predict(k, Xt; return_stdev=true) +add_metric!(s, "branin", "Kriging"; q2=q2(yt, p.mean)) +add_check!(s, "Kriging: Q2", q2(yt, p.mean) > 0.95; value=q2(yt, p.mean), expected="> 0.95") +e = maximum(abs.(LK.predict(k, X; return_stdev=false).mean .- y)) / sd(y) +add_check!(s, "Kriging: interpolates (max |m(X) - y| / sd(y))", e < 1e-6; value=e, expected="< 1e-6") +S = LK.simulate(k, 5, 2, X) +e = maximum(abs.(S .- y)) / sqrt(LK.sigma2(k)) +add_check!(s, "Kriging: conditional simulations go through the data (max |sim(X) - y| / sigma)", e < 1e-4; + value=e, expected="< 1e-4") +add_check!(s, "Kriging: stdev > 0 off the design", minimum(p.stdev) > 0; value=minimum(p.stdev), expected="> 0") +Xu = lhs(5, 2); yu = branin(Xu) +ku = copy(k) +LK.update!(ku, yu, Xu; refit=false) +k35 = LK.Kriging(vcat(y, yu), vcat(X, Xu), "matern5_2"; optim="none", + theta=vec(LK.theta(k)), is_theta_estim=false, sigma2=LK.sigma2(k), is_sigma2_estim=false) +check_close!(s, "Kriging: update!(refit=false) == fit on all points (same theta, sigma2)", + LK.predict(ku, Xt; return_stdev=false).mean, LK.predict(k35, Xt; return_stdev=false).mean; + rtol=1e-6, atol=1e-8 * sd(y)) +roundtrip!(s, "Kriging", k, Xt) + +# %% +foreach(r -> r[5] == "q2" && println(rpad(r[3], 24), rpad(r[4], 28), r[6]), s.metrics) + +# %% +finalize!(s) diff --git a/tools/src/interpolation-simulate-update-save_Kriging_octave.m b/tools/src/interpolation-simulate-update-save_Kriging_octave.m new file mode 100644 index 0000000..bad7144 --- /dev/null +++ b/tools/src/interpolation-simulate-update-save_Kriging_octave.m @@ -0,0 +1,79 @@ +% %% [markdown] +% # Octave · `Kriging`, noise-free — interpolation, conditional simulation, update, save/load +% +% Branin on [0,1]² (n = 30 LHS points unless stated, Q² on 500 uniform test +% points), through mLibKriging. Checks (error-level unless stated): +% +% * interpolation (max |m(X) − y| / sd(y) < 1e-6) and Q² > 0.95 on 500 test points +% * conditional simulations through the data (relative to σ) +% * stdev > 0 off the design +% * `update` without refit = a fit on all the points at the same θ, σ² +% * `save` then `load`: the same predicted mean +% +% The same checks as `interpolation-simulate-update-save_Kriging_R.ipynb`, with the Octave binding. + +% %% +addpath(fullfile(pwd, "tools", "octave")); +s = lkb_suite("interpolation-simulate-update-save_Kriging_octave"); +q2 = @(y, m) 1 - sum((y - m).^2) / sum((y - mean(y)).^2); +branin = @(U) ((15 * U(:, 2)) - 5.1 / (4 * pi^2) * (15 * U(:, 1) - 5).^2 + 5 / pi * (15 * U(:, 1) - 5) - 6).^2 ... + + 10 * (1 - 1 / (8 * pi)) * cos(15 * U(:, 1) - 5) + 10; +rand("seed", 1); randn("seed", 1); +function U = lhs(n, d) + U = zeros(n, d); + for j = 1:d, U(:, j) = (randperm(n)' - rand(n, 1)) / n; end +end +X = lhs(30, 2); y = branin(X); +Xt = rand(500, 2); yt = branin(Xt); +tmp = tempname(); mkdir(tmp); +has_class = @(name) exist(name) == 2; % a class file on the path + +function s = roundtrip(s, label, k, Xn, loader) + f = fullfile(tempdir(), [regexprep(label, "[^A-Za-z0-9]", "_") ".json"]); + try + k.save(f); + k2 = loader(f); + m1 = k.predict(Xn, false, false, false); + m2 = k2.predict(Xn, false, false, false); + s = lkb_close(s, [label ": load(save(k)) predicts the same mean"], m2, m1, 1e-10); + catch err + s = lkb_check(s, [label ": save + load"], false, "error", "no error", err.message); + end +end + +function s = skip(s, check, reason) + s.checks(end + 1, :) = {s.name, "octave", check, "info", "SKIP", "", "", reason}; + printf("[SKIP] %s: %s\n", check, reason); +end + +% %% [markdown] +% ## `Kriging`, noise-free + +% %% +k = Kriging(y, X, "matern5_2"); +[m, sd] = k.predict(Xt, true, false, false); +s = lkb_metric(s, "branin", "Kriging", "q2", q2(yt, m)); +s = lkb_check(s, "Kriging: Q2", q2(yt, m) > 0.95, q2(yt, m), "> 0.95"); +mX = k.predict(X, false, false, false); +e = max(abs(mX - y)) / std(y); +s = lkb_check(s, "Kriging: interpolates (max |m(X) - y| / sd(y))", e < 1e-6, e, "< 1e-6"); +S = k.simulate(int32(5), int32(2), X, false); +e = max(max(abs(S - y))) / sqrt(k.sigma2()); +s = lkb_check(s, "Kriging: conditional simulations go through the data (max |sim(X) - y| / sigma)", e < 1e-4, e, "< 1e-4"); +s = lkb_check(s, "Kriging: stdev > 0 off the design", min(sd) > 0, min(sd), "> 0"); +Xu = lhs(5, 2); yu = branin(Xu); +ku = k.copy(); +ku.update(yu, Xu, false); +th = k.theta(); +k35 = Kriging([y; yu], [X; Xu], "matern5_2", "constant", false, "none", "LL", ... + Params("theta", th(:)', "is_theta_estim", false, "sigma2", k.sigma2(), "is_sigma2_estim", false)); +s = lkb_close(s, "Kriging: update(refit = false) == fit on all points (same theta, sigma2)", ... + ku.predict(Xt, false, false, false), k35.predict(Xt, false, false, false), 1e-6, 1e-8 * std(y)); +s = roundtrip(s, "Kriging", k, Xt, @load_kriging); + +% %% +q = strcmp(s.metrics(:, 5), "q2"); +disp(s.metrics(q, [3, 4, 6])) + +% %% +lkb_finalize(s); diff --git a/tools/src/julia_all-classes_api-tour.jl b/tools/src/julia_all-classes_api-tour.jl deleted file mode 100644 index 5b743d1..0000000 --- a/tools/src/julia_all-classes_api-tour.jl +++ /dev/null @@ -1,198 +0,0 @@ -# %% [markdown] -# # Julia · all classes — every libKriging model class and fit method through JLibKriging -# -# Same tour as `R_all-classes_api-tour.ipynb` (see there for the checks): Branin -# on [0,1]², every class and fit method of the Julia binding, each with the -# checks that hold for it whatever the data. -# -# The module is `JLibKriging` (the packaged binding, [JLibKriging.jl](https://github.com/libKriging/JLibKriging.jl)), -# imported under a short name so that its exported accessors (`X`, `y`, `theta`, ...) -# do not clash with the variables of the notebook. - -# %% -import JLibKriging as LK -using Random -include("tools/julia/checks.jl") -s = Suite("julia_all-classes_api-tour") - -branin(U) = begin - x1 = 15 .* U[:, 1] .- 5; x2 = 15 .* U[:, 2] - (x2 .- 5.1 / (4π^2) .* x1 .^ 2 .+ 5 / π .* x1 .- 6) .^ 2 .+ 10 * (1 - 1 / (8π)) .* cos.(x1) .+ 10 -end -rng = MersenneTwister(1) -lhs(n, d) = hcat([(randperm(rng, n) .- rand(rng, n)) ./ n for _ in 1:d]...) -X = lhs(30, 2); y = branin(X) -Xt = rand(rng, 500, 2); yt = branin(Xt) -sd(v) = sqrt(sum((v .- sum(v) / length(v)) .^ 2) / (length(v) - 1)) -finite_count(v...) = count(!isfinite, vcat(v...)) - -function check_runs!(f, s, check) - try - f() - catch e - add_check!(s, check, false; value=string(typeof(e)), expected="no error", detail=first(sprint(showerror, e), 200)) - end -end - -function roundtrip!(s, label, k, Xn) - check_runs!(s, "$label: save + load") do - f = joinpath(mktempdir(), "k.json") - LK.save(k, f) - k2 = LK.load(f) - check_close!(s, "$label: load(save(k)) predicts the same mean", - LK.predict(k2, Xn; return_stdev=false).mean, LK.predict(k, Xn; return_stdev=false).mean; rtol=1e-10) - end -end - -# %% [markdown] -# ## `Kriging`, noise-free - -# %% -k = LK.Kriging(y, X, "matern5_2") -p = LK.predict(k, Xt; return_stdev=true) -add_metric!(s, "branin", "Kriging"; q2=q2(yt, p.mean)) -add_check!(s, "Kriging: Q2", q2(yt, p.mean) > 0.95; value=q2(yt, p.mean), expected="> 0.95") -e = maximum(abs.(LK.predict(k, X; return_stdev=false).mean .- y)) / sd(y) -add_check!(s, "Kriging: interpolates (max |m(X) - y| / sd(y))", e < 1e-6; value=e, expected="< 1e-6") -S = LK.simulate(k, 5, 2, X) -e = maximum(abs.(S .- y)) / sqrt(LK.sigma2(k)) -add_check!(s, "Kriging: conditional simulations go through the data (max |sim(X) - y| / sigma)", e < 1e-4; - value=e, expected="< 1e-4") -add_check!(s, "Kriging: stdev > 0 off the design", minimum(p.stdev) > 0; value=minimum(p.stdev), expected="> 0") -Xu = lhs(5, 2); yu = branin(Xu) -ku = copy(k) -LK.update!(ku, yu, Xu; refit=false) -k35 = LK.Kriging(vcat(y, yu), vcat(X, Xu), "matern5_2"; optim="none", - theta=vec(LK.theta(k)), is_theta_estim=false, sigma2=LK.sigma2(k), is_sigma2_estim=false) -check_close!(s, "Kriging: update!(refit=false) == fit on all points (same theta, sigma2)", - LK.predict(ku, Xt; return_stdev=false).mean, LK.predict(k35, Xt; return_stdev=false).mean; - rtol=1e-6, atol=1e-8 * sd(y)) -roundtrip!(s, "Kriging", k, Xt) - -# %% [markdown] -# ## `Kriging` with noise: estimated nugget, known variances -# -# Noise of standard deviation 5% of sd(y). The nugget model is fitted with 10 -# restarts (`optim="BFGS10"`); the default single start is compared with it -# (warn-level: a lower likelihood means a local optimum). As DiceKriging's -# nugget, the predictor returns y exactly *at* the design points (the nugget is -# a micro-scale variation): smoothing is checked next to them. - -# %% -sd_noise = 0.05 * sd(y) -yn = y .+ sd_noise .* randn(rng, length(y)) -kn = LK.Kriging(yn, X, "matern5_2"; noise="nugget", optim="BFGS10") -mn = LK.predict(kn, Xt; return_stdev=false).mean -add_metric!(s, "branin noisy", "Kriging nugget BFGS10"; q2=q2(yt, mn)) -add_check!(s, "Kriging(nugget): estimated nugget > 0", LK.nugget(kn) > 0; value=LK.nugget(kn), expected="> 0") -add_check!(s, "Kriging(nugget): Q2", q2(yt, mn) > 0.95; value=q2(yt, mn), expected="> 0.95") -r = sqrt(sum((LK.predict(kn, min.(X .+ 1e-7, 1.0); return_stdev=false).mean .- yn) .^ 2) / length(yn)) / sd_noise -add_check!(s, "Kriging(nugget): smooths the noisy data next to the design points (rms residual / noise sd)", r > 0.3; - value=r, expected="> 0.3") -kn1 = LK.Kriging(yn, X, "matern5_2"; noise="nugget") -add_metric!(s, "branin noisy", "Kriging nugget BFGS"; q2=q2(yt, LK.predict(kn1, Xt; return_stdev=false).mean)) -dll = LK.log_likelihood(kn1) - LK.log_likelihood(kn) -add_check!(s, "Kriging(nugget): LL(single start) - LL(10 restarts)", dll > -1; value=dll, expected="> -1", severity="warn") -roundtrip!(s, "Kriging(nugget)", kn, Xt) -v = fill(sd_noise^2, length(yn)) -kh = LK.Kriging(yn, X, "matern5_2"; noise=v) -check_close!(s, "Kriging(noise = variances): noise() returns them", LK.noise(kh), v; rtol=1e-12) -ph = LK.predict(kh, Xt; return_stdev=true) -add_check!(s, "Kriging(noise = variances): finite predictions", finite_count(ph.mean, ph.stdev) == 0; - value=finite_count(ph.mean, ph.stdev), expected="0 non-finite") -add_metric!(s, "branin noisy", "Kriging known noise"; q2=q2(yt, ph.mean)) - -# %% [markdown] -# ## `Kriging` with an approximate fit: Vecchia, Nyström (n = 400) - -# %% -Xl = lhs(400, 2); yl = branin(Xl) -for m in ("vecchia(20)", "nystrom(60)") - short = split(m, "(")[1] - label = "Kriging(method = '$m')" - check_runs!(s, "$label: fit + predict") do - ka = LK.Kriging(yl, Xl, "matern5_2"; method=m) - add_check!(s, "$label: method()", LK.method(ka) == m; value=LK.method(ka), expected=m) - pa = LK.predict(ka, Xt; return_stdev=true, method=String(short)) - add_check!(s, "$label: finite predictions", finite_count(pa.mean, pa.stdev) == 0; - value=finite_count(pa.mean, pa.stdev), expected="0 non-finite") - add_metric!(s, "branin n=400", "Kriging $m"; q2=q2(yt, pa.mean)) - add_check!(s, "$label: Q2", q2(yt, pa.mean) > 0.95; value=q2(yt, pa.mean), expected="> 0.95", severity="warn") - end -end - -# %% [markdown] -# ## `WarpKriging`: continuous warpings, then a categorical input -# -# The categorical case: x₂ restricted to 5 levels (coded 0, …, 4), embedded in ℝ². - -# %% -check_runs!(s, "WarpKriging(kumaraswamy, kumaraswamy): fit + predict") do - kw = LK.WarpKriging(y, X, ["kumaraswamy", "kumaraswamy"], "matern5_2") - pw = LK.predict(kw, Xt; return_stdev=true) - add_check!(s, "WarpKriging(kumaraswamy): finite predictions", finite_count(pw.mean, pw.stdev) == 0; - value=finite_count(pw.mean, pw.stdev), expected="0 non-finite") - add_metric!(s, "branin", "WarpKriging kumaraswamy"; q2=q2(yt, pw.mean)) - add_check!(s, "WarpKriging(kumaraswamy): Q2", q2(yt, pw.mean) > 0.9; value=q2(yt, pw.mean), expected="> 0.9", - severity="warn") - roundtrip!(s, "WarpKriging(kumaraswamy)", kw, Xt) -end -lev(U) = hcat(U[:, 1], round.(U[:, 2] .* 4)) -unlev(L) = hcat(L[:, 1], L[:, 2] ./ 4) -Xc = lev(lhs(40, 2)); yc = branin(unlev(Xc)) -Xtc = lev(Xt); ytc = branin(unlev(Xtc)) -check_runs!(s, "WarpKriging(none, categorical(5,2)): fit + predict") do - kc = LK.WarpKriging(yc, Xc, ["none", "categorical(5,2)"], "matern5_2") - pc = LK.predict(kc, Xtc; return_stdev=true) - add_check!(s, "WarpKriging(categorical): finite predictions", finite_count(pc.mean, pc.stdev) == 0; - value=finite_count(pc.mean, pc.stdev), expected="0 non-finite") - add_metric!(s, "branin, x2 categorical", "WarpKriging categorical"; q2=q2(ytc, pc.mean)) - add_check!(s, "WarpKriging(categorical): Q2", q2(ytc, pc.mean) > 0.8; value=q2(ytc, pc.mean), expected="> 0.8", - severity="warn") - roundtrip!(s, "WarpKriging(categorical)", kc, Xtc) -end - -# %% [markdown] -# ## `MLPKriging`: a joint feature map ℝ² → ℝ² before the kernel - -# %% -check_runs!(s, "MLPKriging: fit + predict") do - km = LK.MLPKriging(y, X, [8], 2; activation="tanh", kernel="matern5_2", optim="BFGS10") - pm = LK.predict(km, Xt; return_stdev=true) - add_check!(s, "MLPKriging: finite predictions", finite_count(pm.mean, pm.stdev) == 0; - value=finite_count(pm.mean, pm.stdev), expected="0 non-finite") - add_metric!(s, "branin", "MLPKriging tanh [8] -> 2, BFGS10"; q2=q2(yt, pm.mean)) - add_check!(s, "MLPKriging: Q2", q2(yt, pm.mean) > 0.9; value=q2(yt, pm.mean), expected="> 0.9", severity="warn") - roundtrip!(s, "MLPKriging", km, Xt) - kd = LK.MLPKriging(y, X, [8], 2; activation="tanh", kernel="matern5_2") # default optimizer - qd = q2(yt, LK.predict(kd, Xt; return_stdev=false).mean) - add_metric!(s, "branin", "MLPKriging tanh [8] -> 2, default optim"; q2=qd) - add_check!(s, "MLPKriging: Q2(default optim) - Q2(BFGS10)", qd - q2(yt, pm.mean) > -0.05; - value=qd - q2(yt, pm.mean), expected="> -0.05", severity="warn") -end - -# %% [markdown] -# ## `NestedKriging`: 4 groups of 100 points, `NK` and `PoE` aggregations - -# %% -for agg in ("NK", "PoE") - label = "NestedKriging($agg)" - check_runs!(s, "$label: fit + predict") do - kk = LK.NestedKriging(yl, Xl, "matern5_2", 4; aggregation=agg, seed=3) - pk = LK.predict(kk, Xt; return_stdev=true) - add_check!(s, "$label: stdev >= 0", all(pk.stdev .>= 0); value=minimum(pk.stdev), expected=">= 0") - add_metric!(s, "branin n=400", "NestedKriging $agg"; q2=q2(yt, pk.mean)) - add_check!(s, "$label: Q2", q2(yt, pk.mean) > 0.95; value=q2(yt, pk.mean), expected="> 0.95", severity="warn") - if agg == "NK" - e = maximum(abs.(LK.predict(kk, Xl; return_stdev=false).mean .- yl)) / sd(yl) - add_check!(s, "NestedKriging(NK): interpolates (max |m(X) - y| / sd(y))", e < 1e-3; value=e, expected="< 1e-3") - end - roundtrip!(s, label, kk, Xt) - end -end - -# %% -foreach(r -> r[5] == "q2" && println(rpad(r[3], 24), rpad(r[4], 28), r[6]), s.metrics) - -# %% -finalize!(s) diff --git a/tools/src/python_Kriging_kernels-trends-interpolation.py b/tools/src/kernels-trends-interpolation_Kriging_python.py similarity index 98% rename from tools/src/python_Kriging_kernels-trends-interpolation.py rename to tools/src/kernels-trends-interpolation_Kriging_python.py index f57e899..c5f295b 100644 --- a/tools/src/python_Kriging_kernels-trends-interpolation.py +++ b/tools/src/kernels-trends-interpolation_Kriging_python.py @@ -26,7 +26,7 @@ # `matern5_2` / `constant` with a multistart optimizer (`BFGS10`): a # regression shows up here first. The same floor for the default single-start # `BFGS` is warn-level: a single start can end in a poor local optimum of the -# likelihood (see `python_Kriging_objectives-LL-LOO-LMP-gradients.ipynb`), +# likelihood (see `objectives-LL-LOO-LMP-gradients_Kriging_python.ipynb`), # which is worth seeing in the report but is not a defect of the model; # * **sanity of the fitted hyperparameters**: finite, positive θ and σ²; # * **consistency**: `predict` on a batch equals `predict` point by point, the @@ -44,7 +44,7 @@ import lkbench from lkbench import TESTS, design, metrics, compat -suite = lkbench.Suite("python_Kriging_kernels-trends-interpolation") +suite = lkbench.Suite("kernels-trends-interpolation_Kriging_python") print("pylibkriging", compat.version()) KERNELS = ["gauss", "exp", "matern3_2", "matern5_2"] diff --git a/tools/src/large-n_Kriging-vecchia-nystrom_R.R b/tools/src/large-n_Kriging-vecchia-nystrom_R.R new file mode 100644 index 0000000..bd288c7 --- /dev/null +++ b/tools/src/large-n_Kriging-vecchia-nystrom_R.R @@ -0,0 +1,65 @@ +# %% [markdown] +# # R · `Kriging` with an approximate fit — Vecchia, Nyström (n = 400) +# +# Branin on [0,1]² (n = 30 LHS points unless stated, Q² on 500 uniform test +# points), through rlibkriging. Checks (error-level unless stated): +# +# * the fit method is reported by `method()` +# * `predict` with the fit method: finite mean and stdev +# * Q² > 0.95 (warn-level) + +# %% +suppressPackageStartupMessages(library(rlibkriging)) +source("tools/r/checks.R") +s <- new_suite("large-n_Kriging-vecchia-nystrom_R") +cat("rlibkriging", as.character(packageVersion("rlibkriging")), "\n") + +branin <- function(U) { # on [0,1]^2 + x1 <- 15 * U[, 1] - 5; x2 <- 15 * U[, 2] + (x2 - 5.1 / (4 * pi^2) * x1^2 + 5 / pi * x1 - 6)^2 + 10 * (1 - 1 / (8 * pi)) * cos(x1) + 10 +} +set.seed(1) +lhs <- function(n, d) sapply(1:d, function(j) (sample(n) - runif(n)) / n) +X <- lhs(30, 2); y <- branin(X) +Xt <- matrix(runif(1000), ncol = 2); yt <- branin(Xt) +tmp <- tempdir() +has_class <- function(name) exists(name, mode = "function") +skip <- function(s, check, reason) { + s$checks[[length(s$checks) + 1]] <- data.frame(suite = s$name, lang = "R", check = check, severity = "info", + status = "SKIP", value = "", expected = "", detail = reason, stringsAsFactors = FALSE) + message("[SKIP] ", check, ": ", reason) +} +roundtrip_same_mean <- function(s, label, k, Xn) { + f <- file.path(tmp, paste0(gsub("[^A-Za-z0-9]", "_", label), ".json")) + check_runs(s, paste0(label, ": save + load"), { + save(k, f) + k2 <- rlibkriging::load(f) + check_close(s, paste0(label, ": load(save(k)) predicts the same mean"), + predict(k2, Xn, return_stdev = FALSE)$mean, predict(k, Xn, return_stdev = FALSE)$mean, rtol = 1e-10) + }) +} + +# %% [markdown] +# ## `Kriging` with an approximate fit: Vecchia, Nyström (n = 400) + +# %% +Xl <- lhs(400, 2); yl <- branin(Xl) +for (m in c("vecchia(20)", "nystrom(60)")) { + short <- sub("\\(.*", "", m) + check_runs(s, paste0("Kriging(method = '", m, "'): fit + predict"), { + ka <- Kriging(yl, Xl, "matern5_2", method = m) + add_check(s, paste0("Kriging(method = '", m, "'): method()"), identical(ka$method(), m), ka$method(), m) + pa <- predict(ka, Xt, return_stdev = TRUE, method = short) + check_less(s, paste0("Kriging(method = '", m, "'): non-finite predictions"), + sum(!is.finite(c(pa$mean, pa$stdev))), 1) + add_metric(s, "branin n=400", paste("Kriging", m), q2 = q2(yt, pa$mean)) + check_greater(s, paste0("Kriging(method = '", m, "'): Q2"), q2(yt, pa$mean), 0.95, severity = "warn") + }) +} + +# %% +me <- do.call(rbind, s$metrics) +print(me[me$metric == "q2", c("case", "package", "value")], row.names = FALSE) + +# %% +finalize(s) diff --git a/tools/src/large-n_Kriging-vecchia-nystrom_julia.jl b/tools/src/large-n_Kriging-vecchia-nystrom_julia.jl new file mode 100644 index 0000000..71cfd59 --- /dev/null +++ b/tools/src/large-n_Kriging-vecchia-nystrom_julia.jl @@ -0,0 +1,71 @@ +# %% [markdown] +# # Julia · `Kriging` with an approximate fit — Vecchia, Nyström (n = 400) +# +# Branin on [0,1]² (n = 30 LHS points unless stated, Q² on 500 uniform test +# points), through JLibKriging. Checks (error-level unless stated): +# +# * the fit method is reported by `method()` +# * `predict` with the fit method: finite mean and stdev +# * Q² > 0.95 (warn-level) +# +# The same checks as `large-n_Kriging-vecchia-nystrom_R.ipynb`, with the Julia binding. + +# %% +import JLibKriging as LK +using Random +include("tools/julia/checks.jl") +s = Suite("large-n_Kriging-vecchia-nystrom_julia") + +branin(U) = begin + x1 = 15 .* U[:, 1] .- 5; x2 = 15 .* U[:, 2] + (x2 .- 5.1 / (4π^2) .* x1 .^ 2 .+ 5 / π .* x1 .- 6) .^ 2 .+ 10 * (1 - 1 / (8π)) .* cos.(x1) .+ 10 +end +rng = MersenneTwister(1) +lhs(n, d) = hcat([(randperm(rng, n) .- rand(rng, n)) ./ n for _ in 1:d]...) +X = lhs(30, 2); y = branin(X) +Xt = rand(rng, 500, 2); yt = branin(Xt) +sd(v) = sqrt(sum((v .- sum(v) / length(v)) .^ 2) / (length(v) - 1)) +finite_count(v...) = count(!isfinite, vcat(v...)) + +function check_runs!(f, s, check) + try + f() + catch e + add_check!(s, check, false; value=string(typeof(e)), expected="no error", detail=first(sprint(showerror, e), 200)) + end +end + +function roundtrip!(s, label, k, Xn) + check_runs!(s, "$label: save + load") do + f = joinpath(mktempdir(), "k.json") + LK.save(k, f) + k2 = LK.load(f) + check_close!(s, "$label: load(save(k)) predicts the same mean", + LK.predict(k2, Xn; return_stdev=false).mean, LK.predict(k, Xn; return_stdev=false).mean; rtol=1e-10) + end +end + +# %% [markdown] +# ## `Kriging` with an approximate fit: Vecchia, Nyström (n = 400) + +# %% +Xl = lhs(400, 2); yl = branin(Xl) +for m in ("vecchia(20)", "nystrom(60)") + short = split(m, "(")[1] + label = "Kriging(method = '$m')" + check_runs!(s, "$label: fit + predict") do + ka = LK.Kriging(yl, Xl, "matern5_2"; method=m) + add_check!(s, "$label: method()", LK.method(ka) == m; value=LK.method(ka), expected=m) + pa = LK.predict(ka, Xt; return_stdev=true, method=String(short)) + add_check!(s, "$label: finite predictions", finite_count(pa.mean, pa.stdev) == 0; + value=finite_count(pa.mean, pa.stdev), expected="0 non-finite") + add_metric!(s, "branin n=400", "Kriging $m"; q2=q2(yt, pa.mean)) + add_check!(s, "$label: Q2", q2(yt, pa.mean) > 0.95; value=q2(yt, pa.mean), expected="> 0.95", severity="warn") + end +end + +# %% +foreach(r -> r[5] == "q2" && println(rpad(r[3], 24), rpad(r[4], 28), r[6]), s.metrics) + +# %% +finalize!(s) diff --git a/tools/src/large-n_Kriging-vecchia-nystrom_octave.m b/tools/src/large-n_Kriging-vecchia-nystrom_octave.m new file mode 100644 index 0000000..b3c4521 --- /dev/null +++ b/tools/src/large-n_Kriging-vecchia-nystrom_octave.m @@ -0,0 +1,73 @@ +% %% [markdown] +% # Octave · `Kriging` with an approximate fit — Vecchia, Nyström (n = 400) +% +% Branin on [0,1]² (n = 30 LHS points unless stated, Q² on 500 uniform test +% points), through mLibKriging. Checks (error-level unless stated): +% +% * the fit method is reported by `method()` +% * `predict` with the fit method: finite mean and stdev +% * Q² > 0.95 (warn-level) +% +% The same checks as `large-n_Kriging-vecchia-nystrom_R.ipynb`, with the Octave binding. + +% %% +addpath(fullfile(pwd, "tools", "octave")); +s = lkb_suite("large-n_Kriging-vecchia-nystrom_octave"); +q2 = @(y, m) 1 - sum((y - m).^2) / sum((y - mean(y)).^2); +branin = @(U) ((15 * U(:, 2)) - 5.1 / (4 * pi^2) * (15 * U(:, 1) - 5).^2 + 5 / pi * (15 * U(:, 1) - 5) - 6).^2 ... + + 10 * (1 - 1 / (8 * pi)) * cos(15 * U(:, 1) - 5) + 10; +rand("seed", 1); randn("seed", 1); +function U = lhs(n, d) + U = zeros(n, d); + for j = 1:d, U(:, j) = (randperm(n)' - rand(n, 1)) / n; end +end +X = lhs(30, 2); y = branin(X); +Xt = rand(500, 2); yt = branin(Xt); +tmp = tempname(); mkdir(tmp); +has_class = @(name) exist(name) == 2; % a class file on the path + +function s = roundtrip(s, label, k, Xn, loader) + f = fullfile(tempdir(), [regexprep(label, "[^A-Za-z0-9]", "_") ".json"]); + try + k.save(f); + k2 = loader(f); + m1 = k.predict(Xn, false, false, false); + m2 = k2.predict(Xn, false, false, false); + s = lkb_close(s, [label ": load(save(k)) predicts the same mean"], m2, m1, 1e-10); + catch err + s = lkb_check(s, [label ": save + load"], false, "error", "no error", err.message); + end +end + +function s = skip(s, check, reason) + s.checks(end + 1, :) = {s.name, "octave", check, "info", "SKIP", "", "", reason}; + printf("[SKIP] %s: %s\n", check, reason); +end + +% %% [markdown] +% ## `Kriging` with an approximate fit: Vecchia, Nyström (n = 400) + +% %% +Xl = lhs(400, 2); yl = branin(Xl); +for mth = {"vecchia(20)", "nystrom(60)"} + mth = mth{1}; short = regexprep(mth, "\\(.*", ""); + label = ["Kriging(method = '" mth "')"]; + try + ka = Kriging(yl, Xl, "matern5_2", "constant", false, "BFGS", "LL", [], "none", [], mth); + got = ka.method(); + s = lkb_check(s, [label ": method()"], strcmp(got, mth), got, mth); + [ma, sa] = ka.predict(Xt, true, false, false, short); + s = lkb_check(s, [label ": finite predictions"], all(isfinite([ma; sa])), sum(~isfinite([ma; sa])), "0 non-finite"); + s = lkb_metric(s, "branin n=400", ["Kriging " mth], "q2", q2(yt, ma)); + s = lkb_check(s, [label ": Q2"], q2(yt, ma) > 0.95, q2(yt, ma), "> 0.95", "", "warn"); + catch err + s = lkb_check(s, [label ": fit + predict"], false, "error", "no error", err.message); + end +end + +% %% +q = strcmp(s.metrics(:, 5), "q2"); +disp(s.metrics(q, [3, 4, 6])) + +% %% +lkb_finalize(s); diff --git a/tools/src/python_Kriging-vecchia-nystrom_large-n.py b/tools/src/large-n_Kriging-vecchia-nystrom_python.py similarity index 97% rename from tools/src/python_Kriging-vecchia-nystrom_large-n.py rename to tools/src/large-n_Kriging-vecchia-nystrom_python.py index 6759b35..f5e0b17 100644 --- a/tools/src/python_Kriging-vecchia-nystrom_large-n.py +++ b/tools/src/large-n_Kriging-vecchia-nystrom_python.py @@ -10,7 +10,7 @@ # * `Kriging.subsetOfData`: a representative subset before an exact fit. # # (`NestedKriging`, the partition-based alternative, on the same problems: -# `python_NestedKriging_aggregations-large-n.ipynb`.) +# `aggregations-large-n_NestedKriging_python.ipynb`.) # # Problems: Hartmann 3 (3D, local structure: Vecchia's ground) and the OTL # circuit (6D, smooth: Nyström's ground), n = 1500 (exact fit still affordable, @@ -33,7 +33,7 @@ import lkbench from lkbench import TESTS, design, metrics, compat -suite = lkbench.Suite("python_Kriging-vecchia-nystrom_large-n") +suite = lkbench.Suite("large-n_Kriging-vecchia-nystrom_python") print("pylibkriging", compat.version()) HAS_METHOD = compat.has("method") if not HAS_METHOD: diff --git a/tools/src/noisy-data_Kriging-nugget-noise_R.R b/tools/src/noisy-data_Kriging-nugget-noise_R.R new file mode 100644 index 0000000..be4a553 --- /dev/null +++ b/tools/src/noisy-data_Kriging-nugget-noise_R.R @@ -0,0 +1,82 @@ +# %% [markdown] +# # R · `Kriging` with noise — estimated nugget, known variances +# +# Branin on [0,1]² (n = 30 LHS points unless stated, Q² on 500 uniform test +# points), through rlibkriging. Checks (error-level unless stated): +# +# * `noise = "nugget"` (10 restarts): nugget > 0, Q² > 0.95 on the noise-free function, smoothing next to the design points +# * the default single start reaches the multistart likelihood (warn-level) +# * `save` then `load`: the same predicted mean +# * `noise = `: the stored variances, finite predictions + +# %% +suppressPackageStartupMessages(library(rlibkriging)) +source("tools/r/checks.R") +s <- new_suite("noisy-data_Kriging-nugget-noise_R") +cat("rlibkriging", as.character(packageVersion("rlibkriging")), "\n") + +branin <- function(U) { # on [0,1]^2 + x1 <- 15 * U[, 1] - 5; x2 <- 15 * U[, 2] + (x2 - 5.1 / (4 * pi^2) * x1^2 + 5 / pi * x1 - 6)^2 + 10 * (1 - 1 / (8 * pi)) * cos(x1) + 10 +} +set.seed(1) +lhs <- function(n, d) sapply(1:d, function(j) (sample(n) - runif(n)) / n) +X <- lhs(30, 2); y <- branin(X) +Xt <- matrix(runif(1000), ncol = 2); yt <- branin(Xt) +tmp <- tempdir() +has_class <- function(name) exists(name, mode = "function") +skip <- function(s, check, reason) { + s$checks[[length(s$checks) + 1]] <- data.frame(suite = s$name, lang = "R", check = check, severity = "info", + status = "SKIP", value = "", expected = "", detail = reason, stringsAsFactors = FALSE) + message("[SKIP] ", check, ": ", reason) +} +roundtrip_same_mean <- function(s, label, k, Xn) { + f <- file.path(tmp, paste0(gsub("[^A-Za-z0-9]", "_", label), ".json")) + check_runs(s, paste0(label, ": save + load"), { + save(k, f) + k2 <- rlibkriging::load(f) + check_close(s, paste0(label, ": load(save(k)) predicts the same mean"), + predict(k2, Xn, return_stdev = FALSE)$mean, predict(k, Xn, return_stdev = FALSE)$mean, rtol = 1e-10) + }) +} + +# %% [markdown] +# ## `Kriging` with noise: estimated nugget, known variances +# +# Noise of standard deviation 5% of sd(y). The nugget model is fitted with 10 +# restarts (`optim = "BFGS10"`); the default single start is compared with it +# (warn-level: a lower likelihood means a local optimum). As DiceKriging's +# nugget, the predictor returns y exactly *at* the design points (the nugget is +# a micro-scale variation): smoothing is checked next to them. + +# %% +sd_noise <- 0.05 * sd(y) +yn <- y + rnorm(length(y), sd = sd_noise) +kn <- Kriging(yn, X, "matern5_2", noise = "nugget", optim = "BFGS10") +pn <- predict(kn, Xt, return_stdev = TRUE) +add_metric(s, "branin noisy", "Kriging nugget BFGS10", q2 = q2(yt, pn$mean), nugget = kn$nugget(), + ll = logLikelihood(kn)) +check_greater(s, "Kriging(nugget): estimated nugget > 0", kn$nugget(), 0) +check_greater(s, "Kriging(nugget): Q2", q2(yt, pn$mean), 0.95) +near <- pmin(X + 1e-7, 1) +check_greater(s, "Kriging(nugget): smooths the noisy data next to the design points (rms residual / noise sd)", + rmse(yn, predict(kn, near, return_stdev = FALSE)$mean) / sd_noise, 0.3) +kn1 <- Kriging(yn, X, "matern5_2", noise = "nugget") +add_metric(s, "branin noisy", "Kriging nugget BFGS", q2 = q2(yt, predict(kn1, Xt, return_stdev = FALSE)$mean), + nugget = kn1$nugget(), ll = logLikelihood(kn1)) +check_greater(s, "Kriging(nugget): LL(single start) - LL(10 restarts)", logLikelihood(kn1) - logLikelihood(kn), -1, + severity = "warn") +roundtrip_same_mean(s, "Kriging(nugget)", kn, Xt) +kh <- Kriging(yn, X, "matern5_2", noise = rep(sd_noise^2, length(yn))) +check_close(s, "Kriging(noise = variances): noise() returns them", kh$noise(), rep(sd_noise^2, length(yn)), + rtol = 1e-12) +ph <- predict(kh, Xt, return_stdev = TRUE) +check_less(s, "Kriging(noise = variances): non-finite predictions", sum(!is.finite(c(ph$mean, ph$stdev))), 1) +add_metric(s, "branin noisy", "Kriging known noise", q2 = q2(yt, ph$mean)) + +# %% +me <- do.call(rbind, s$metrics) +print(me[me$metric == "q2", c("case", "package", "value")], row.names = FALSE) + +# %% +finalize(s) diff --git a/tools/src/noisy-data_Kriging-nugget-noise_julia.jl b/tools/src/noisy-data_Kriging-nugget-noise_julia.jl new file mode 100644 index 0000000..ad25148 --- /dev/null +++ b/tools/src/noisy-data_Kriging-nugget-noise_julia.jl @@ -0,0 +1,86 @@ +# %% [markdown] +# # Julia · `Kriging` with noise — estimated nugget, known variances +# +# Branin on [0,1]² (n = 30 LHS points unless stated, Q² on 500 uniform test +# points), through JLibKriging. Checks (error-level unless stated): +# +# * `noise = "nugget"` (10 restarts): nugget > 0, Q² > 0.95 on the noise-free function, smoothing next to the design points +# * the default single start reaches the multistart likelihood (warn-level) +# * `save` then `load`: the same predicted mean +# * `noise = `: the stored variances, finite predictions +# +# The same checks as `noisy-data_Kriging-nugget-noise_R.ipynb`, with the Julia binding. + +# %% +import JLibKriging as LK +using Random +include("tools/julia/checks.jl") +s = Suite("noisy-data_Kriging-nugget-noise_julia") + +branin(U) = begin + x1 = 15 .* U[:, 1] .- 5; x2 = 15 .* U[:, 2] + (x2 .- 5.1 / (4π^2) .* x1 .^ 2 .+ 5 / π .* x1 .- 6) .^ 2 .+ 10 * (1 - 1 / (8π)) .* cos.(x1) .+ 10 +end +rng = MersenneTwister(1) +lhs(n, d) = hcat([(randperm(rng, n) .- rand(rng, n)) ./ n for _ in 1:d]...) +X = lhs(30, 2); y = branin(X) +Xt = rand(rng, 500, 2); yt = branin(Xt) +sd(v) = sqrt(sum((v .- sum(v) / length(v)) .^ 2) / (length(v) - 1)) +finite_count(v...) = count(!isfinite, vcat(v...)) + +function check_runs!(f, s, check) + try + f() + catch e + add_check!(s, check, false; value=string(typeof(e)), expected="no error", detail=first(sprint(showerror, e), 200)) + end +end + +function roundtrip!(s, label, k, Xn) + check_runs!(s, "$label: save + load") do + f = joinpath(mktempdir(), "k.json") + LK.save(k, f) + k2 = LK.load(f) + check_close!(s, "$label: load(save(k)) predicts the same mean", + LK.predict(k2, Xn; return_stdev=false).mean, LK.predict(k, Xn; return_stdev=false).mean; rtol=1e-10) + end +end + +# %% [markdown] +# ## `Kriging` with noise: estimated nugget, known variances +# +# Noise of standard deviation 5% of sd(y). The nugget model is fitted with 10 +# restarts (`optim="BFGS10"`); the default single start is compared with it +# (warn-level: a lower likelihood means a local optimum). As DiceKriging's +# nugget, the predictor returns y exactly *at* the design points (the nugget is +# a micro-scale variation): smoothing is checked next to them. + +# %% +sd_noise = 0.05 * sd(y) +yn = y .+ sd_noise .* randn(rng, length(y)) +kn = LK.Kriging(yn, X, "matern5_2"; noise="nugget", optim="BFGS10") +mn = LK.predict(kn, Xt; return_stdev=false).mean +add_metric!(s, "branin noisy", "Kriging nugget BFGS10"; q2=q2(yt, mn)) +add_check!(s, "Kriging(nugget): estimated nugget > 0", LK.nugget(kn) > 0; value=LK.nugget(kn), expected="> 0") +add_check!(s, "Kriging(nugget): Q2", q2(yt, mn) > 0.95; value=q2(yt, mn), expected="> 0.95") +r = sqrt(sum((LK.predict(kn, min.(X .+ 1e-7, 1.0); return_stdev=false).mean .- yn) .^ 2) / length(yn)) / sd_noise +add_check!(s, "Kriging(nugget): smooths the noisy data next to the design points (rms residual / noise sd)", r > 0.3; + value=r, expected="> 0.3") +kn1 = LK.Kriging(yn, X, "matern5_2"; noise="nugget") +add_metric!(s, "branin noisy", "Kriging nugget BFGS"; q2=q2(yt, LK.predict(kn1, Xt; return_stdev=false).mean)) +dll = LK.log_likelihood(kn1) - LK.log_likelihood(kn) +add_check!(s, "Kriging(nugget): LL(single start) - LL(10 restarts)", dll > -1; value=dll, expected="> -1", severity="warn") +roundtrip!(s, "Kriging(nugget)", kn, Xt) +v = fill(sd_noise^2, length(yn)) +kh = LK.Kriging(yn, X, "matern5_2"; noise=v) +check_close!(s, "Kriging(noise = variances): noise() returns them", LK.noise(kh), v; rtol=1e-12) +ph = LK.predict(kh, Xt; return_stdev=true) +add_check!(s, "Kriging(noise = variances): finite predictions", finite_count(ph.mean, ph.stdev) == 0; + value=finite_count(ph.mean, ph.stdev), expected="0 non-finite") +add_metric!(s, "branin noisy", "Kriging known noise"; q2=q2(yt, ph.mean)) + +# %% +foreach(r -> r[5] == "q2" && println(rpad(r[3], 24), rpad(r[4], 28), r[6]), s.metrics) + +# %% +finalize!(s) diff --git a/tools/src/noisy-data_Kriging-nugget-noise_octave.m b/tools/src/noisy-data_Kriging-nugget-noise_octave.m new file mode 100644 index 0000000..f9bd732 --- /dev/null +++ b/tools/src/noisy-data_Kriging-nugget-noise_octave.m @@ -0,0 +1,85 @@ +% %% [markdown] +% # Octave · `Kriging` with noise — estimated nugget, known variances +% +% Branin on [0,1]² (n = 30 LHS points unless stated, Q² on 500 uniform test +% points), through mLibKriging. Checks (error-level unless stated): +% +% * `noise = "nugget"` (10 restarts): nugget > 0, Q² > 0.95 on the noise-free function, smoothing next to the design points +% * the default single start reaches the multistart likelihood (warn-level) +% * `save` then `load`: the same predicted mean +% * `noise = `: the stored variances, finite predictions +% +% The same checks as `noisy-data_Kriging-nugget-noise_R.ipynb`, with the Octave binding. + +% %% +addpath(fullfile(pwd, "tools", "octave")); +s = lkb_suite("noisy-data_Kriging-nugget-noise_octave"); +q2 = @(y, m) 1 - sum((y - m).^2) / sum((y - mean(y)).^2); +branin = @(U) ((15 * U(:, 2)) - 5.1 / (4 * pi^2) * (15 * U(:, 1) - 5).^2 + 5 / pi * (15 * U(:, 1) - 5) - 6).^2 ... + + 10 * (1 - 1 / (8 * pi)) * cos(15 * U(:, 1) - 5) + 10; +rand("seed", 1); randn("seed", 1); +function U = lhs(n, d) + U = zeros(n, d); + for j = 1:d, U(:, j) = (randperm(n)' - rand(n, 1)) / n; end +end +X = lhs(30, 2); y = branin(X); +Xt = rand(500, 2); yt = branin(Xt); +tmp = tempname(); mkdir(tmp); +has_class = @(name) exist(name) == 2; % a class file on the path + +function s = roundtrip(s, label, k, Xn, loader) + f = fullfile(tempdir(), [regexprep(label, "[^A-Za-z0-9]", "_") ".json"]); + try + k.save(f); + k2 = loader(f); + m1 = k.predict(Xn, false, false, false); + m2 = k2.predict(Xn, false, false, false); + s = lkb_close(s, [label ": load(save(k)) predicts the same mean"], m2, m1, 1e-10); + catch err + s = lkb_check(s, [label ": save + load"], false, "error", "no error", err.message); + end +end + +function s = skip(s, check, reason) + s.checks(end + 1, :) = {s.name, "octave", check, "info", "SKIP", "", "", reason}; + printf("[SKIP] %s: %s\n", check, reason); +end + +% %% [markdown] +% ## `Kriging` with noise: estimated nugget, known variances +% +% Noise of standard deviation 5% of sd(y). The nugget model is fitted with 10 +% restarts (`"BFGS10"`); the default single start is compared with it +% (warn-level: a lower likelihood means a local optimum). As DiceKriging's +% nugget, the predictor returns y exactly *at* the design points (the nugget is +% a micro-scale variation): smoothing is checked next to them. + +% %% +sd_noise = 0.05 * std(y); +yn = y + sd_noise * randn(size(y)); +kn = Kriging(yn, X, "matern5_2", "constant", false, "BFGS10", "LL", [], "nugget"); +mn = kn.predict(Xt, false, false, false); +nug = kn.nugget(); ll10 = kn.logLikelihood(); +s = lkb_metric(s, "branin noisy", "Kriging nugget BFGS10", "q2", q2(yt, mn)); +s = lkb_check(s, "Kriging(nugget): estimated nugget > 0", nug > 0, nug, "> 0"); +s = lkb_check(s, "Kriging(nugget): Q2", q2(yt, mn) > 0.95, q2(yt, mn), "> 0.95"); +r = sqrt(mean((kn.predict(min(X + 1e-7, 1), false, false, false) - yn).^2)) / sd_noise; +s = lkb_check(s, "Kriging(nugget): smooths the noisy data next to the design points (rms residual / noise sd)", r > 0.3, r, "> 0.3"); +kn1 = Kriging(yn, X, "matern5_2", "constant", false, "BFGS", "LL", [], "nugget"); +ll1 = kn1.logLikelihood(); +s = lkb_metric(s, "branin noisy", "Kriging nugget BFGS", "q2", q2(yt, kn1.predict(Xt, false, false, false))); +s = lkb_check(s, "Kriging(nugget): LL(single start) - LL(10 restarts)", ll1 - ll10 > -1, ll1 - ll10, "> -1", "", "warn"); +s = roundtrip(s, "Kriging(nugget)", kn, Xt, @load_kriging); +v = sd_noise^2 * ones(size(yn)); +kh = Kriging(yn, X, "matern5_2", "constant", false, "BFGS", "LL", [], "heterogeneous", v); +s = lkb_close(s, "Kriging(noise = variances): noise() returns them", kh.noise(), v, 1e-12); +[mh, sh] = kh.predict(Xt, true, false, false); +s = lkb_check(s, "Kriging(noise = variances): finite predictions", all(isfinite([mh; sh])), sum(~isfinite([mh; sh])), "0 non-finite"); +s = lkb_metric(s, "branin noisy", "Kriging known noise", "q2", q2(yt, mh)); + +% %% +q = strcmp(s.metrics(:, 5), "q2"); +disp(s.metrics(q, [3, 4, 6])) + +% %% +lkb_finalize(s); diff --git a/tools/src/python_Kriging-nugget-noise_noisy-data.py b/tools/src/noisy-data_Kriging-nugget-noise_python.py similarity index 99% rename from tools/src/python_Kriging-nugget-noise_noisy-data.py rename to tools/src/noisy-data_Kriging-nugget-noise_python.py index e5497d6..48aaff1 100644 --- a/tools/src/python_Kriging-nugget-noise_noisy-data.py +++ b/tools/src/noisy-data_Kriging-nugget-noise_python.py @@ -35,7 +35,7 @@ import lkbench from lkbench import TESTS, design, metrics, compat -suite = lkbench.Suite("python_Kriging-nugget-noise_noisy-data") +suite = lkbench.Suite("noisy-data_Kriging-nugget-noise_python") print("pylibkriging", compat.version()) diff --git a/tools/src/python_Kriging_objectives-LL-LOO-LMP-gradients.py b/tools/src/objectives-LL-LOO-LMP-gradients_Kriging_python.py similarity index 99% rename from tools/src/python_Kriging_objectives-LL-LOO-LMP-gradients.py rename to tools/src/objectives-LL-LOO-LMP-gradients_Kriging_python.py index 38c8af4..4a986d2 100644 --- a/tools/src/python_Kriging_objectives-LL-LOO-LMP-gradients.py +++ b/tools/src/objectives-LL-LOO-LMP-gradients_Kriging_python.py @@ -32,7 +32,7 @@ import lkbench from lkbench import TESTS, design, metrics, compat -suite = lkbench.Suite("python_Kriging_objectives-LL-LOO-LMP-gradients") +suite = lkbench.Suite("objectives-LL-LOO-LMP-gradients_Kriging_python") print("pylibkriging", compat.version()) CASES = {"branin": 25, "hartmann3": 40, "ishigami": 60, "friedman": 50} KERNELS = ["gauss", "exp", "matern3_2", "matern5_2"] diff --git a/tools/src/octave_all-classes_api-tour.m b/tools/src/octave_all-classes_api-tour.m deleted file mode 100644 index fa88339..0000000 --- a/tools/src/octave_all-classes_api-tour.m +++ /dev/null @@ -1,213 +0,0 @@ -% %% [markdown] -% # Octave · all classes — every libKriging model class and fit method through mLibKriging -% -% Same tour as `R_all-classes_api-tour.ipynb` (see there for the checks): Branin -% on [0,1]², every class and fit method of the Octave binding (positional -% arguments), each with the checks that hold for it whatever the data. A class -% missing from the installed mLibKriging is recorded as SKIP. mLibKriging must -% be on the path (`OCTAVE_PATH` or `addpath`), see `tools/install/octave.sh`. - -% %% -addpath(fullfile(pwd, "tools", "octave")); -s = lkb_suite("octave_all-classes_api-tour"); -q2 = @(y, m) 1 - sum((y - m).^2) / sum((y - mean(y)).^2); -branin = @(U) ((15 * U(:, 2)) - 5.1 / (4 * pi^2) * (15 * U(:, 1) - 5).^2 + 5 / pi * (15 * U(:, 1) - 5) - 6).^2 ... - + 10 * (1 - 1 / (8 * pi)) * cos(15 * U(:, 1) - 5) + 10; -rand("seed", 1); randn("seed", 1); -function U = lhs(n, d) - U = zeros(n, d); - for j = 1:d, U(:, j) = (randperm(n)' - rand(n, 1)) / n; end -end -X = lhs(30, 2); y = branin(X); -Xt = rand(500, 2); yt = branin(Xt); -tmp = tempname(); mkdir(tmp); -has_class = @(name) exist(name) == 2; % a class file on the path - -function s = roundtrip(s, label, k, Xn, loader) - f = fullfile(tempdir(), [regexprep(label, "[^A-Za-z0-9]", "_") ".json"]); - try - k.save(f); - k2 = loader(f); - m1 = k.predict(Xn, false, false, false); - m2 = k2.predict(Xn, false, false, false); - s = lkb_close(s, [label ": load(save(k)) predicts the same mean"], m2, m1, 1e-10); - catch err - s = lkb_check(s, [label ": save + load"], false, "error", "no error", err.message); - end -end - -function s = skip(s, check, reason) - s.checks(end + 1, :) = {s.name, "octave", check, "info", "SKIP", "", "", reason}; - printf("[SKIP] %s: %s\n", check, reason); -end - -% %% [markdown] -% ## `Kriging`, noise-free - -% %% -k = Kriging(y, X, "matern5_2"); -[m, sd] = k.predict(Xt, true, false, false); -s = lkb_metric(s, "branin", "Kriging", "q2", q2(yt, m)); -s = lkb_check(s, "Kriging: Q2", q2(yt, m) > 0.95, q2(yt, m), "> 0.95"); -mX = k.predict(X, false, false, false); -e = max(abs(mX - y)) / std(y); -s = lkb_check(s, "Kriging: interpolates (max |m(X) - y| / sd(y))", e < 1e-6, e, "< 1e-6"); -S = k.simulate(int32(5), int32(2), X, false); -e = max(max(abs(S - y))) / sqrt(k.sigma2()); -s = lkb_check(s, "Kriging: conditional simulations go through the data (max |sim(X) - y| / sigma)", e < 1e-4, e, "< 1e-4"); -s = lkb_check(s, "Kriging: stdev > 0 off the design", min(sd) > 0, min(sd), "> 0"); -Xu = lhs(5, 2); yu = branin(Xu); -ku = k.copy(); -ku.update(yu, Xu, false); -th = k.theta(); -k35 = Kriging([y; yu], [X; Xu], "matern5_2", "constant", false, "none", "LL", ... - Params("theta", th(:)', "is_theta_estim", false, "sigma2", k.sigma2(), "is_sigma2_estim", false)); -s = lkb_close(s, "Kriging: update(refit = false) == fit on all points (same theta, sigma2)", ... - ku.predict(Xt, false, false, false), k35.predict(Xt, false, false, false), 1e-6, 1e-8 * std(y)); -s = roundtrip(s, "Kriging", k, Xt, @load_kriging); - -% %% [markdown] -% ## `Kriging` with noise: estimated nugget, known variances -% -% Noise of standard deviation 5% of sd(y). The nugget model is fitted with 10 -% restarts (`"BFGS10"`); the default single start is compared with it -% (warn-level: a lower likelihood means a local optimum). As DiceKriging's -% nugget, the predictor returns y exactly *at* the design points (the nugget is -% a micro-scale variation): smoothing is checked next to them. - -% %% -sd_noise = 0.05 * std(y); -yn = y + sd_noise * randn(size(y)); -kn = Kriging(yn, X, "matern5_2", "constant", false, "BFGS10", "LL", [], "nugget"); -mn = kn.predict(Xt, false, false, false); -nug = kn.nugget(); ll10 = kn.logLikelihood(); -s = lkb_metric(s, "branin noisy", "Kriging nugget BFGS10", "q2", q2(yt, mn)); -s = lkb_check(s, "Kriging(nugget): estimated nugget > 0", nug > 0, nug, "> 0"); -s = lkb_check(s, "Kriging(nugget): Q2", q2(yt, mn) > 0.95, q2(yt, mn), "> 0.95"); -r = sqrt(mean((kn.predict(min(X + 1e-7, 1), false, false, false) - yn).^2)) / sd_noise; -s = lkb_check(s, "Kriging(nugget): smooths the noisy data next to the design points (rms residual / noise sd)", r > 0.3, r, "> 0.3"); -kn1 = Kriging(yn, X, "matern5_2", "constant", false, "BFGS", "LL", [], "nugget"); -ll1 = kn1.logLikelihood(); -s = lkb_metric(s, "branin noisy", "Kriging nugget BFGS", "q2", q2(yt, kn1.predict(Xt, false, false, false))); -s = lkb_check(s, "Kriging(nugget): LL(single start) - LL(10 restarts)", ll1 - ll10 > -1, ll1 - ll10, "> -1", "", "warn"); -s = roundtrip(s, "Kriging(nugget)", kn, Xt, @load_kriging); -v = sd_noise^2 * ones(size(yn)); -kh = Kriging(yn, X, "matern5_2", "constant", false, "BFGS", "LL", [], "heterogeneous", v); -s = lkb_close(s, "Kriging(noise = variances): noise() returns them", kh.noise(), v, 1e-12); -[mh, sh] = kh.predict(Xt, true, false, false); -s = lkb_check(s, "Kriging(noise = variances): finite predictions", all(isfinite([mh; sh])), sum(~isfinite([mh; sh])), "0 non-finite"); -s = lkb_metric(s, "branin noisy", "Kriging known noise", "q2", q2(yt, mh)); - -% %% [markdown] -% ## `Kriging` with an approximate fit: Vecchia, Nyström (n = 400) - -% %% -Xl = lhs(400, 2); yl = branin(Xl); -for mth = {"vecchia(20)", "nystrom(60)"} - mth = mth{1}; short = regexprep(mth, "\\(.*", ""); - label = ["Kriging(method = '" mth "')"]; - try - ka = Kriging(yl, Xl, "matern5_2", "constant", false, "BFGS", "LL", [], "none", [], mth); - got = ka.method(); - s = lkb_check(s, [label ": method()"], strcmp(got, mth), got, mth); - [ma, sa] = ka.predict(Xt, true, false, false, short); - s = lkb_check(s, [label ": finite predictions"], all(isfinite([ma; sa])), sum(~isfinite([ma; sa])), "0 non-finite"); - s = lkb_metric(s, "branin n=400", ["Kriging " mth], "q2", q2(yt, ma)); - s = lkb_check(s, [label ": Q2"], q2(yt, ma) > 0.95, q2(yt, ma), "> 0.95", "", "warn"); - catch err - s = lkb_check(s, [label ": fit + predict"], false, "error", "no error", err.message); - end -end - -% %% [markdown] -% ## `WarpKriging`: continuous warpings, then a categorical input -% -% The categorical case: x₂ restricted to 5 levels (coded 0, …, 4), embedded in ℝ². - -% %% -if has_class("WarpKriging") - try - kw = WarpKriging(y, X, {"kumaraswamy", "kumaraswamy"}, "matern5_2"); - [mw, sw] = kw.predict(Xt, true, false, false); - s = lkb_check(s, "WarpKriging(kumaraswamy): finite predictions", all(isfinite([mw; sw])), sum(~isfinite([mw; sw])), "0 non-finite"); - s = lkb_metric(s, "branin", "WarpKriging kumaraswamy", "q2", q2(yt, mw)); - s = lkb_check(s, "WarpKriging(kumaraswamy): Q2", q2(yt, mw) > 0.9, q2(yt, mw), "> 0.9", "", "warn"); - s = roundtrip(s, "WarpKriging(kumaraswamy)", kw, Xt, @load_kriging); - catch err - s = lkb_check(s, "WarpKriging(kumaraswamy): fit + predict", false, "error", "no error", err.message); - end - lev = @(U) [U(:, 1), round(U(:, 2) * 4)]; - unlev = @(L) [L(:, 1), L(:, 2) / 4]; - Xc = lev(lhs(40, 2)); yc = branin(unlev(Xc)); - Xtc = lev(Xt); ytc = branin(unlev(Xtc)); - try - kc = WarpKriging(yc, Xc, {"none", "categorical(5,2)"}, "matern5_2"); - [mc, sc] = kc.predict(Xtc, true, false, false); - s = lkb_check(s, "WarpKriging(categorical): finite predictions", all(isfinite([mc; sc])), sum(~isfinite([mc; sc])), "0 non-finite"); - s = lkb_metric(s, "branin, x2 categorical", "WarpKriging categorical", "q2", q2(ytc, mc)); - s = lkb_check(s, "WarpKriging(categorical): Q2", q2(ytc, mc) > 0.8, q2(ytc, mc), "> 0.8", "", "warn"); - s = roundtrip(s, "WarpKriging(categorical)", kc, Xtc, @load_kriging); - catch err - s = lkb_check(s, "WarpKriging(categorical): fit + predict", false, "error", "no error", err.message); - end -else - s = skip(s, "WarpKriging", "not in this mLibKriging"); -end - -% %% [markdown] -% ## `MLPKriging`: a joint feature map ℝ² → ℝ² before the kernel - -% %% -if has_class("MLPKriging") - try - km = MLPKriging(y, X, [8], 2, "tanh", "matern5_2", "constant", false, "BFGS10"); - [mm, sm] = km.predict(Xt, true, false, false); - s = lkb_check(s, "MLPKriging: finite predictions", all(isfinite([mm; sm])), sum(~isfinite([mm; sm])), "0 non-finite"); - s = lkb_metric(s, "branin", "MLPKriging tanh [8] -> 2, BFGS10", "q2", q2(yt, mm)); - s = lkb_check(s, "MLPKriging: Q2", q2(yt, mm) > 0.9, q2(yt, mm), "> 0.9", "", "warn"); - s = roundtrip(s, "MLPKriging", km, Xt, @load_kriging); - kd = MLPKriging(y, X, [8], 2, "tanh", "matern5_2"); % default optimizer - qd = q2(yt, kd.predict(Xt, false, false, false)); - s = lkb_metric(s, "branin", "MLPKriging tanh [8] -> 2, default optim", "q2", qd); - s = lkb_check(s, "MLPKriging: Q2(default optim) - Q2(BFGS10)", qd - q2(yt, mm) > -0.05, qd - q2(yt, mm), "> -0.05", "", "warn"); - catch err - s = lkb_check(s, "MLPKriging: fit + predict", false, "error", "no error", err.message); - end -else - s = skip(s, "MLPKriging", "not in this mLibKriging"); -end - -% %% [markdown] -% ## `NestedKriging`: 4 groups of 100 points, `NK` and `PoE` aggregations - -% %% -if has_class("NestedKriging") - for agg = {"NK", "PoE"} - agg = agg{1}; label = ["NestedKriging(" agg ")"]; - try - kk = NestedKriging(yl, Xl, "matern5_2", 4, agg, "kmeans", 3); - [mk, sk] = kk.predict(Xt); % stdev returned when asked for (2 outputs) - s = lkb_check(s, [label ": stdev >= 0"], all(sk >= 0), min(sk), ">= 0"); - s = lkb_metric(s, "branin n=400", ["NestedKriging " agg], "q2", q2(yt, mk)); - s = lkb_check(s, [label ": Q2"], q2(yt, mk) > 0.95, q2(yt, mk), "> 0.95", "", "warn"); - if strcmp(agg, "NK") - e = max(abs(kk.predict(Xl) - yl)) / std(yl); - s = lkb_check(s, "NestedKriging(NK): interpolates (max |m(X) - y| / sd(y))", e < 1e-3, e, "< 1e-3"); - end - f = fullfile(tmp, ["nested_" agg ".json"]); - kk.save(f); k2 = load_kriging(f); - s = lkb_close(s, [label ": load(save(k)) predicts the same mean"], k2.predict(Xt), kk.predict(Xt), 1e-10); - catch err - s = lkb_check(s, [label ": fit + predict"], false, "error", "no error", err.message); - end - end -else - s = skip(s, "NestedKriging", "not in this mLibKriging"); -end - -% %% -q = strcmp(s.metrics(:, 5), "q2"); -disp(s.metrics(q, [3, 4, 6])) - -% %% -lkb_finalize(s); diff --git a/tools/src/python_all-classes_save-load-pickle-sklearn.py b/tools/src/python_all-classes_save-load-pickle-sklearn.py deleted file mode 100644 index ea2533a..0000000 --- a/tools/src/python_all-classes_save-load-pickle-sklearn.py +++ /dev/null @@ -1,200 +0,0 @@ -# %% [markdown] -# # Python · all classes — save/load, pickle, copy, accessors, scikit-learn estimators -# -# Persistence and introspection, for every model class: -# -# * `save` → `load` (class-specific and generic `pylibkriging.load`) gives a model -# with **identical** predictions and hyperparameters; -# * `pickle` / `copy` round trips, and a copy is independent of the original -# (updating one leaves the other untouched); -# * accessors mirror the constructor arguments (`kernel()`, `regmodel()`, -# `optim()`, `objective()`, `X()`, `y()`, ...); -# * scikit-learn estimators (`pylibkriging.sklearn`): `fit`/`predict`, -# `get_params`/`set_params`/`clone`, use inside `Pipeline` and `GridSearchCV`, -# same predictions as the native class. - -# %% -import copy -import pickle -import tempfile -from pathlib import Path - -import numpy as np -import pylibkriging as lk - -import lkbench -from lkbench import TESTS, design, compat - -suite = lkbench.Suite("python_all-classes_save-load-pickle-sklearn") -print("pylibkriging", compat.version()) -tmp = Path(tempfile.mkdtemp()) - -tf = TESTS["hartmann3"] -X, y, Xt, yt = design.dataset(tf, 40, n_test=100, seed=61) - - -def mean_sd(k, Xn): - out = k.predict(Xn, return_stdev=True) - return np.ravel(out[0]), np.ravel(out[1]) - - -MODELS = { - "Kriging": lambda: lk.Kriging(y, X, "matern5_2", regmodel="linear"), - "Kriging nugget": lambda: lk.Kriging(y, X, "matern3_2", noise="nugget"), - "Kriging noise vector": lambda: lk.Kriging(y, X, "gauss", noise=np.full(len(y), 1e-4)), - "Kriging LOO": lambda: lk.Kriging(y, X, "matern5_2", objective="LOO"), - "Kriging LMP": lambda: lk.Kriging(y, X, "matern5_2", objective="LMP"), -} -if compat.has("method"): - MODELS["Kriging vecchia"] = lambda: lk.Kriging(y, X, "matern5_2", method="vecchia(10)") - MODELS["Kriging nystrom"] = lambda: lk.Kriging(y, X, "matern5_2", method="nystrom(20)") -if compat.has("WarpKriging"): - MODELS["WarpKriging"] = lambda: lk.WarpKriging(y, X, ["kumaraswamy", "none", "affine"], "matern5_2") -if compat.has("MLPKriging"): - MODELS["MLPKriging"] = lambda: lk.MLPKriging(y, X, hidden_dims=[8], d_out=2, activation="tanh", - kernel="matern5_2") -if compat.has("NestedKriging"): - MODELS["NestedKriging"] = lambda: lk.NestedKriging(y, X, "matern5_2", nb_groups=4, seed=1) - - -def predict_any(k, Xn): - meth = k.method() if hasattr(k, "method") and isinstance(k, lk.Kriging) else None - if meth and meth != "exact": - out = k.predict(Xn, return_stdev=True, method=meth.split("(")[0]) - return np.ravel(out[0]), np.ravel(out[1]) - return mean_sd(k, Xn) - -# %% [markdown] -# ## save / load, pickle, copy - -# %% -for name, make in MODELS.items(): - with suite.runs(f"{name}: fit"): - k = make() - m0, s0 = predict_any(k, Xt) - cls = type(k) - path = tmp / f"{name.replace(' ', '_')}.json" - if not hasattr(k, "save"): - suite.skip(f"{name}: save + load", "no save() in this pylibkriging") - continue - with suite.runs(f"{name}: save + load"): - k.save(str(path)) - k1 = cls.load(str(path)) - m1, s1 = predict_any(k1, Xt) - suite.close(f"{name}: load(save(k)) predicts the same mean", m1, m0, rtol=1e-12, atol=1e-12) - suite.close(f"{name}: load(save(k)) predicts the same stdev", s1, s0, rtol=1e-10, atol=1e-12) - if hasattr(k, "theta"): - suite.close(f"{name}: same theta after load", np.ravel(k1.theta()), np.ravel(k.theta()), - rtol=1e-14, atol=0) - if hasattr(lk, "load") and (compat.has("load(noise)") or "noise" not in name and "nugget" not in name): - with suite.runs(f"{name}: generic pylibkriging.load"): - k2 = lk.load(str(path)) - suite.true(f"{name}: generic load returns {cls.__name__}", type(k2).__name__ == cls.__name__, - detail=type(k2).__name__) - suite.close(f"{name}: generic load predicts the same", predict_any(k2, Xt)[0], m0, rtol=1e-12, - atol=1e-12) - if not compat.has("pickle"): - suite.skip(f"{name}: pickle / deepcopy", "models not picklable in this pylibkriging") - continue - with suite.runs(f"{name}: pickle round trip"): - k3 = pickle.loads(pickle.dumps(k)) - suite.close(f"{name}: pickle predicts the same", predict_any(k3, Xt)[0], m0, rtol=1e-12, atol=1e-12) - with suite.runs(f"{name}: deepcopy"): - k4 = copy.deepcopy(k) - suite.close(f"{name}: deepcopy predicts the same", predict_any(k4, Xt)[0], m0, rtol=1e-12, atol=1e-12) - -# %% [markdown] -# A copy is independent: updating it leaves the original model unchanged. - -# %% -k = lk.Kriging(y[:30], X[:30], "matern5_2") -m_before = mean_sd(k, Xt)[0] -kc = k.copy() if hasattr(k, "copy") else copy.deepcopy(k) -kc.update(y[30:], X[30:], refit=False) -suite.close("update on a copy leaves the original unchanged", mean_sd(k, Xt)[0], m_before, rtol=0, atol=0) -suite.true("the updated copy did change", not np.allclose(mean_sd(kc, Xt)[0], m_before)) - -# %% [markdown] -# ## Accessors mirror the constructor - -# %% -k = lk.Kriging(y, X, "matern3_2", regmodel="linear", normalize=False, optim="BFGS", objective="LMP") -for acc, expected in [("kernel", "matern3_2"), ("regmodel", "linear"), ("optim", "BFGS"), ("objective", "LMP"), - ("normalize", False)]: - suite.true(f"Kriging.{acc}() == {expected!r}", getattr(k, acc)() == expected, detail=str(getattr(k, acc)())) -suite.close("Kriging.X() is the design", np.asarray(k.X()), X, rtol=0, atol=0) -suite.close("Kriging.y() is the response", np.ravel(k.y()), y, rtol=0, atol=0) -# with normalize=True: do the accessors return the data as given, or as rescaled internally? -kn = lk.Kriging(y, X * 10, "matern3_2", normalize=True) -suite.true("Kriging.normalize() == True", kn.normalize() is True or kn.normalize() == 1) -suite.close("normalize=True: X() is the design as given", np.asarray(kn.X()), X * 10, rtol=1e-12, atol=0, - severity="warn") -suite.close("normalize=True: y() is the response as given", np.ravel(kn.y()), y, rtol=1e-12, atol=0, - severity="warn") -suite.true("Kriging.beta() has 1 + d coefficients (linear trend)", np.ravel(k.beta()).size == 1 + X.shape[1]) -suite.true("summary() is a non-empty string", isinstance(k.summary(), str) and len(k.summary()) > 20) -suite.true("is_theta_estim() after an optimized fit", bool(k.is_theta_estim())) - -# %% [markdown] -# ## Installing scikit-learn -# -# Needed by the estimators of `pylibkriging.sklearn` below. With pip, into the -# kernel's environment, numpy pinned to its installed version (a package that -# would downgrade it is not installed: the libKriging extension is built -# against it). A package that cannot be installed is skipped, and -# reported. - -# %% -import importlib -import subprocess -import sys - -import numpy - -for package in ["scikit-learn"]: - r = subprocess.run([sys.executable, "-m", "pip", "install", "--quiet", package, f"numpy=={numpy.__version__}"], - capture_output=True, text=True) - print(f"{package}: " + ("installed" if r.returncode == 0 else "NOT installed\n" + r.stderr.strip()[-800:])) -importlib.invalidate_caches() - -# %% [markdown] -# ## scikit-learn estimators - -# %% -if compat.has("sklearn"): - from sklearn.base import clone - from sklearn.model_selection import GridSearchCV, cross_val_score - from sklearn.pipeline import make_pipeline - from sklearn.preprocessing import MinMaxScaler - from pylibkriging.sklearn import KrigingRegressor - - est = KrigingRegressor(kernel="matern5_2").fit(X, y) - m_sk, s_sk = est.predict(Xt, return_std=True) - suite.true("KrigingRegressor.predict returns 1-D arrays (scikit-learn convention)", - np.ndim(m_sk) == 1 and np.ndim(s_sk) == 1, detail=f"shapes {np.shape(m_sk)}, {np.shape(s_sk)}", - severity="warn") - m_sk, s_sk = np.ravel(m_sk), np.ravel(s_sk) - m_nat, s_nat = mean_sd(lk.Kriging(y, X, "matern5_2"), Xt) - suite.close("KrigingRegressor == Kriging (mean)", m_sk, m_nat, rtol=1e-8, atol=1e-10) - suite.close("KrigingRegressor == Kriging (stdev)", s_sk, s_nat, rtol=1e-6, atol=1e-10) - est2 = clone(est).set_params(kernel="gauss") - suite.true("clone + set_params", est2.get_params()["kernel"] == "gauss" and est.get_params()["kernel"] == - "matern5_2") - with suite.runs("Pipeline(MinMaxScaler, KrigingRegressor)"): - pipe = make_pipeline(MinMaxScaler(), KrigingRegressor(kernel="matern5_2")).fit(tf.scale(X), y) - suite.greater("pipeline Q2 on native inputs", pipe.score(tf.scale(Xt), yt), 0.8) - with suite.runs("GridSearchCV over kernels"): - gs = GridSearchCV(KrigingRegressor(), {"kernel": ["gauss", "matern3_2", "matern5_2"]}, cv=3).fit(X, y) - suite.true("GridSearchCV picked a kernel", gs.best_params_["kernel"] in ("gauss", "matern3_2", "matern5_2"), - detail=str(gs.best_params_)) - with suite.runs("cross_val_score"): - sc = cross_val_score(KrigingRegressor(kernel="matern5_2"), X, y, cv=4) - suite.finite("cross_val_score finite", sc) - with suite.runs("sample_y"): - S = est.sample_y(Xt[:10], n_samples=5, random_state=1) - suite.true("sample_y shape (q, n_samples)", np.shape(S) == (10, 5), detail=str(np.shape(S))) -else: - suite.skip("sklearn estimators", "pylibkriging.sklearn or scikit-learn not available") - -# %% -suite.finalize() diff --git a/tools/src/python_Kriging_robustness-invalid-inputs.py b/tools/src/robustness-invalid-inputs_Kriging_python.py similarity index 99% rename from tools/src/python_Kriging_robustness-invalid-inputs.py rename to tools/src/robustness-invalid-inputs_Kriging_python.py index fff6257..b948abf 100644 --- a/tools/src/python_Kriging_robustness-invalid-inputs.py +++ b/tools/src/robustness-invalid-inputs_Kriging_python.py @@ -27,7 +27,7 @@ import lkbench from lkbench import TESTS, design, compat -suite = lkbench.Suite("python_Kriging_robustness-invalid-inputs") +suite = lkbench.Suite("robustness-invalid-inputs_Kriging_python") print("pylibkriging", compat.version()) tf = TESTS["branin"] X, y, Xt, yt = design.dataset(tf, 20, n_test=200, seed=71) diff --git a/tools/src/save-load-pickle-copy-accessors_Kriging_python.py b/tools/src/save-load-pickle-copy-accessors_Kriging_python.py new file mode 100644 index 0000000..e609406 --- /dev/null +++ b/tools/src/save-load-pickle-copy-accessors_Kriging_python.py @@ -0,0 +1,99 @@ +# %% [markdown] +# # Python · Kriging — save/load, pickle, copy, accessors +# +# Persistence and introspection of `Kriging`, for each of its variants (noise +# models, objectives, approximate fit methods): +# +# * `save` → `load` (class-specific and generic `pylibkriging.load`) gives a model +# with **identical** predictions and hyperparameters; +# * `pickle` / `copy` round trips, and a copy is independent of the original +# (updating one leaves the other untouched); +# * accessors mirror the constructor arguments (`kernel()`, `regmodel()`, +# `optim()`, `objective()`, `X()`, `y()`, ...). +# +# The same persistence checks for the other classes are in their own notebooks +# (`*_WarpKriging_python`, `*_MLPKriging_python`, `*_NestedKriging_python`); the +# scikit-learn estimators in `sklearn-estimators_Kriging_python.ipynb`. + +# %% +import copy + +import numpy as np +import pylibkriging as lk + +import lkbench +from lkbench import TESTS, design, compat + +suite = lkbench.Suite("save-load-pickle-copy-accessors_Kriging_python") +print("pylibkriging", compat.version()) + +tf = TESTS["hartmann3"] +X, y, Xt, yt = design.dataset(tf, 40, n_test=100, seed=61) + + +def mean_sd(k, Xn): + out = k.predict(Xn, return_stdev=True) + return np.ravel(out[0]), np.ravel(out[1]) + + +MODELS = { + "Kriging": lambda: lk.Kriging(y, X, "matern5_2", regmodel="linear"), + "Kriging nugget": lambda: lk.Kriging(y, X, "matern3_2", noise="nugget"), + "Kriging noise vector": lambda: lk.Kriging(y, X, "gauss", noise=np.full(len(y), 1e-4)), + "Kriging LOO": lambda: lk.Kriging(y, X, "matern5_2", objective="LOO"), + "Kriging LMP": lambda: lk.Kriging(y, X, "matern5_2", objective="LMP"), +} +if compat.has("method"): + MODELS["Kriging vecchia"] = lambda: lk.Kriging(y, X, "matern5_2", method="vecchia(10)") + MODELS["Kriging nystrom"] = lambda: lk.Kriging(y, X, "matern5_2", method="nystrom(20)") + + +def predict_any(k, Xn): + meth = k.method() if hasattr(k, "method") and isinstance(k, lk.Kriging) else None + if meth and meth != "exact": + out = k.predict(Xn, return_stdev=True, method=meth.split("(")[0]) + return np.ravel(out[0]), np.ravel(out[1]) + return mean_sd(k, Xn) + +# %% [markdown] +# ## save / load, pickle, copy + +# %% +for name, make in MODELS.items(): + with suite.runs(f"{name}: fit"): + lkbench.persistence.check(suite, name, make(), Xt, predict_any) + +# %% [markdown] +# A copy is independent: updating it leaves the original model unchanged. + +# %% +k = lk.Kriging(y[:30], X[:30], "matern5_2") +m_before = mean_sd(k, Xt)[0] +kc = k.copy() if hasattr(k, "copy") else copy.deepcopy(k) +kc.update(y[30:], X[30:], refit=False) +suite.close("update on a copy leaves the original unchanged", mean_sd(k, Xt)[0], m_before, rtol=0, atol=0) +suite.true("the updated copy did change", not np.allclose(mean_sd(kc, Xt)[0], m_before)) + +# %% [markdown] +# ## Accessors mirror the constructor + +# %% +k = lk.Kriging(y, X, "matern3_2", regmodel="linear", normalize=False, optim="BFGS", objective="LMP") +for acc, expected in [("kernel", "matern3_2"), ("regmodel", "linear"), ("optim", "BFGS"), ("objective", "LMP"), + ("normalize", False)]: + suite.true(f"Kriging.{acc}() == {expected!r}", getattr(k, acc)() == expected, detail=str(getattr(k, acc)())) +suite.close("Kriging.X() is the design", np.asarray(k.X()), X, rtol=0, atol=0) +suite.close("Kriging.y() is the response", np.ravel(k.y()), y, rtol=0, atol=0) +# with normalize=True: do the accessors return the data as given, or as rescaled internally? +kn = lk.Kriging(y, X * 10, "matern3_2", normalize=True) +suite.true("Kriging.normalize() == True", kn.normalize() is True or kn.normalize() == 1) +suite.close("normalize=True: X() is the design as given", np.asarray(kn.X()), X * 10, rtol=1e-12, atol=0, + severity="warn") +suite.close("normalize=True: y() is the response as given", np.ravel(kn.y()), y, rtol=1e-12, atol=0, + severity="warn") +suite.true("Kriging.beta() has 1 + d coefficients (linear trend)", np.ravel(k.beta()).size == 1 + X.shape[1]) +suite.true("summary() is a non-empty string", isinstance(k.summary(), str) and len(k.summary()) > 20) +suite.true("is_theta_estim() after an optimized fit", bool(k.is_theta_estim())) + +# %% +suite.finalize() diff --git a/tools/src/sklearn-estimators_Kriging_python.py b/tools/src/sklearn-estimators_Kriging_python.py new file mode 100644 index 0000000..80df91d --- /dev/null +++ b/tools/src/sklearn-estimators_Kriging_python.py @@ -0,0 +1,88 @@ +# %% [markdown] +# # Python · Kriging — scikit-learn estimators (`pylibkriging.sklearn`) +# +# `KrigingRegressor` (the scikit-learn face of `Kriging`) on Hartmann 3: +# `fit`/`predict` (scikit-learn conventions), the same predictions as the native +# class, `get_params`/`set_params`/`clone`, use inside `Pipeline`, +# `GridSearchCV`, `cross_val_score`, and `sample_y`. + +# %% +import numpy as np +import pylibkriging as lk + +import lkbench +from lkbench import TESTS, design, compat + +suite = lkbench.Suite("sklearn-estimators_Kriging_python") +print("pylibkriging", compat.version()) + +tf = TESTS["hartmann3"] +X, y, Xt, yt = design.dataset(tf, 40, n_test=100, seed=61) + + +def mean_sd(k, Xn): + out = k.predict(Xn, return_stdev=True) + return np.ravel(out[0]), np.ravel(out[1]) + +# %% [markdown] +# ## Installing scikit-learn +# +# With pip, into the kernel's environment, numpy pinned to its installed version +# (a package that would downgrade it is not installed: the libKriging extension +# is built against it). A package that cannot be installed is skipped, and +# reported. + +# %% +import importlib +import subprocess +import sys + +import numpy + +for package in ["scikit-learn"]: + r = subprocess.run([sys.executable, "-m", "pip", "install", "--quiet", package, f"numpy=={numpy.__version__}"], + capture_output=True, text=True) + print(f"{package}: " + ("installed" if r.returncode == 0 else "NOT installed\n" + r.stderr.strip()[-800:])) +importlib.invalidate_caches() + +# %% [markdown] +# ## scikit-learn estimators + +# %% +if compat.has("sklearn"): + from sklearn.base import clone + from sklearn.model_selection import GridSearchCV, cross_val_score + from sklearn.pipeline import make_pipeline + from sklearn.preprocessing import MinMaxScaler + from pylibkriging.sklearn import KrigingRegressor + + est = KrigingRegressor(kernel="matern5_2").fit(X, y) + m_sk, s_sk = est.predict(Xt, return_std=True) + suite.true("KrigingRegressor.predict returns 1-D arrays (scikit-learn convention)", + np.ndim(m_sk) == 1 and np.ndim(s_sk) == 1, detail=f"shapes {np.shape(m_sk)}, {np.shape(s_sk)}", + severity="warn") + m_sk, s_sk = np.ravel(m_sk), np.ravel(s_sk) + m_nat, s_nat = mean_sd(lk.Kriging(y, X, "matern5_2"), Xt) + suite.close("KrigingRegressor == Kriging (mean)", m_sk, m_nat, rtol=1e-8, atol=1e-10) + suite.close("KrigingRegressor == Kriging (stdev)", s_sk, s_nat, rtol=1e-6, atol=1e-10) + est2 = clone(est).set_params(kernel="gauss") + suite.true("clone + set_params", est2.get_params()["kernel"] == "gauss" and est.get_params()["kernel"] == + "matern5_2") + with suite.runs("Pipeline(MinMaxScaler, KrigingRegressor)"): + pipe = make_pipeline(MinMaxScaler(), KrigingRegressor(kernel="matern5_2")).fit(tf.scale(X), y) + suite.greater("pipeline Q2 on native inputs", pipe.score(tf.scale(Xt), yt), 0.8) + with suite.runs("GridSearchCV over kernels"): + gs = GridSearchCV(KrigingRegressor(), {"kernel": ["gauss", "matern3_2", "matern5_2"]}, cv=3).fit(X, y) + suite.true("GridSearchCV picked a kernel", gs.best_params_["kernel"] in ("gauss", "matern3_2", "matern5_2"), + detail=str(gs.best_params_)) + with suite.runs("cross_val_score"): + sc = cross_val_score(KrigingRegressor(kernel="matern5_2"), X, y, cv=4) + suite.finite("cross_val_score finite", sc) + with suite.runs("sample_y"): + S = est.sample_y(Xt[:10], n_samples=5, random_state=1) + suite.true("sample_y shape (q, n_samples)", np.shape(S) == (10, 5), detail=str(np.shape(S))) +else: + suite.skip("sklearn estimators", "pylibkriging.sklearn or scikit-learn not available") + +# %% +suite.finalize() diff --git a/tools/src/python_no-model_test-functions-reference-values.py b/tools/src/test-functions-reference-values_no-model_python.py similarity index 97% rename from tools/src/python_no-model_test-functions-reference-values.py rename to tools/src/test-functions-reference-values_no-model_python.py index 9135f87..c355439 100644 --- a/tools/src/python_no-model_test-functions-reference-values.py +++ b/tools/src/test-functions-reference-values_no-model_python.py @@ -15,7 +15,7 @@ import lkbench from lkbench import TESTS, design -suite = lkbench.Suite("python_no-model_test-functions-reference-values") +suite = lkbench.Suite("test-functions-reference-values_no-model_python") # %% [markdown] # ## Reference values diff --git a/tools/src/python_Kriging_update-simulate.py b/tools/src/update-simulate_Kriging_python.py similarity index 99% rename from tools/src/python_Kriging_update-simulate.py rename to tools/src/update-simulate_Kriging_python.py index 0239782..3c3ed9c 100644 --- a/tools/src/python_Kriging_update-simulate.py +++ b/tools/src/update-simulate_Kriging_python.py @@ -24,7 +24,7 @@ import lkbench from lkbench import TESTS, design, compat -suite = lkbench.Suite("python_Kriging_update-simulate") +suite = lkbench.Suite("update-simulate_Kriging_python") print("pylibkriging", compat.version()) diff --git a/tools/src/R_Kriging_vs-DiceKriging-RobustGaSP.R b/tools/src/vs-DiceKriging-RobustGaSP_Kriging_R.R similarity index 99% rename from tools/src/R_Kriging_vs-DiceKriging-RobustGaSP.R rename to tools/src/vs-DiceKriging-RobustGaSP_Kriging_R.R index 4fffb7f..373a231 100644 --- a/tools/src/R_Kriging_vs-DiceKriging-RobustGaSP.R +++ b/tools/src/vs-DiceKriging-RobustGaSP_Kriging_R.R @@ -38,7 +38,7 @@ suppressPackageStartupMessages({ }) has_rgasp <- requireNamespace("RobustGaSP", quietly = TRUE) source("tools/r/checks.R") -s <- new_suite("R_Kriging_vs-DiceKriging-RobustGaSP") +s <- new_suite("vs-DiceKriging-RobustGaSP_Kriging_R") cat("rlibkriging", as.character(packageVersion("rlibkriging")), "| DiceKriging", as.character(packageVersion("DiceKriging")), "| RobustGaSP", if (has_rgasp) as.character(packageVersion("RobustGaSP")) else "absent", "\n") DATASETS <- c("branin", "hartmann3", "borehole") diff --git a/tools/src/julia_Kriging_vs-GaussianProcesses.jl b/tools/src/vs-GaussianProcesses_Kriging_julia.jl similarity index 98% rename from tools/src/julia_Kriging_vs-GaussianProcesses.jl rename to tools/src/vs-GaussianProcesses_Kriging_julia.jl index e5f6a01..1abaada 100644 --- a/tools/src/julia_Kriging_vs-GaussianProcesses.jl +++ b/tools/src/vs-GaussianProcesses_Kriging_julia.jl @@ -13,7 +13,7 @@ # %% import JLibKriging as LK include("tools/julia/checks.jl") -s = Suite("julia_Kriging_vs-GaussianProcesses") +s = Suite("vs-GaussianProcesses_Kriging_julia") # %% [markdown] # ## Installing GaussianProcesses.jl diff --git a/tools/src/octave_Kriging_vs-STK.m b/tools/src/vs-STK_Kriging_octave.m similarity index 99% rename from tools/src/octave_Kriging_vs-STK.m rename to tools/src/vs-STK_Kriging_octave.m index 9e10b6c..bac5706 100644 --- a/tools/src/octave_Kriging_vs-STK.m +++ b/tools/src/vs-STK_Kriging_octave.m @@ -19,7 +19,7 @@ % %% addpath(fullfile(pwd, "tools", "octave")); -s = lkb_suite("octave_Kriging_vs-STK"); +s = lkb_suite("vs-STK_Kriging_octave"); % %% [markdown] % ## Installing STK diff --git a/tools/src/python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy.py b/tools/src/vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python.py similarity index 99% rename from tools/src/python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy.py rename to tools/src/vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python.py index 9ea756b..7a251d9 100644 --- a/tools/src/python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy.py +++ b/tools/src/vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python.py @@ -51,7 +51,7 @@ from lkbench import TESTS, design, metrics, compat warnings.filterwarnings("ignore") -suite = lkbench.Suite("python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy") +suite = lkbench.Suite("vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python") sk = compat.optional("sklearn.gaussian_process") smt = compat.optional("smt.surrogate_models") ot = compat.optional("openturns") diff --git a/tools/src/python_WarpKriging_warpings-categorical-ordinal.py b/tools/src/warpings-categorical-ordinal_WarpKriging_python.py similarity index 92% rename from tools/src/python_WarpKriging_warpings-categorical-ordinal.py rename to tools/src/warpings-categorical-ordinal_WarpKriging_python.py index 95388c7..ed6485a 100644 --- a/tools/src/python_WarpKriging_warpings-categorical-ordinal.py +++ b/tools/src/warpings-categorical-ordinal_WarpKriging_python.py @@ -24,7 +24,7 @@ import lkbench from lkbench import TESTS, design, metrics, compat -suite = lkbench.Suite("python_WarpKriging_warpings-categorical-ordinal") +suite = lkbench.Suite("warpings-categorical-ordinal_WarpKriging_python") print("pylibkriging", compat.version()) if not compat.has("WarpKriging"): suite.skip("WarpKriging", "not available in this pylibkriging") @@ -179,5 +179,18 @@ def f_ord(Xo): # %% suite.table().round(4) +# %% [markdown] +# ## save / load, pickle, copy +# +# On Hartmann 3 (n = 40): a model saved and loaded back (class-specific and +# generic `pylibkriging.load`), pickled, or deep-copied predicts exactly as the +# original. + +# %% +if compat.has("WarpKriging"): + X, y, Xt, yt = design.dataset(TESTS["hartmann3"], 40, n_test=100, seed=61) + with suite.runs("WarpKriging: fit (persistence)"): + lkbench.persistence.check(suite, "WarpKriging", lk.WarpKriging(y, X, ["kumaraswamy", "none", "affine"], "matern5_2"), Xt, pred) + # %% suite.finalize() diff --git a/tools/src/warpings-categorical_WarpKriging_R.R b/tools/src/warpings-categorical_WarpKriging_R.R new file mode 100644 index 0000000..0d2106b --- /dev/null +++ b/tools/src/warpings-categorical_WarpKriging_R.R @@ -0,0 +1,78 @@ +# %% [markdown] +# # R · `WarpKriging` — continuous warpings, a categorical input +# +# Branin on [0,1]² (n = 30 LHS points unless stated, Q² on 500 uniform test +# points), through rlibkriging. Checks (error-level unless stated): +# +# * `kumaraswamy` on both inputs, then x₂ restricted to 5 levels with `categorical(5,2)`: finite predictions +# * Q² floors (warn-level) +# * `save` then `load`: the same predicted mean +# +# A class missing from the installed rlibkriging (release channel) is recorded as SKIP. + +# %% +suppressPackageStartupMessages(library(rlibkriging)) +source("tools/r/checks.R") +s <- new_suite("warpings-categorical_WarpKriging_R") +cat("rlibkriging", as.character(packageVersion("rlibkriging")), "\n") + +branin <- function(U) { # on [0,1]^2 + x1 <- 15 * U[, 1] - 5; x2 <- 15 * U[, 2] + (x2 - 5.1 / (4 * pi^2) * x1^2 + 5 / pi * x1 - 6)^2 + 10 * (1 - 1 / (8 * pi)) * cos(x1) + 10 +} +set.seed(1) +lhs <- function(n, d) sapply(1:d, function(j) (sample(n) - runif(n)) / n) +X <- lhs(30, 2); y <- branin(X) +Xt <- matrix(runif(1000), ncol = 2); yt <- branin(Xt) +tmp <- tempdir() +has_class <- function(name) exists(name, mode = "function") +skip <- function(s, check, reason) { + s$checks[[length(s$checks) + 1]] <- data.frame(suite = s$name, lang = "R", check = check, severity = "info", + status = "SKIP", value = "", expected = "", detail = reason, stringsAsFactors = FALSE) + message("[SKIP] ", check, ": ", reason) +} +roundtrip_same_mean <- function(s, label, k, Xn) { + f <- file.path(tmp, paste0(gsub("[^A-Za-z0-9]", "_", label), ".json")) + check_runs(s, paste0(label, ": save + load"), { + save(k, f) + k2 <- rlibkriging::load(f) + check_close(s, paste0(label, ": load(save(k)) predicts the same mean"), + predict(k2, Xn, return_stdev = FALSE)$mean, predict(k, Xn, return_stdev = FALSE)$mean, rtol = 1e-10) + }) +} + +# %% [markdown] +# ## `WarpKriging`: continuous warpings, then a categorical input +# +# The categorical case: x₂ restricted to 5 levels (coded 0, …, 4), embedded in ℝ². + +# %% +if (has_class("WarpKriging")) { + check_runs(s, "WarpKriging(kumaraswamy, kumaraswamy): fit + predict", { + kw <- WarpKriging(y, X, warping = c("kumaraswamy", "kumaraswamy"), kernel = "matern5_2") + pw <- predict(kw, Xt, return_stdev = TRUE) + check_less(s, "WarpKriging(kumaraswamy): non-finite predictions", sum(!is.finite(c(pw$mean, pw$stdev))), 1) + add_metric(s, "branin", "WarpKriging kumaraswamy", q2 = q2(yt, pw$mean)) + check_greater(s, "WarpKriging(kumaraswamy): Q2", q2(yt, pw$mean), 0.9, severity = "warn") + roundtrip_same_mean(s, "WarpKriging(kumaraswamy)", kw, Xt) + }) + lev <- function(U) cbind(U[, 1], round(U[, 2] * 4)) # level codes 0..4 + unlev <- function(L) cbind(L[, 1], L[, 2] / 4) + Xc <- lev(lhs(40, 2)); yc <- branin(unlev(Xc)) + Xtc <- lev(Xt); ytc <- branin(unlev(Xtc)) + check_runs(s, "WarpKriging(none, categorical(5,2)): fit + predict", { + kc <- WarpKriging(yc, Xc, warping = c("none", "categorical(5,2)"), kernel = "matern5_2") + pc <- predict(kc, Xtc, return_stdev = TRUE) + check_less(s, "WarpKriging(categorical): non-finite predictions", sum(!is.finite(c(pc$mean, pc$stdev))), 1) + add_metric(s, "branin, x2 categorical", "WarpKriging categorical", q2 = q2(ytc, pc$mean)) + check_greater(s, "WarpKriging(categorical): Q2", q2(ytc, pc$mean), 0.8, severity = "warn") + roundtrip_same_mean(s, "WarpKriging(categorical)", kc, Xtc) + }) +} else skip(s, "WarpKriging", "not in this rlibkriging") + +# %% +me <- do.call(rbind, s$metrics) +print(me[me$metric == "q2", c("case", "package", "value")], row.names = FALSE) + +# %% +finalize(s) diff --git a/tools/src/warpings-categorical_WarpKriging_julia.jl b/tools/src/warpings-categorical_WarpKriging_julia.jl new file mode 100644 index 0000000..2a30927 --- /dev/null +++ b/tools/src/warpings-categorical_WarpKriging_julia.jl @@ -0,0 +1,83 @@ +# %% [markdown] +# # Julia · `WarpKriging` — continuous warpings, a categorical input +# +# Branin on [0,1]² (n = 30 LHS points unless stated, Q² on 500 uniform test +# points), through JLibKriging. Checks (error-level unless stated): +# +# * `kumaraswamy` on both inputs, then x₂ restricted to 5 levels with `categorical(5,2)`: finite predictions +# * Q² floors (warn-level) +# * `save` then `load`: the same predicted mean +# +# The same checks as `warpings-categorical_WarpKriging_R.ipynb`, with the Julia binding. + +# %% +import JLibKriging as LK +using Random +include("tools/julia/checks.jl") +s = Suite("warpings-categorical_WarpKriging_julia") + +branin(U) = begin + x1 = 15 .* U[:, 1] .- 5; x2 = 15 .* U[:, 2] + (x2 .- 5.1 / (4π^2) .* x1 .^ 2 .+ 5 / π .* x1 .- 6) .^ 2 .+ 10 * (1 - 1 / (8π)) .* cos.(x1) .+ 10 +end +rng = MersenneTwister(1) +lhs(n, d) = hcat([(randperm(rng, n) .- rand(rng, n)) ./ n for _ in 1:d]...) +X = lhs(30, 2); y = branin(X) +Xt = rand(rng, 500, 2); yt = branin(Xt) +sd(v) = sqrt(sum((v .- sum(v) / length(v)) .^ 2) / (length(v) - 1)) +finite_count(v...) = count(!isfinite, vcat(v...)) + +function check_runs!(f, s, check) + try + f() + catch e + add_check!(s, check, false; value=string(typeof(e)), expected="no error", detail=first(sprint(showerror, e), 200)) + end +end + +function roundtrip!(s, label, k, Xn) + check_runs!(s, "$label: save + load") do + f = joinpath(mktempdir(), "k.json") + LK.save(k, f) + k2 = LK.load(f) + check_close!(s, "$label: load(save(k)) predicts the same mean", + LK.predict(k2, Xn; return_stdev=false).mean, LK.predict(k, Xn; return_stdev=false).mean; rtol=1e-10) + end +end + +# %% [markdown] +# ## `WarpKriging`: continuous warpings, then a categorical input +# +# The categorical case: x₂ restricted to 5 levels (coded 0, …, 4), embedded in ℝ². + +# %% +check_runs!(s, "WarpKriging(kumaraswamy, kumaraswamy): fit + predict") do + kw = LK.WarpKriging(y, X, ["kumaraswamy", "kumaraswamy"], "matern5_2") + pw = LK.predict(kw, Xt; return_stdev=true) + add_check!(s, "WarpKriging(kumaraswamy): finite predictions", finite_count(pw.mean, pw.stdev) == 0; + value=finite_count(pw.mean, pw.stdev), expected="0 non-finite") + add_metric!(s, "branin", "WarpKriging kumaraswamy"; q2=q2(yt, pw.mean)) + add_check!(s, "WarpKriging(kumaraswamy): Q2", q2(yt, pw.mean) > 0.9; value=q2(yt, pw.mean), expected="> 0.9", + severity="warn") + roundtrip!(s, "WarpKriging(kumaraswamy)", kw, Xt) +end +lev(U) = hcat(U[:, 1], round.(U[:, 2] .* 4)) +unlev(L) = hcat(L[:, 1], L[:, 2] ./ 4) +Xc = lev(lhs(40, 2)); yc = branin(unlev(Xc)) +Xtc = lev(Xt); ytc = branin(unlev(Xtc)) +check_runs!(s, "WarpKriging(none, categorical(5,2)): fit + predict") do + kc = LK.WarpKriging(yc, Xc, ["none", "categorical(5,2)"], "matern5_2") + pc = LK.predict(kc, Xtc; return_stdev=true) + add_check!(s, "WarpKriging(categorical): finite predictions", finite_count(pc.mean, pc.stdev) == 0; + value=finite_count(pc.mean, pc.stdev), expected="0 non-finite") + add_metric!(s, "branin, x2 categorical", "WarpKriging categorical"; q2=q2(ytc, pc.mean)) + add_check!(s, "WarpKriging(categorical): Q2", q2(ytc, pc.mean) > 0.8; value=q2(ytc, pc.mean), expected="> 0.8", + severity="warn") + roundtrip!(s, "WarpKriging(categorical)", kc, Xtc) +end + +# %% +foreach(r -> r[5] == "q2" && println(rpad(r[3], 24), rpad(r[4], 28), r[6]), s.metrics) + +# %% +finalize!(s) diff --git a/tools/src/warpings-categorical_WarpKriging_octave.m b/tools/src/warpings-categorical_WarpKriging_octave.m new file mode 100644 index 0000000..adc32f6 --- /dev/null +++ b/tools/src/warpings-categorical_WarpKriging_octave.m @@ -0,0 +1,89 @@ +% %% [markdown] +% # Octave · `WarpKriging` — continuous warpings, a categorical input +% +% Branin on [0,1]² (n = 30 LHS points unless stated, Q² on 500 uniform test +% points), through mLibKriging. Checks (error-level unless stated): +% +% * `kumaraswamy` on both inputs, then x₂ restricted to 5 levels with `categorical(5,2)`: finite predictions +% * Q² floors (warn-level) +% * `save` then `load`: the same predicted mean +% +% The same checks as `warpings-categorical_WarpKriging_R.ipynb`, with the Octave binding. +% +% A class missing from the installed mLibKriging (release channel) is recorded as SKIP. + +% %% +addpath(fullfile(pwd, "tools", "octave")); +s = lkb_suite("warpings-categorical_WarpKriging_octave"); +q2 = @(y, m) 1 - sum((y - m).^2) / sum((y - mean(y)).^2); +branin = @(U) ((15 * U(:, 2)) - 5.1 / (4 * pi^2) * (15 * U(:, 1) - 5).^2 + 5 / pi * (15 * U(:, 1) - 5) - 6).^2 ... + + 10 * (1 - 1 / (8 * pi)) * cos(15 * U(:, 1) - 5) + 10; +rand("seed", 1); randn("seed", 1); +function U = lhs(n, d) + U = zeros(n, d); + for j = 1:d, U(:, j) = (randperm(n)' - rand(n, 1)) / n; end +end +X = lhs(30, 2); y = branin(X); +Xt = rand(500, 2); yt = branin(Xt); +tmp = tempname(); mkdir(tmp); +has_class = @(name) exist(name) == 2; % a class file on the path + +function s = roundtrip(s, label, k, Xn, loader) + f = fullfile(tempdir(), [regexprep(label, "[^A-Za-z0-9]", "_") ".json"]); + try + k.save(f); + k2 = loader(f); + m1 = k.predict(Xn, false, false, false); + m2 = k2.predict(Xn, false, false, false); + s = lkb_close(s, [label ": load(save(k)) predicts the same mean"], m2, m1, 1e-10); + catch err + s = lkb_check(s, [label ": save + load"], false, "error", "no error", err.message); + end +end + +function s = skip(s, check, reason) + s.checks(end + 1, :) = {s.name, "octave", check, "info", "SKIP", "", "", reason}; + printf("[SKIP] %s: %s\n", check, reason); +end + +% %% [markdown] +% ## `WarpKriging`: continuous warpings, then a categorical input +% +% The categorical case: x₂ restricted to 5 levels (coded 0, …, 4), embedded in ℝ². + +% %% +if has_class("WarpKriging") + try + kw = WarpKriging(y, X, {"kumaraswamy", "kumaraswamy"}, "matern5_2"); + [mw, sw] = kw.predict(Xt, true, false, false); + s = lkb_check(s, "WarpKriging(kumaraswamy): finite predictions", all(isfinite([mw; sw])), sum(~isfinite([mw; sw])), "0 non-finite"); + s = lkb_metric(s, "branin", "WarpKriging kumaraswamy", "q2", q2(yt, mw)); + s = lkb_check(s, "WarpKriging(kumaraswamy): Q2", q2(yt, mw) > 0.9, q2(yt, mw), "> 0.9", "", "warn"); + s = roundtrip(s, "WarpKriging(kumaraswamy)", kw, Xt, @load_kriging); + catch err + s = lkb_check(s, "WarpKriging(kumaraswamy): fit + predict", false, "error", "no error", err.message); + end + lev = @(U) [U(:, 1), round(U(:, 2) * 4)]; + unlev = @(L) [L(:, 1), L(:, 2) / 4]; + Xc = lev(lhs(40, 2)); yc = branin(unlev(Xc)); + Xtc = lev(Xt); ytc = branin(unlev(Xtc)); + try + kc = WarpKriging(yc, Xc, {"none", "categorical(5,2)"}, "matern5_2"); + [mc, sc] = kc.predict(Xtc, true, false, false); + s = lkb_check(s, "WarpKriging(categorical): finite predictions", all(isfinite([mc; sc])), sum(~isfinite([mc; sc])), "0 non-finite"); + s = lkb_metric(s, "branin, x2 categorical", "WarpKriging categorical", "q2", q2(ytc, mc)); + s = lkb_check(s, "WarpKriging(categorical): Q2", q2(ytc, mc) > 0.8, q2(ytc, mc), "> 0.8", "", "warn"); + s = roundtrip(s, "WarpKriging(categorical)", kc, Xtc, @load_kriging); + catch err + s = lkb_check(s, "WarpKriging(categorical): fit + predict", false, "error", "no error", err.message); + end +else + s = skip(s, "WarpKriging", "not in this mLibKriging"); +end + +% %% +q = strcmp(s.metrics(:, 5), "q2"); +disp(s.metrics(q, [3, 4, 6])) + +% %% +lkb_finalize(s); diff --git a/python_Kriging_update-simulate.ipynb b/update-simulate_Kriging_python.ipynb similarity index 99% rename from python_Kriging_update-simulate.ipynb rename to update-simulate_Kriging_python.ipynb index 2e0c8a4..f1a3496 100644 --- a/python_Kriging_update-simulate.ipynb +++ b/update-simulate_Kriging_python.ipynb @@ -37,7 +37,7 @@ "import lkbench\n", "from lkbench import TESTS, design, compat\n", "\n", - "suite = lkbench.Suite(\"python_Kriging_update-simulate\")\n", + "suite = lkbench.Suite(\"update-simulate_Kriging_python\")\n", "print(\"pylibkriging\", compat.version())\n", "\n", "\n", diff --git a/R_Kriging_vs-DiceKriging-RobustGaSP.ipynb b/vs-DiceKriging-RobustGaSP_Kriging_R.ipynb similarity index 99% rename from R_Kriging_vs-DiceKriging-RobustGaSP.ipynb rename to vs-DiceKriging-RobustGaSP_Kriging_R.ipynb index e48cbac..804bc82 100644 --- a/R_Kriging_vs-DiceKriging-RobustGaSP.ipynb +++ b/vs-DiceKriging-RobustGaSP_Kriging_R.ipynb @@ -63,7 +63,7 @@ "})\n", "has_rgasp <- requireNamespace(\"RobustGaSP\", quietly = TRUE)\n", "source(\"tools/r/checks.R\")\n", - "s <- new_suite(\"R_Kriging_vs-DiceKriging-RobustGaSP\")\n", + "s <- new_suite(\"vs-DiceKriging-RobustGaSP_Kriging_R\")\n", "cat(\"rlibkriging\", as.character(packageVersion(\"rlibkriging\")), \"| DiceKriging\", as.character(packageVersion(\"DiceKriging\")),\n", " \"| RobustGaSP\", if (has_rgasp) as.character(packageVersion(\"RobustGaSP\")) else \"absent\", \"\\n\")\n", "DATASETS <- c(\"branin\", \"hartmann3\", \"borehole\")" diff --git a/julia_Kriging_vs-GaussianProcesses.ipynb b/vs-GaussianProcesses_Kriging_julia.ipynb similarity index 98% rename from julia_Kriging_vs-GaussianProcesses.ipynb rename to vs-GaussianProcesses_Kriging_julia.ipynb index 1fbfb40..5a99dd3 100644 --- a/julia_Kriging_vs-GaussianProcesses.ipynb +++ b/vs-GaussianProcesses_Kriging_julia.ipynb @@ -26,7 +26,7 @@ "source": [ "import JLibKriging as LK\n", "include(\"tools/julia/checks.jl\")\n", - "s = Suite(\"julia_Kriging_vs-GaussianProcesses\")" + "s = Suite(\"vs-GaussianProcesses_Kriging_julia\")" ] }, { diff --git a/octave_Kriging_vs-STK.ipynb b/vs-STK_Kriging_octave.ipynb similarity index 99% rename from octave_Kriging_vs-STK.ipynb rename to vs-STK_Kriging_octave.ipynb index 198080d..0605a12 100644 --- a/octave_Kriging_vs-STK.ipynb +++ b/vs-STK_Kriging_octave.ipynb @@ -32,7 +32,7 @@ "outputs": [], "source": [ "addpath(fullfile(pwd, \"tools\", \"octave\"));\n", - "s = lkb_suite(\"octave_Kriging_vs-STK\");" + "s = lkb_suite(\"vs-STK_Kriging_octave\");" ] }, { diff --git a/python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy.ipynb b/vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python.ipynb similarity index 99% rename from python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy.ipynb rename to vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python.ipynb index abd91e1..bbdaf49 100644 --- a/python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy.ipynb +++ b/vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python.ipynb @@ -76,7 +76,7 @@ "from lkbench import TESTS, design, metrics, compat\n", "\n", "warnings.filterwarnings(\"ignore\")\n", - "suite = lkbench.Suite(\"python_Kriging_vs-sklearn-SMT-OpenTURNS-GPy\")\n", + "suite = lkbench.Suite(\"vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python\")\n", "sk = compat.optional(\"sklearn.gaussian_process\")\n", "smt = compat.optional(\"smt.surrogate_models\")\n", "ot = compat.optional(\"openturns\")\n", diff --git a/python_WarpKriging_warpings-categorical-ordinal.ipynb b/warpings-categorical-ordinal_WarpKriging_python.ipynb similarity index 92% rename from python_WarpKriging_warpings-categorical-ordinal.ipynb rename to warpings-categorical-ordinal_WarpKriging_python.ipynb index 11e4741..bd201a6 100644 --- a/python_WarpKriging_warpings-categorical-ordinal.ipynb +++ b/warpings-categorical-ordinal_WarpKriging_python.ipynb @@ -37,7 +37,7 @@ "import lkbench\n", "from lkbench import TESTS, design, metrics, compat\n", "\n", - "suite = lkbench.Suite(\"python_WarpKriging_warpings-categorical-ordinal\")\n", + "suite = lkbench.Suite(\"warpings-categorical-ordinal_WarpKriging_python\")\n", "print(\"pylibkriging\", compat.version())\n", "if not compat.has(\"WarpKriging\"):\n", " suite.skip(\"WarpKriging\", \"not available in this pylibkriging\")\n", @@ -260,10 +260,35 @@ "suite.table().round(4)" ] }, + { + "cell_type": "markdown", + "id": "c013", + "metadata": {}, + "source": [ + "## save / load, pickle, copy\n", + "\n", + "On Hartmann 3 (n = 40): a model saved and loaded back (class-specific and\n", + "generic `pylibkriging.load`), pickled, or deep-copied predicts exactly as the\n", + "original." + ] + }, { "cell_type": "code", "execution_count": null, - "id": "c013", + "id": "c014", + "metadata": {}, + "outputs": [], + "source": [ + "if compat.has(\"WarpKriging\"):\n", + " X, y, Xt, yt = design.dataset(TESTS[\"hartmann3\"], 40, n_test=100, seed=61)\n", + " with suite.runs(\"WarpKriging: fit (persistence)\"):\n", + " lkbench.persistence.check(suite, \"WarpKriging\", lk.WarpKriging(y, X, [\"kumaraswamy\", \"none\", \"affine\"], \"matern5_2\"), Xt, pred)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c015", "metadata": {}, "outputs": [], "source": [ diff --git a/warpings-categorical_WarpKriging_R.ipynb b/warpings-categorical_WarpKriging_R.ipynb new file mode 100644 index 0000000..869874e --- /dev/null +++ b/warpings-categorical_WarpKriging_R.ipynb @@ -0,0 +1,133 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "c000", + "metadata": {}, + "source": [ + "# R · `WarpKriging` — continuous warpings, a categorical input\n", + "\n", + "Branin on [0,1]² (n = 30 LHS points unless stated, Q² on 500 uniform test\n", + "points), through rlibkriging. Checks (error-level unless stated):\n", + "\n", + "* `kumaraswamy` on both inputs, then x₂ restricted to 5 levels with `categorical(5,2)`: finite predictions\n", + "* Q² floors (warn-level)\n", + "* `save` then `load`: the same predicted mean\n", + "\n", + "A class missing from the installed rlibkriging (release channel) is recorded as SKIP." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c001", + "metadata": {}, + "outputs": [], + "source": [ + "suppressPackageStartupMessages(library(rlibkriging))\n", + "source(\"tools/r/checks.R\")\n", + "s <- new_suite(\"warpings-categorical_WarpKriging_R\")\n", + "cat(\"rlibkriging\", as.character(packageVersion(\"rlibkriging\")), \"\\n\")\n", + "\n", + "branin <- function(U) { # on [0,1]^2\n", + " x1 <- 15 * U[, 1] - 5; x2 <- 15 * U[, 2]\n", + " (x2 - 5.1 / (4 * pi^2) * x1^2 + 5 / pi * x1 - 6)^2 + 10 * (1 - 1 / (8 * pi)) * cos(x1) + 10\n", + "}\n", + "set.seed(1)\n", + "lhs <- function(n, d) sapply(1:d, function(j) (sample(n) - runif(n)) / n)\n", + "X <- lhs(30, 2); y <- branin(X)\n", + "Xt <- matrix(runif(1000), ncol = 2); yt <- branin(Xt)\n", + "tmp <- tempdir()\n", + "has_class <- function(name) exists(name, mode = \"function\")\n", + "skip <- function(s, check, reason) {\n", + " s$checks[[length(s$checks) + 1]] <- data.frame(suite = s$name, lang = \"R\", check = check, severity = \"info\",\n", + " status = \"SKIP\", value = \"\", expected = \"\", detail = reason, stringsAsFactors = FALSE)\n", + " message(\"[SKIP] \", check, \": \", reason)\n", + "}\n", + "roundtrip_same_mean <- function(s, label, k, Xn) {\n", + " f <- file.path(tmp, paste0(gsub(\"[^A-Za-z0-9]\", \"_\", label), \".json\"))\n", + " check_runs(s, paste0(label, \": save + load\"), {\n", + " save(k, f)\n", + " k2 <- rlibkriging::load(f)\n", + " check_close(s, paste0(label, \": load(save(k)) predicts the same mean\"),\n", + " predict(k2, Xn, return_stdev = FALSE)$mean, predict(k, Xn, return_stdev = FALSE)$mean, rtol = 1e-10)\n", + " })\n", + "}" + ] + }, + { + "cell_type": "markdown", + "id": "c002", + "metadata": {}, + "source": [ + "## `WarpKriging`: continuous warpings, then a categorical input\n", + "\n", + "The categorical case: x₂ restricted to 5 levels (coded 0, …, 4), embedded in ℝ²." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c003", + "metadata": {}, + "outputs": [], + "source": [ + "if (has_class(\"WarpKriging\")) {\n", + " check_runs(s, \"WarpKriging(kumaraswamy, kumaraswamy): fit + predict\", {\n", + " kw <- WarpKriging(y, X, warping = c(\"kumaraswamy\", \"kumaraswamy\"), kernel = \"matern5_2\")\n", + " pw <- predict(kw, Xt, return_stdev = TRUE)\n", + " check_less(s, \"WarpKriging(kumaraswamy): non-finite predictions\", sum(!is.finite(c(pw$mean, pw$stdev))), 1)\n", + " add_metric(s, \"branin\", \"WarpKriging kumaraswamy\", q2 = q2(yt, pw$mean))\n", + " check_greater(s, \"WarpKriging(kumaraswamy): Q2\", q2(yt, pw$mean), 0.9, severity = \"warn\")\n", + " roundtrip_same_mean(s, \"WarpKriging(kumaraswamy)\", kw, Xt)\n", + " })\n", + " lev <- function(U) cbind(U[, 1], round(U[, 2] * 4)) # level codes 0..4\n", + " unlev <- function(L) cbind(L[, 1], L[, 2] / 4)\n", + " Xc <- lev(lhs(40, 2)); yc <- branin(unlev(Xc))\n", + " Xtc <- lev(Xt); ytc <- branin(unlev(Xtc))\n", + " check_runs(s, \"WarpKriging(none, categorical(5,2)): fit + predict\", {\n", + " kc <- WarpKriging(yc, Xc, warping = c(\"none\", \"categorical(5,2)\"), kernel = \"matern5_2\")\n", + " pc <- predict(kc, Xtc, return_stdev = TRUE)\n", + " check_less(s, \"WarpKriging(categorical): non-finite predictions\", sum(!is.finite(c(pc$mean, pc$stdev))), 1)\n", + " add_metric(s, \"branin, x2 categorical\", \"WarpKriging categorical\", q2 = q2(ytc, pc$mean))\n", + " check_greater(s, \"WarpKriging(categorical): Q2\", q2(ytc, pc$mean), 0.8, severity = \"warn\")\n", + " roundtrip_same_mean(s, \"WarpKriging(categorical)\", kc, Xtc)\n", + " })\n", + "} else skip(s, \"WarpKriging\", \"not in this rlibkriging\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c004", + "metadata": {}, + "outputs": [], + "source": [ + "me <- do.call(rbind, s$metrics)\n", + "print(me[me$metric == \"q2\", c(\"case\", \"package\", \"value\")], row.names = FALSE)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c005", + "metadata": {}, + "outputs": [], + "source": [ + "finalize(s)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "R", + "language": "R", + "name": "ir" + }, + "language_info": { + "name": "R" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/warpings-categorical_WarpKriging_julia.ipynb b/warpings-categorical_WarpKriging_julia.ipynb new file mode 100644 index 0000000..61daf60 --- /dev/null +++ b/warpings-categorical_WarpKriging_julia.ipynb @@ -0,0 +1,138 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "c000", + "metadata": {}, + "source": [ + "# Julia · `WarpKriging` — continuous warpings, a categorical input\n", + "\n", + "Branin on [0,1]² (n = 30 LHS points unless stated, Q² on 500 uniform test\n", + "points), through JLibKriging. Checks (error-level unless stated):\n", + "\n", + "* `kumaraswamy` on both inputs, then x₂ restricted to 5 levels with `categorical(5,2)`: finite predictions\n", + "* Q² floors (warn-level)\n", + "* `save` then `load`: the same predicted mean\n", + "\n", + "The same checks as `warpings-categorical_WarpKriging_R.ipynb`, with the Julia binding." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c001", + "metadata": {}, + "outputs": [], + "source": [ + "import JLibKriging as LK\n", + "using Random\n", + "include(\"tools/julia/checks.jl\")\n", + "s = Suite(\"warpings-categorical_WarpKriging_julia\")\n", + "\n", + "branin(U) = begin\n", + " x1 = 15 .* U[:, 1] .- 5; x2 = 15 .* U[:, 2]\n", + " (x2 .- 5.1 / (4π^2) .* x1 .^ 2 .+ 5 / π .* x1 .- 6) .^ 2 .+ 10 * (1 - 1 / (8π)) .* cos.(x1) .+ 10\n", + "end\n", + "rng = MersenneTwister(1)\n", + "lhs(n, d) = hcat([(randperm(rng, n) .- rand(rng, n)) ./ n for _ in 1:d]...)\n", + "X = lhs(30, 2); y = branin(X)\n", + "Xt = rand(rng, 500, 2); yt = branin(Xt)\n", + "sd(v) = sqrt(sum((v .- sum(v) / length(v)) .^ 2) / (length(v) - 1))\n", + "finite_count(v...) = count(!isfinite, vcat(v...))\n", + "\n", + "function check_runs!(f, s, check)\n", + " try\n", + " f()\n", + " catch e\n", + " add_check!(s, check, false; value=string(typeof(e)), expected=\"no error\", detail=first(sprint(showerror, e), 200))\n", + " end\n", + "end\n", + "\n", + "function roundtrip!(s, label, k, Xn)\n", + " check_runs!(s, \"$label: save + load\") do\n", + " f = joinpath(mktempdir(), \"k.json\")\n", + " LK.save(k, f)\n", + " k2 = LK.load(f)\n", + " check_close!(s, \"$label: load(save(k)) predicts the same mean\",\n", + " LK.predict(k2, Xn; return_stdev=false).mean, LK.predict(k, Xn; return_stdev=false).mean; rtol=1e-10)\n", + " end\n", + "end" + ] + }, + { + "cell_type": "markdown", + "id": "c002", + "metadata": {}, + "source": [ + "## `WarpKriging`: continuous warpings, then a categorical input\n", + "\n", + "The categorical case: x₂ restricted to 5 levels (coded 0, …, 4), embedded in ℝ²." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c003", + "metadata": {}, + "outputs": [], + "source": [ + "check_runs!(s, \"WarpKriging(kumaraswamy, kumaraswamy): fit + predict\") do\n", + " kw = LK.WarpKriging(y, X, [\"kumaraswamy\", \"kumaraswamy\"], \"matern5_2\")\n", + " pw = LK.predict(kw, Xt; return_stdev=true)\n", + " add_check!(s, \"WarpKriging(kumaraswamy): finite predictions\", finite_count(pw.mean, pw.stdev) == 0;\n", + " value=finite_count(pw.mean, pw.stdev), expected=\"0 non-finite\")\n", + " add_metric!(s, \"branin\", \"WarpKriging kumaraswamy\"; q2=q2(yt, pw.mean))\n", + " add_check!(s, \"WarpKriging(kumaraswamy): Q2\", q2(yt, pw.mean) > 0.9; value=q2(yt, pw.mean), expected=\"> 0.9\",\n", + " severity=\"warn\")\n", + " roundtrip!(s, \"WarpKriging(kumaraswamy)\", kw, Xt)\n", + "end\n", + "lev(U) = hcat(U[:, 1], round.(U[:, 2] .* 4))\n", + "unlev(L) = hcat(L[:, 1], L[:, 2] ./ 4)\n", + "Xc = lev(lhs(40, 2)); yc = branin(unlev(Xc))\n", + "Xtc = lev(Xt); ytc = branin(unlev(Xtc))\n", + "check_runs!(s, \"WarpKriging(none, categorical(5,2)): fit + predict\") do\n", + " kc = LK.WarpKriging(yc, Xc, [\"none\", \"categorical(5,2)\"], \"matern5_2\")\n", + " pc = LK.predict(kc, Xtc; return_stdev=true)\n", + " add_check!(s, \"WarpKriging(categorical): finite predictions\", finite_count(pc.mean, pc.stdev) == 0;\n", + " value=finite_count(pc.mean, pc.stdev), expected=\"0 non-finite\")\n", + " add_metric!(s, \"branin, x2 categorical\", \"WarpKriging categorical\"; q2=q2(ytc, pc.mean))\n", + " add_check!(s, \"WarpKriging(categorical): Q2\", q2(ytc, pc.mean) > 0.8; value=q2(ytc, pc.mean), expected=\"> 0.8\",\n", + " severity=\"warn\")\n", + " roundtrip!(s, \"WarpKriging(categorical)\", kc, Xtc)\n", + "end" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c004", + "metadata": {}, + "outputs": [], + "source": [ + "foreach(r -> r[5] == \"q2\" && println(rpad(r[3], 24), rpad(r[4], 28), r[6]), s.metrics)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c005", + "metadata": {}, + "outputs": [], + "source": [ + "finalize!(s)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Julia 1.11", + "language": "julia", + "name": "julia-1.11" + }, + "language_info": { + "name": "julia" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/warpings-categorical_WarpKriging_octave.ipynb b/warpings-categorical_WarpKriging_octave.ipynb new file mode 100644 index 0000000..877690f --- /dev/null +++ b/warpings-categorical_WarpKriging_octave.ipynb @@ -0,0 +1,144 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "c000", + "metadata": {}, + "source": [ + "# Octave · `WarpKriging` — continuous warpings, a categorical input\n", + "\n", + "Branin on [0,1]² (n = 30 LHS points unless stated, Q² on 500 uniform test\n", + "points), through mLibKriging. Checks (error-level unless stated):\n", + "\n", + "* `kumaraswamy` on both inputs, then x₂ restricted to 5 levels with `categorical(5,2)`: finite predictions\n", + "* Q² floors (warn-level)\n", + "* `save` then `load`: the same predicted mean\n", + "\n", + "The same checks as `warpings-categorical_WarpKriging_R.ipynb`, with the Octave binding.\n", + "\n", + "A class missing from the installed mLibKriging (release channel) is recorded as SKIP." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c001", + "metadata": {}, + "outputs": [], + "source": [ + "addpath(fullfile(pwd, \"tools\", \"octave\"));\n", + "s = lkb_suite(\"warpings-categorical_WarpKriging_octave\");\n", + "q2 = @(y, m) 1 - sum((y - m).^2) / sum((y - mean(y)).^2);\n", + "branin = @(U) ((15 * U(:, 2)) - 5.1 / (4 * pi^2) * (15 * U(:, 1) - 5).^2 + 5 / pi * (15 * U(:, 1) - 5) - 6).^2 ...\n", + " + 10 * (1 - 1 / (8 * pi)) * cos(15 * U(:, 1) - 5) + 10;\n", + "rand(\"seed\", 1); randn(\"seed\", 1);\n", + "function U = lhs(n, d)\n", + " U = zeros(n, d);\n", + " for j = 1:d, U(:, j) = (randperm(n)' - rand(n, 1)) / n; end\n", + "end\n", + "X = lhs(30, 2); y = branin(X);\n", + "Xt = rand(500, 2); yt = branin(Xt);\n", + "tmp = tempname(); mkdir(tmp);\n", + "has_class = @(name) exist(name) == 2; % a class file on the path\n", + "\n", + "function s = roundtrip(s, label, k, Xn, loader)\n", + " f = fullfile(tempdir(), [regexprep(label, \"[^A-Za-z0-9]\", \"_\") \".json\"]);\n", + " try\n", + " k.save(f);\n", + " k2 = loader(f);\n", + " m1 = k.predict(Xn, false, false, false);\n", + " m2 = k2.predict(Xn, false, false, false);\n", + " s = lkb_close(s, [label \": load(save(k)) predicts the same mean\"], m2, m1, 1e-10);\n", + " catch err\n", + " s = lkb_check(s, [label \": save + load\"], false, \"error\", \"no error\", err.message);\n", + " end\n", + "end\n", + "\n", + "function s = skip(s, check, reason)\n", + " s.checks(end + 1, :) = {s.name, \"octave\", check, \"info\", \"SKIP\", \"\", \"\", reason};\n", + " printf(\"[SKIP] %s: %s\\n\", check, reason);\n", + "end" + ] + }, + { + "cell_type": "markdown", + "id": "c002", + "metadata": {}, + "source": [ + "## `WarpKriging`: continuous warpings, then a categorical input\n", + "\n", + "The categorical case: x₂ restricted to 5 levels (coded 0, …, 4), embedded in ℝ²." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c003", + "metadata": {}, + "outputs": [], + "source": [ + "if has_class(\"WarpKriging\")\n", + " try\n", + " kw = WarpKriging(y, X, {\"kumaraswamy\", \"kumaraswamy\"}, \"matern5_2\");\n", + " [mw, sw] = kw.predict(Xt, true, false, false);\n", + " s = lkb_check(s, \"WarpKriging(kumaraswamy): finite predictions\", all(isfinite([mw; sw])), sum(~isfinite([mw; sw])), \"0 non-finite\");\n", + " s = lkb_metric(s, \"branin\", \"WarpKriging kumaraswamy\", \"q2\", q2(yt, mw));\n", + " s = lkb_check(s, \"WarpKriging(kumaraswamy): Q2\", q2(yt, mw) > 0.9, q2(yt, mw), \"> 0.9\", \"\", \"warn\");\n", + " s = roundtrip(s, \"WarpKriging(kumaraswamy)\", kw, Xt, @load_kriging);\n", + " catch err\n", + " s = lkb_check(s, \"WarpKriging(kumaraswamy): fit + predict\", false, \"error\", \"no error\", err.message);\n", + " end\n", + " lev = @(U) [U(:, 1), round(U(:, 2) * 4)];\n", + " unlev = @(L) [L(:, 1), L(:, 2) / 4];\n", + " Xc = lev(lhs(40, 2)); yc = branin(unlev(Xc));\n", + " Xtc = lev(Xt); ytc = branin(unlev(Xtc));\n", + " try\n", + " kc = WarpKriging(yc, Xc, {\"none\", \"categorical(5,2)\"}, \"matern5_2\");\n", + " [mc, sc] = kc.predict(Xtc, true, false, false);\n", + " s = lkb_check(s, \"WarpKriging(categorical): finite predictions\", all(isfinite([mc; sc])), sum(~isfinite([mc; sc])), \"0 non-finite\");\n", + " s = lkb_metric(s, \"branin, x2 categorical\", \"WarpKriging categorical\", \"q2\", q2(ytc, mc));\n", + " s = lkb_check(s, \"WarpKriging(categorical): Q2\", q2(ytc, mc) > 0.8, q2(ytc, mc), \"> 0.8\", \"\", \"warn\");\n", + " s = roundtrip(s, \"WarpKriging(categorical)\", kc, Xtc, @load_kriging);\n", + " catch err\n", + " s = lkb_check(s, \"WarpKriging(categorical): fit + predict\", false, \"error\", \"no error\", err.message);\n", + " end\n", + "else\n", + " s = skip(s, \"WarpKriging\", \"not in this mLibKriging\");\n", + "end" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c004", + "metadata": {}, + "outputs": [], + "source": [ + "q = strcmp(s.metrics(:, 5), \"q2\");\n", + "disp(s.metrics(q, [3, 4, 6]))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c005", + "metadata": {}, + "outputs": [], + "source": [ + "lkb_finalize(s);" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Octave", + "language": "octave", + "name": "octave" + }, + "language_info": { + "name": "octave" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From 3dd292da7a9510ecb8557e590f78d792bb077013 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 14 Oct 2026 14:19:12 +0000 Subject: [PATCH 05/16] One binding per theme and Kriging type; references and asymptotics; Colab and Binder links Notebooks - the 18 per-class R/Octave/Julia notebooks removed: their themes are covered, in more depth, by the Python notebooks; kept outside Python: the comparisons whose reference is specific to a language (DiceKriging/RobustGaSP, STK, GaussianProcesses.jl) and the cross-binding notebooks; - cross-binding protocol extended to every class and fit method (Branin): nugget and known noise at fixed hyperparameters, Vecchia and Nystrom at fixed theta (error level), nugget, WarpKriging, MLPKriging and NestedKriging fits (warn level); notebooks renamed cross-binding(-reference)_all-classes_; method= guarded for releases without it, and a quantity missing on the Python side is a warning; - large-n_Kriging-vecchia-nystrom_python: convergence to the exact model at fixed theta as m and k grow, down to m = n - 1 and k = n (log-log plot, checks); - aggregations-large-n_NestedKriging_python: from 1 group (the exact predictor) to 32, NK vs rBCM, gap and fit time; - demo plots: conditional paths before/after update_simulate, predict time vs q with O(q) / O(q^2) references, Welch screening, Q2 per package, MLPKriging Q2; noisy-data: single start vs 10 restarts. Colab / Binder - tools/notebook_setup.py, inserted by py2nb as the first code cell of every Python notebook: a no-op in the repository; elsewhere it clones the repository and installs the released pylibkriging, with an explicit message on Python 3.13 (PyPI has wheels for 3.10-3.12 only; Colab's default runtime is 3.13); - .binder/: Python 3.12 and pylibkriging; - README: Colab and Binder links for the Python notebooks, and their limits. Checked locally: every Python notebook against master (ef052e1) and against the PyPI release 1.2.2 (Python 3.12); a Colab-like fresh environment (notebook outside the repository) and a Binder-like one; the R and Octave cross-binding notebooks (49/49 quantities equal to Python) and vs-STK. Not run: Julia, vs-DiceKriging (CRAN unreachable here), Colab and Binder themselves. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01QiFEE514dpwS57mcFWDeZG --- .binder/requirements.txt | 10 + .binder/runtime.txt | 1 + .github/workflows/notebooks.yml | 2 +- README.md | 115 +++++---- ...gations-large-n_NestedKriging_python.ipynb | 107 ++++++++- aggregations_NestedKriging_R.ipynb | 125 ---------- aggregations_NestedKriging_julia.ipynb | 128 ---------- aggregations_NestedKriging_octave.ipynb | 138 ----------- cross-binding-reference_Kriging_python.ipynb | 121 ---------- ...binding-reference_all-classes_python.ipynb | 223 ++++++++++++++++++ ...ipynb => cross-binding_all-classes_R.ipynb | 62 ++++- ...b => cross-binding_all-classes_julia.ipynb | 60 ++++- ... => cross-binding_all-classes_octave.ipynb | 68 +++++- ...l-projected-branin_MLPKriging_python.ipynb | 49 +++- deep-kernel_MLPKriging_R.ipynb | 124 ---------- deep-kernel_MLPKriging_julia.ipynb | 126 ---------- deep-kernel_MLPKriging_octave.ipynb | 132 ----------- ...n-normalize-screening_Kriging_python.ipynb | 51 +++- ...ation-simulate-update-save_Kriging_R.ipynb | 131 ---------- ...n-simulate-update-save_Kriging_julia.ipynb | 134 ----------- ...-simulate-update-save_Kriging_octave.ipynb | 134 ----------- ...-trends-interpolation_Kriging_python.ipynb | 59 ++++- large-n_Kriging-vecchia-nystrom_R.ipynb | 120 ---------- large-n_Kriging-vecchia-nystrom_julia.ipynb | 126 ---------- large-n_Kriging-vecchia-nystrom_octave.ipynb | 128 ---------- large-n_Kriging-vecchia-nystrom_python.ipynb | 103 +++++++- noisy-data_Kriging-nugget-noise_R.ipynb | 137 ----------- noisy-data_Kriging-nugget-noise_julia.ipynb | 141 ----------- noisy-data_Kriging-nugget-noise_octave.ipynb | 140 ----------- noisy-data_Kriging-nugget-noise_python.ipynb | 58 ++++- ...-LL-LOO-LMP-gradients_Kriging_python.ipynb | 59 ++++- ...stness-invalid-inputs_Kriging_python.ipynb | 67 +++++- ...pickle-copy-accessors_Kriging_python.ipynb | 49 +++- sklearn-estimators_Kriging_python.ipynb | 45 +++- ...ons-reference-values_no-model_python.ipynb | 51 +++- tools/known_issues.csv | 6 - tools/notebook_setup.py | 26 ++ tools/py2nb.py | 8 +- tools/report.py | 12 +- tools/run_notebooks.py | 2 +- ...gregations-large-n_NestedKriging_python.py | 46 ++++ tools/src/aggregations_NestedKriging_R.R | 70 ------ tools/src/aggregations_NestedKriging_julia.jl | 73 ------ tools/src/aggregations_NestedKriging_octave.m | 83 ------- .../cross-binding-reference_Kriging_python.py | 78 ------ ...ss-binding-reference_all-classes_python.py | 126 ++++++++++ tools/src/cross-binding_Kriging_R.R | 49 ---- tools/src/cross-binding_Kriging_julia.jl | 52 ---- tools/src/cross-binding_Kriging_octave.m | 60 ----- tools/src/cross-binding_all-classes_R.R | 82 +++++++ tools/src/cross-binding_all-classes_julia.jl | 85 +++++++ tools/src/cross-binding_all-classes_octave.m | 101 ++++++++ ...rnel-projected-branin_MLPKriging_python.py | 6 +- tools/src/deep-kernel_MLPKriging_R.R | 69 ------ tools/src/deep-kernel_MLPKriging_julia.jl | 71 ------ tools/src/deep-kernel_MLPKriging_octave.m | 77 ------ ...sion-normalize-screening_Kriging_python.py | 6 + ...rpolation-simulate-update-save_Kriging_R.R | 76 ------ ...tion-simulate-update-save_Kriging_julia.jl | 79 ------- ...tion-simulate-update-save_Kriging_octave.m | 79 ------- tools/src/large-n_Kriging-vecchia-nystrom_R.R | 65 ----- .../large-n_Kriging-vecchia-nystrom_julia.jl | 71 ------ .../large-n_Kriging-vecchia-nystrom_octave.m | 73 ------ .../large-n_Kriging-vecchia-nystrom_python.py | 46 ++++ tools/src/noisy-data_Kriging-nugget-noise_R.R | 82 ------- .../noisy-data_Kriging-nugget-noise_julia.jl | 86 ------- .../noisy-data_Kriging-nugget-noise_octave.m | 85 ------- .../noisy-data_Kriging-nugget-noise_python.py | 5 + ...obustness-invalid-inputs_Kriging_python.py | 8 + tools/src/update-simulate_Kriging_python.py | 26 ++ ...klearn-SMT-OpenTURNS-GPy_Kriging_python.py | 7 + .../src/warpings-categorical_WarpKriging_R.R | 78 ------ .../warpings-categorical_WarpKriging_julia.jl | 83 ------- .../warpings-categorical_WarpKriging_octave.m | 89 ------- update-simulate_Kriging_python.ipynb | 91 ++++++- ...arn-SMT-OpenTURNS-GPy_Kriging_python.ipynb | 71 +++++- ...tegorical-ordinal_WarpKriging_python.ipynb | 63 +++-- warpings-categorical_WarpKriging_R.ipynb | 133 ----------- warpings-categorical_WarpKriging_julia.ipynb | 138 ----------- warpings-categorical_WarpKriging_octave.ipynb | 144 ----------- 80 files changed, 1858 insertions(+), 4332 deletions(-) create mode 100644 .binder/requirements.txt create mode 100644 .binder/runtime.txt delete mode 100644 aggregations_NestedKriging_R.ipynb delete mode 100644 aggregations_NestedKriging_julia.ipynb delete mode 100644 aggregations_NestedKriging_octave.ipynb delete mode 100644 cross-binding-reference_Kriging_python.ipynb create mode 100644 cross-binding-reference_all-classes_python.ipynb rename cross-binding_Kriging_R.ipynb => cross-binding_all-classes_R.ipynb (50%) rename cross-binding_Kriging_julia.ipynb => cross-binding_all-classes_julia.ipynb (53%) rename cross-binding_Kriging_octave.ipynb => cross-binding_all-classes_octave.ipynb (50%) delete mode 100644 deep-kernel_MLPKriging_R.ipynb delete mode 100644 deep-kernel_MLPKriging_julia.ipynb delete mode 100644 deep-kernel_MLPKriging_octave.ipynb delete mode 100644 interpolation-simulate-update-save_Kriging_R.ipynb delete mode 100644 interpolation-simulate-update-save_Kriging_julia.ipynb delete mode 100644 interpolation-simulate-update-save_Kriging_octave.ipynb delete mode 100644 large-n_Kriging-vecchia-nystrom_R.ipynb delete mode 100644 large-n_Kriging-vecchia-nystrom_julia.ipynb delete mode 100644 large-n_Kriging-vecchia-nystrom_octave.ipynb delete mode 100644 noisy-data_Kriging-nugget-noise_R.ipynb delete mode 100644 noisy-data_Kriging-nugget-noise_julia.ipynb delete mode 100644 noisy-data_Kriging-nugget-noise_octave.ipynb create mode 100644 tools/notebook_setup.py delete mode 100644 tools/src/aggregations_NestedKriging_R.R delete mode 100644 tools/src/aggregations_NestedKriging_julia.jl delete mode 100644 tools/src/aggregations_NestedKriging_octave.m delete mode 100644 tools/src/cross-binding-reference_Kriging_python.py create mode 100644 tools/src/cross-binding-reference_all-classes_python.py delete mode 100644 tools/src/cross-binding_Kriging_R.R delete mode 100644 tools/src/cross-binding_Kriging_julia.jl delete mode 100644 tools/src/cross-binding_Kriging_octave.m create mode 100644 tools/src/cross-binding_all-classes_R.R create mode 100644 tools/src/cross-binding_all-classes_julia.jl create mode 100644 tools/src/cross-binding_all-classes_octave.m delete mode 100644 tools/src/deep-kernel_MLPKriging_R.R delete mode 100644 tools/src/deep-kernel_MLPKriging_julia.jl delete mode 100644 tools/src/deep-kernel_MLPKriging_octave.m delete mode 100644 tools/src/interpolation-simulate-update-save_Kriging_R.R delete mode 100644 tools/src/interpolation-simulate-update-save_Kriging_julia.jl delete mode 100644 tools/src/interpolation-simulate-update-save_Kriging_octave.m delete mode 100644 tools/src/large-n_Kriging-vecchia-nystrom_R.R delete mode 100644 tools/src/large-n_Kriging-vecchia-nystrom_julia.jl delete mode 100644 tools/src/large-n_Kriging-vecchia-nystrom_octave.m delete mode 100644 tools/src/noisy-data_Kriging-nugget-noise_R.R delete mode 100644 tools/src/noisy-data_Kriging-nugget-noise_julia.jl delete mode 100644 tools/src/noisy-data_Kriging-nugget-noise_octave.m delete mode 100644 tools/src/warpings-categorical_WarpKriging_R.R delete mode 100644 tools/src/warpings-categorical_WarpKriging_julia.jl delete mode 100644 tools/src/warpings-categorical_WarpKriging_octave.m delete mode 100644 warpings-categorical_WarpKriging_R.ipynb delete mode 100644 warpings-categorical_WarpKriging_julia.ipynb delete mode 100644 warpings-categorical_WarpKriging_octave.ipynb diff --git a/.binder/requirements.txt b/.binder/requirements.txt new file mode 100644 index 0000000..0a370dc --- /dev/null +++ b/.binder/requirements.txt @@ -0,0 +1,10 @@ +# Binder (mybinder.org) environment for the Python notebooks: the released pylibkriging +# (PyPI wheels exist for CPython 3.10 to 3.12, hence runtime.txt) and lkbench's +# dependencies; lkbench itself is found in tools/ by the notebooks' setup cell, and the +# comparison packages are installed by the notebooks that use them. +pylibkriging +numpy +scipy +pandas +matplotlib +jinja2 diff --git a/.binder/runtime.txt b/.binder/runtime.txt new file mode 100644 index 0000000..d2aca3a --- /dev/null +++ b/.binder/runtime.txt @@ -0,0 +1 @@ +python-3.12 diff --git a/.github/workflows/notebooks.yml b/.github/workflows/notebooks.yml index a51f081..d9bcb92 100644 --- a/.github/workflows/notebooks.yml +++ b/.github/workflows/notebooks.yml @@ -18,7 +18,7 @@ name: notebooks # # A notebook job fails when one of its error-level checks fails; the report job # fails when any notebook failed or crashed, or when the bindings disagree -# (cross-binding protocol, cross-binding-reference_Kriging_python.ipynb). +# (cross-binding protocol, cross-binding-reference_all-classes_python.ipynb). # The binding installations are the scripts of tools/install/, usable locally too. on: diff --git a/README.md b/README.md index ce7e202..55db8df 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,8 @@ and aggregates the findings of every language into one report. The notebooks are at the root of the repository, named `__.ipynb`; everything else (notebook sources, helpers, binding installation, CI scripts, known issues, baselines) is under -[`tools/`](tools), and the shared datasets under [`data/`](data). +[`tools/`](tools), the shared datasets under [`data/`](data), and the Binder +environment under [`.binder/`](.binder). ## Notebooks @@ -25,64 +26,41 @@ All the executed notebooks of a run, in one zip: the aggregated report with all CSV results: [report-master](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/report-master.zip). +One binding per theme and Kriging type: Python, the most complete binding, unless the +reference is specific to another language (DiceKriging and RobustGaSP in R, STK in +Octave, GaussianProcesses.jl in Julia). The other bindings are checked against +Python on every model class and fit method by the cross-binding notebooks. + ### Python (`pylibkriging`) -| notebook | kriging type | theme: problems, features | compared with | executed | -|---|---|---|---|---| -| [test-functions-reference-values_no-model_python](test-functions-reference-values_no-model_python.ipynb) | — | the 21 test functions of [`tools/lkbench/functions.py`](tools/lkbench/functions.py) against published optima / ranges | — | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-test-functions-reference-values_no-model_python.zip) | -| [kernels-trends-interpolation_Kriging_python](kernels-trends-interpolation_Kriging_python.ipynb) | `Kriging` | 4 kernels × 4 trends on Forrester, Gramacy & Lee, Branin, six-hump camel, log-Goldstein-Price, Lim, Ishigami, Hartmann 3: interpolation, `predict` (covariance, derivatives) | finite differences | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-kernels-trends-interpolation_Kriging_python.zip) | -| [objectives-LL-LOO-LMP-gradients_Kriging_python](objectives-LL-LOO-LMP-gradients_Kriging_python.ipynb) | `Kriging` | `LL` / `LOO` / `LMP` objectives and gradients, optimality of θ̂, `leaveOneOutVec`, multistart (Branin, Hartmann 3, Ishigami, Friedman) | finite differences, explicit refits | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-objectives-LL-LOO-LMP-gradients_Kriging_python.zip) | -| [update-simulate_Kriging_python](update-simulate_Kriging_python.ipynb) | `Kriging` | `update`, `simulate`, `update_simulate` (Branin, Hartmann 3, Hartmann 6) | fits from scratch, moments | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-update-simulate_Kriging_python.zip) | -| [high-dimension-normalize-screening_Kriging_python](high-dimension-normalize-screening_Kriging_python.ipynb) | `Kriging` | `normalize`, screening, 6D to 20D engineering functions (OTL circuit, piston, borehole, robot arm, Dette-Pepelyshev, wing weight, Welch) | unit-cube fit | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-high-dimension-normalize-screening_Kriging_python.zip) | -| [robustness-invalid-inputs_Kriging_python](robustness-invalid-inputs_Kriging_python.ipynb) | `Kriging` | invalid inputs, duplicates, constant y, ill-conditioning, scale equivariance, extrapolation, cost in q (Branin) | — | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-robustness-invalid-inputs_Kriging_python.zip) | -| [save-load-pickle-copy-accessors_Kriging_python](save-load-pickle-copy-accessors_Kriging_python.ipynb) | `Kriging` | `save`/`load`, pickle, copy, accessors, for every noise model, objective and fit method (Hartmann 3) | — | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-save-load-pickle-copy-accessors_Kriging_python.zip) | -| [sklearn-estimators_Kriging_python](sklearn-estimators_Kriging_python.ipynb) | `Kriging` | `pylibkriging.sklearn.KrigingRegressor`: `Pipeline`, `GridSearchCV`, `clone`, `sample_y` (Hartmann 3) | the native class | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-sklearn-estimators_Kriging_python.zip) | -| [vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python](vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python.ipynb) | `Kriging` | default fits on 10 functions, 2D to 10D | scikit-learn, SMT, OpenTURNS, GPy (exact equality at fixed hyperparameters + accuracy/cost) | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python.zip) | -| [cross-binding-reference_Kriging_python](cross-binding-reference_Kriging_python.ipynb) | `Kriging` | the cross-binding protocol on the datasets of `data/`: the Python reference | the other bindings (report) | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-cross-binding-reference_Kriging_python.zip) | -| [noisy-data_Kriging-nugget-noise_python](noisy-data_Kriging-nugget-noise_python.ipynb) | `Kriging` with `noise="nugget"` or variances | estimated nugget, heteroscedastic noise, calibration (Forrester, Friedman, Gramacy & Lee, Branin with replicates) | true noise, replicate averaging | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-noisy-data_Kriging-nugget-noise_python.zip) | -| [large-n_Kriging-vecchia-nystrom_python](large-n_Kriging-vecchia-nystrom_python.ipynb) | `Kriging` with `method="vecchia(m)"` / `"nystrom(k)"` | large n (Hartmann 3, OTL circuit, n up to 5000), limit cases, `subsetOfData` | exact fit | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-large-n_Kriging-vecchia-nystrom_python.zip) | -| [warpings-categorical-ordinal_WarpKriging_python](warpings-categorical-ordinal_WarpKriging_python.ipynb) | `WarpKriging` | affine, boxcox, kumaraswamy, knots, neural_mono, categorical, ordinal warpings (Gramacy & Lee 2012 / 2008, mixed categorical Branin, ordinal Hartmann 3); save/load, pickle | plain `Kriging` | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-warpings-categorical-ordinal_WarpKriging_python.zip) | -| [deep-kernel-projected-branin_MLPKriging_python](deep-kernel-projected-branin_MLPKriging_python.ipynb) | `MLPKriging` | deep kernel on a 10D function with a 2D structure, activations, multistart; save/load, pickle | plain `Kriging` | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-deep-kernel-projected-branin_MLPKriging_python.zip) | -| [aggregations-large-n_NestedKriging_python](aggregations-large-n_NestedKriging_python.ipynb) | `NestedKriging` | `NK`, `PoE`, `gPoE`, `BCM`, `rBCM` aggregations, large n; save/load, pickle | exact fit | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-aggregations-large-n_NestedKriging_python.zip) | - -### R (`rlibkriging`) - -| notebook | kriging type | theme: problems, features | compared with | executed | -|---|---|---|---|---| -| [interpolation-simulate-update-save_Kriging_R](interpolation-simulate-update-save_Kriging_R.ipynb) | `Kriging` | interpolation, conditional simulation, `update`, `save`/`load` (Branin) | fit on all points | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-interpolation-simulate-update-save_Kriging_R.zip) | -| [cross-binding_Kriging_R](cross-binding_Kriging_R.ipynb) | `Kriging` | the cross-binding protocol, R side | Python (report) | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-cross-binding_Kriging_R.zip) | -| [vs-DiceKriging-RobustGaSP_Kriging_R](vs-DiceKriging-RobustGaSP_Kriging_R.ipynb) | `Kriging` | default fits (Branin, Hartmann 3, Borehole) | DiceKriging (exact equality), RobustGaSP | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-vs-DiceKriging-RobustGaSP_Kriging_R.zip) | -| [noisy-data_Kriging-nugget-noise_R](noisy-data_Kriging-nugget-noise_R.ipynb) | `Kriging` with nugget or known variances | estimated nugget (10 restarts vs single start), known noise variances (Branin + noise) | — | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-noisy-data_Kriging-nugget-noise_R.zip) | -| [large-n_Kriging-vecchia-nystrom_R](large-n_Kriging-vecchia-nystrom_R.ipynb) | `Kriging` with Vecchia / Nyström | approximate fits, `method()`, prediction with the fit method (Branin, n = 400) | — | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-large-n_Kriging-vecchia-nystrom_R.zip) | -| [warpings-categorical_WarpKriging_R](warpings-categorical_WarpKriging_R.ipynb) | `WarpKriging` | kumaraswamy warpings, a categorical input (Branin) | — | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-warpings-categorical_WarpKriging_R.zip) | -| [deep-kernel_MLPKriging_R](deep-kernel_MLPKriging_R.ipynb) | `MLPKriging` | joint feature map, multistart vs default optimizer (Branin) | — | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-deep-kernel_MLPKriging_R.zip) | -| [aggregations_NestedKriging_R](aggregations_NestedKriging_R.ipynb) | `NestedKriging` | `NK` and `PoE` aggregations, interpolation, `save`/`load` (Branin, n = 400) | — | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-aggregations_NestedKriging_R.zip) | - -### Octave (`mLibKriging`) - -| notebook | kriging type | theme: problems, features | compared with | executed | -|---|---|---|---|---| -| [interpolation-simulate-update-save_Kriging_octave](interpolation-simulate-update-save_Kriging_octave.ipynb) | `Kriging` | interpolation, conditional simulation, `update`, `save`/`load` (Branin) | fit on all points | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-interpolation-simulate-update-save_Kriging_octave.zip) | -| [cross-binding_Kriging_octave](cross-binding_Kriging_octave.ipynb) | `Kriging` | the cross-binding protocol, Octave side | Python (report) | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-cross-binding_Kriging_octave.zip) | -| [vs-STK_Kriging_octave](vs-STK_Kriging_octave.ipynb) | `Kriging` | default fits (Branin, Hartmann 3, Borehole) | STK (exact equality, Gaussian kernel) | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-vs-STK_Kriging_octave.zip) | -| [noisy-data_Kriging-nugget-noise_octave](noisy-data_Kriging-nugget-noise_octave.ipynb) | `Kriging` with nugget or known variances | estimated nugget (10 restarts vs single start), known noise variances (Branin + noise) | — | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-noisy-data_Kriging-nugget-noise_octave.zip) | -| [large-n_Kriging-vecchia-nystrom_octave](large-n_Kriging-vecchia-nystrom_octave.ipynb) | `Kriging` with Vecchia / Nyström | approximate fits, `method()`, prediction with the fit method (Branin, n = 400) | — | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-large-n_Kriging-vecchia-nystrom_octave.zip) | -| [warpings-categorical_WarpKriging_octave](warpings-categorical_WarpKriging_octave.ipynb) | `WarpKriging` | kumaraswamy warpings, a categorical input (Branin) | — | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-warpings-categorical_WarpKriging_octave.zip) | -| [deep-kernel_MLPKriging_octave](deep-kernel_MLPKriging_octave.ipynb) | `MLPKriging` | joint feature map, multistart vs default optimizer (Branin) | — | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-deep-kernel_MLPKriging_octave.zip) | -| [aggregations_NestedKriging_octave](aggregations_NestedKriging_octave.ipynb) | `NestedKriging` | `NK` and `PoE` aggregations, interpolation, `save`/`load` (Branin, n = 400) | — | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-aggregations_NestedKriging_octave.zip) | - -### Julia (`JLibKriging`) - -| notebook | kriging type | theme: problems, features | compared with | executed | -|---|---|---|---|---| -| [interpolation-simulate-update-save_Kriging_julia](interpolation-simulate-update-save_Kriging_julia.ipynb) | `Kriging` | interpolation, conditional simulation, `update`, `save`/`load` (Branin) | fit on all points | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-interpolation-simulate-update-save_Kriging_julia.zip) | -| [cross-binding_Kriging_julia](cross-binding_Kriging_julia.ipynb) | `Kriging` | the cross-binding protocol, Julia side | Python (report) | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-cross-binding_Kriging_julia.zip) | -| [vs-GaussianProcesses_Kriging_julia](vs-GaussianProcesses_Kriging_julia.ipynb) | `Kriging` | default fits (Branin, Hartmann 3, Borehole) | GaussianProcesses.jl | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-vs-GaussianProcesses_Kriging_julia.zip) | -| [noisy-data_Kriging-nugget-noise_julia](noisy-data_Kriging-nugget-noise_julia.ipynb) | `Kriging` with nugget or known variances | estimated nugget (10 restarts vs single start), known noise variances (Branin + noise) | — | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-noisy-data_Kriging-nugget-noise_julia.zip) | -| [large-n_Kriging-vecchia-nystrom_julia](large-n_Kriging-vecchia-nystrom_julia.ipynb) | `Kriging` with Vecchia / Nyström | approximate fits, `method()`, prediction with the fit method (Branin, n = 400) | — | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-large-n_Kriging-vecchia-nystrom_julia.zip) | -| [warpings-categorical_WarpKriging_julia](warpings-categorical_WarpKriging_julia.ipynb) | `WarpKriging` | kumaraswamy warpings, a categorical input (Branin) | — | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-warpings-categorical_WarpKriging_julia.zip) | -| [deep-kernel_MLPKriging_julia](deep-kernel_MLPKriging_julia.ipynb) | `MLPKriging` | joint feature map, multistart vs default optimizer (Branin) | — | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-deep-kernel_MLPKriging_julia.zip) | -| [aggregations_NestedKriging_julia](aggregations_NestedKriging_julia.ipynb) | `NestedKriging` | `NK` and `PoE` aggregations, interpolation, `save`/`load` (Branin, n = 400) | — | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-aggregations_NestedKriging_julia.zip) | +| notebook | kriging type | theme: problems, features | reference: compared with | executed | open in | +|---|---|---|---|---|---| +| [test-functions-reference-values_no-model_python](test-functions-reference-values_no-model_python.ipynb) | — | the 21 test functions of [`tools/lkbench/functions.py`](tools/lkbench/functions.py), plotted | published optima / values | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-test-functions-reference-values_no-model_python.zip) | [Colab](https://colab.research.google.com/github/libKriging/benchmarks/blob/main/test-functions-reference-values_no-model_python.ipynb) · [Binder](https://mybinder.org/v2/gh/libKriging/benchmarks/main?labpath=test-functions-reference-values_no-model_python.ipynb) | +| [kernels-trends-interpolation_Kriging_python](kernels-trends-interpolation_Kriging_python.ipynb) | `Kriging` | 4 kernels × 4 trends on 8 functions (1D to 3D): interpolation, `predict` with covariance and derivatives | finite differences, Q² floors | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-kernels-trends-interpolation_Kriging_python.zip) | [Colab](https://colab.research.google.com/github/libKriging/benchmarks/blob/main/kernels-trends-interpolation_Kriging_python.ipynb) · [Binder](https://mybinder.org/v2/gh/libKriging/benchmarks/main?labpath=kernels-trends-interpolation_Kriging_python.ipynb) | +| [objectives-LL-LOO-LMP-gradients_Kriging_python](objectives-LL-LOO-LMP-gradients_Kriging_python.ipynb) | `Kriging` | `LL` / `LOO` / `LMP` objectives and gradients, optimality of θ̂, `leaveOneOutVec`, multistart | finite differences, explicit refits | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-objectives-LL-LOO-LMP-gradients_Kriging_python.zip) | [Colab](https://colab.research.google.com/github/libKriging/benchmarks/blob/main/objectives-LL-LOO-LMP-gradients_Kriging_python.ipynb) · [Binder](https://mybinder.org/v2/gh/libKriging/benchmarks/main?labpath=objectives-LL-LOO-LMP-gradients_Kriging_python.ipynb) | +| [update-simulate_Kriging_python](update-simulate_Kriging_python.ipynb) | `Kriging` | `update`, `simulate`, `update_simulate`, conditional paths in pictures | fit from scratch, Monte Carlo moments | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-update-simulate_Kriging_python.zip) | [Colab](https://colab.research.google.com/github/libKriging/benchmarks/blob/main/update-simulate_Kriging_python.ipynb) · [Binder](https://mybinder.org/v2/gh/libKriging/benchmarks/main?labpath=update-simulate_Kriging_python.ipynb) | +| [high-dimension-normalize-screening_Kriging_python](high-dimension-normalize-screening_Kriging_python.ipynb) | `Kriging` | 6D to 20D engineering functions, `normalize`, screening (Welch) | explicit rescaling, known active inputs | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-high-dimension-normalize-screening_Kriging_python.zip) | [Colab](https://colab.research.google.com/github/libKriging/benchmarks/blob/main/high-dimension-normalize-screening_Kriging_python.ipynb) · [Binder](https://mybinder.org/v2/gh/libKriging/benchmarks/main?labpath=high-dimension-normalize-screening_Kriging_python.ipynb) | +| [robustness-invalid-inputs_Kriging_python](robustness-invalid-inputs_Kriging_python.ipynb) | `Kriging` | invalid and degenerate inputs, ill-conditioning, scale equivariance, extrapolation, cost in q | equivariance identities, O(q) cost | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-robustness-invalid-inputs_Kriging_python.zip) | [Colab](https://colab.research.google.com/github/libKriging/benchmarks/blob/main/robustness-invalid-inputs_Kriging_python.ipynb) · [Binder](https://mybinder.org/v2/gh/libKriging/benchmarks/main?labpath=robustness-invalid-inputs_Kriging_python.ipynb) | +| [save-load-pickle-copy-accessors_Kriging_python](save-load-pickle-copy-accessors_Kriging_python.ipynb) | `Kriging` | `save`/`load`, pickle, copy, accessors, for every noise model, objective and fit method | the original model | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-save-load-pickle-copy-accessors_Kriging_python.zip) | [Colab](https://colab.research.google.com/github/libKriging/benchmarks/blob/main/save-load-pickle-copy-accessors_Kriging_python.ipynb) · [Binder](https://mybinder.org/v2/gh/libKriging/benchmarks/main?labpath=save-load-pickle-copy-accessors_Kriging_python.ipynb) | +| [sklearn-estimators_Kriging_python](sklearn-estimators_Kriging_python.ipynb) | `Kriging` | `pylibkriging.sklearn`: `Pipeline`, `GridSearchCV`, `clone`, `sample_y` | the native class | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-sklearn-estimators_Kriging_python.zip) | [Colab](https://colab.research.google.com/github/libKriging/benchmarks/blob/main/sklearn-estimators_Kriging_python.ipynb) · [Binder](https://mybinder.org/v2/gh/libKriging/benchmarks/main?labpath=sklearn-estimators_Kriging_python.ipynb) | +| [vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python](vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python.ipynb) | `Kriging` | default fits on 10 functions, 2D to 10D | scikit-learn, SMT, OpenTURNS, GPy (equality at fixed hyperparameters, accuracy, cost) | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python.zip) | [Colab](https://colab.research.google.com/github/libKriging/benchmarks/blob/main/vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python.ipynb) · [Binder](https://mybinder.org/v2/gh/libKriging/benchmarks/main?labpath=vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python.ipynb) | +| [noisy-data_Kriging-nugget-noise_python](noisy-data_Kriging-nugget-noise_python.ipynb) | `Kriging` with `noise="nugget"` or variances | estimated nugget, heteroscedastic noise, calibration, single start vs multistart | true noise, replicate averaging, interpolating fit | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-noisy-data_Kriging-nugget-noise_python.zip) | [Colab](https://colab.research.google.com/github/libKriging/benchmarks/blob/main/noisy-data_Kriging-nugget-noise_python.ipynb) · [Binder](https://mybinder.org/v2/gh/libKriging/benchmarks/main?labpath=noisy-data_Kriging-nugget-noise_python.ipynb) | +| [large-n_Kriging-vecchia-nystrom_python](large-n_Kriging-vecchia-nystrom_python.ipynb) | `Kriging` with `method="vecchia(m)"` / `"nystrom(k)"` | large n (up to 5000), convergence to the exact model as m, k grow, `subsetOfData` | exact model (limit m = n − 1, k = n), exact fit | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-large-n_Kriging-vecchia-nystrom_python.zip) | [Colab](https://colab.research.google.com/github/libKriging/benchmarks/blob/main/large-n_Kriging-vecchia-nystrom_python.ipynb) · [Binder](https://mybinder.org/v2/gh/libKriging/benchmarks/main?labpath=large-n_Kriging-vecchia-nystrom_python.ipynb) | +| [aggregations-large-n_NestedKriging_python](aggregations-large-n_NestedKriging_python.ipynb) | `NestedKriging` | `NK`, `PoE`, `gPoE`, `BCM`, `rBCM`, from 1 group to 32, large n; save/load, pickle | exact predictor (limit: 1 group), exact fit | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-aggregations-large-n_NestedKriging_python.zip) | [Colab](https://colab.research.google.com/github/libKriging/benchmarks/blob/main/aggregations-large-n_NestedKriging_python.ipynb) · [Binder](https://mybinder.org/v2/gh/libKriging/benchmarks/main?labpath=aggregations-large-n_NestedKriging_python.ipynb) | +| [warpings-categorical-ordinal_WarpKriging_python](warpings-categorical-ordinal_WarpKriging_python.ipynb) | `WarpKriging` | affine, boxcox, kumaraswamy, knots, neural_mono, categorical, ordinal warpings; save/load, pickle | plain `Kriging` | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-warpings-categorical-ordinal_WarpKriging_python.zip) | [Colab](https://colab.research.google.com/github/libKriging/benchmarks/blob/main/warpings-categorical-ordinal_WarpKriging_python.ipynb) · [Binder](https://mybinder.org/v2/gh/libKriging/benchmarks/main?labpath=warpings-categorical-ordinal_WarpKriging_python.ipynb) | +| [deep-kernel-projected-branin_MLPKriging_python](deep-kernel-projected-branin_MLPKriging_python.ipynb) | `MLPKriging` | deep kernel on a 10D function with a 2D structure, activations, multistart; save/load, pickle | plain `Kriging` | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-deep-kernel-projected-branin_MLPKriging_python.zip) | [Colab](https://colab.research.google.com/github/libKriging/benchmarks/blob/main/deep-kernel-projected-branin_MLPKriging_python.ipynb) · [Binder](https://mybinder.org/v2/gh/libKriging/benchmarks/main?labpath=deep-kernel-projected-branin_MLPKriging_python.ipynb) | +| [cross-binding-reference_all-classes_python](cross-binding-reference_all-classes_python.ipynb) | all classes | the cross-binding protocol (all classes and fit methods): the Python reference | the other bindings (report) | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-cross-binding-reference_all-classes_python.zip) | [Colab](https://colab.research.google.com/github/libKriging/benchmarks/blob/main/cross-binding-reference_all-classes_python.ipynb) · [Binder](https://mybinder.org/v2/gh/libKriging/benchmarks/main?labpath=cross-binding-reference_all-classes_python.ipynb) | + +### R, Octave, Julia (`rlibkriging`, `mLibKriging`, `JLibKriging`) + +| notebook | kriging type | theme: problems, features | reference: compared with | executed | open in | +|---|---|---|---|---|---| +| [vs-DiceKriging-RobustGaSP_Kriging_R](vs-DiceKriging-RobustGaSP_Kriging_R.ipynb) | `Kriging` | rlibkriging: default fits (Branin, Hartmann 3, Borehole) | DiceKriging (equality at fixed hyperparameters), RobustGaSP | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-vs-DiceKriging-RobustGaSP_Kriging_R.zip) | — | +| [vs-STK_Kriging_octave](vs-STK_Kriging_octave.ipynb) | `Kriging` | mLibKriging: default fits (Branin, Hartmann 3, Borehole) | STK (equality at fixed hyperparameters, Gaussian kernel) | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-vs-STK_Kriging_octave.zip) | — | +| [vs-GaussianProcesses_Kriging_julia](vs-GaussianProcesses_Kriging_julia.ipynb) | `Kriging` | JLibKriging: default fits (Branin, Hartmann 3, Borehole) | GaussianProcesses.jl | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-vs-GaussianProcesses_Kriging_julia.zip) | — | +| [cross-binding_all-classes_R](cross-binding_all-classes_R.ipynb) | all classes | the cross-binding protocol, R side | Python (report) | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-cross-binding_all-classes_R.zip) | — | +| [cross-binding_all-classes_octave](cross-binding_all-classes_octave.ipynb) | all classes | the cross-binding protocol, Octave side | Python (report) | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-cross-binding_all-classes_octave.zip) | — | +| [cross-binding_all-classes_julia](cross-binding_all-classes_julia.ipynb) | all classes | the cross-binding protocol, Julia side | Python (report) | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-cross-binding_all-classes_julia.zip) | — | A notebook that compares libKriging with another package installs that package itself, in its first cells (pip, CRAN, `Pkg.add`, or a clone of STK); a package @@ -92,6 +70,27 @@ Test functions: [`tools/lkbench/functions.py`](tools/lkbench/functions.py), afte [Virtual Library of Simulation Experiments](https://www.sfu.ca/~ssurjano/) (Surjanovic & Bingham). +## Running a notebook in Colab or Binder + +The **open in** links run a Python notebook in a free hosted service, against the +**released** pylibkriging (from PyPI), not against libKriging master as CI does: a +feature not released yet is reported as SKIP. The first code cell of each Python +notebook ([`tools/notebook_setup.py`](tools/notebook_setup.py), a no-op in this +repository) clones the repository for its datasets and helpers and installs +pylibkriging when missing; the notebooks that compare with other packages install +them themselves. + +* **Colab**: PyPI has pylibkriging wheels for CPython 3.10 to 3.12 only, and Colab's + default runtime has used Python 3.13 since August-September 2026: choose + *Runtime > Change runtime version > 2026.07* (Python 3.12) before running, + otherwise the setup cell stops with this message. +* **Binder**: the environment of [`.binder/`](.binder) (Python 3.12, pylibkriging); + the first start builds the image, which takes a few minutes. + +Not offered for the R, Octave and Julia notebooks: Octave is not available on Colab, +and the installation of rlibkriging (compiled from CRAN sources) and of JLibKriging.jl +(which compiles libKriging) on these services has not been checked. + ## Checks and statuses Each notebook records its checks with `lkbench.Suite` (Python) or the small @@ -133,7 +132,7 @@ what it needs (BLAS/LAPACK headers, Octave, R, Julia). Notebooks are generated from plain-text sources ([`tools/src/`](tools/src), cells separated by `# %%` / `# %% [markdown]`, or `% %%` in Octave), so that reviews are -plain diffs; outputs are not committed (CI archives the executed notebooks as +plain diffs, and get the setup cell of `tools/notebook_setup.py` (Python); outputs are not committed (CI archives the executed notebooks as artifacts, see [Notebooks](#notebooks)). ```sh diff --git a/aggregations-large-n_NestedKriging_python.ipynb b/aggregations-large-n_NestedKriging_python.ipynb index bed6171..2b79601 100644 --- a/aggregations-large-n_NestedKriging_python.ipynb +++ b/aggregations-large-n_NestedKriging_python.ipynb @@ -18,6 +18,9 @@ "every aggregation on Branin.\n", "\n", "Checks:\n", + "* **limit and asymptotic behaviour** at fixed θ (Hartmann 3, n = 300): with one\n", + " group, `NK` *is* the exact kriging predictor; the gap to the exact predictor\n", + " as the number of groups grows, against the fit and prediction cost;\n", "* accuracy close to the exact fit (warn-level gap, error-level floor), and\n", " faster than the exact fit at n = 1500 (warn-level);\n", "* `NK` interpolates the data (it is a proper kriging predictor); the PoE family\n", @@ -31,9 +34,45 @@ "id": "c001", "metadata": {}, "outputs": [], + "source": [ + "# Setup for running this notebook outside the repository (Colab, Binder, a lone copy):\n", + "# nothing to do in the repository itself (CI, local runs). Otherwise the repository\n", + "# (datasets of data/, helpers of tools/) is cloned and the released pylibkriging installed.\n", + "import importlib\n", + "import os\n", + "import subprocess\n", + "import sys\n", + "from pathlib import Path\n", + "\n", + "if not Path(\"tools/known_issues.csv\").is_file():\n", + " if not Path(\"benchmarks/tools/known_issues.csv\").is_file():\n", + " subprocess.run([\"git\", \"clone\", \"--depth\", \"1\", \"https://github.com/libKriging/benchmarks\"], check=True)\n", + " os.chdir(\"benchmarks\")\n", + " try:\n", + " import pylibkriging # noqa: F401\n", + " except ImportError:\n", + " r = subprocess.run([sys.executable, \"-m\", \"pip\", \"install\", \"--quiet\", \"pylibkriging\"],\n", + " capture_output=True, text=True)\n", + " if r.returncode != 0:\n", + " raise RuntimeError(\n", + " f\"pylibkriging could not be installed for Python {sys.version_info.major}.{sys.version_info.minor} \"\n", + " \"(PyPI has wheels for CPython 3.10 to 3.12 only). On Colab: Runtime > Change runtime version > \"\n", + " \"2026.07 (Python 3.12), then run again.\\n\" + r.stderr[-500:])\n", + " importlib.invalidate_caches()\n", + "sys.path.insert(0, str(Path(\"tools\").resolve())) # lkbench, when not installed (Colab, Binder)\n", + "print(\"working directory:\", os.getcwd())" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c002", + "metadata": {}, + "outputs": [], "source": [ "import time\n", "\n", + "import matplotlib.pyplot as plt\n", "import numpy as np\n", "import pandas as pd\n", "import pylibkriging as lk\n", @@ -60,7 +99,61 @@ }, { "cell_type": "markdown", - "id": "c002", + "id": "c003", + "metadata": {}, + "source": [ + "## From one group (the exact predictor) to many, at fixed θ\n", + "\n", + "Hartmann 3, n = 300, θ of the exact fit kept fixed in every sub-model: the gap\n", + "is that of the aggregation alone. RMS gap of the predicted mean to the exact\n", + "predictor on 500 test points, relative to its standard deviation, and fit time." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c004", + "metadata": {}, + "outputs": [], + "source": [ + "if compat.has(\"NestedKriging\"):\n", + " tf = TESTS[\"hartmann3\"]\n", + " X, y, Xt, yt = design.dataset(tf, 300, n_test=500, seed=54)\n", + " theta = np.asarray(lk.Kriging(y, X, \"matern5_2\").theta()).reshape(1, -1)\n", + " fixed = {\"theta\": theta, \"is_theta_estim\": False}\n", + " m_e = pred(lk.Kriging(y, X, \"matern5_2\", optim=\"none\", parameters=fixed), Xt)[0]\n", + " rows = []\n", + " for agg in [\"NK\", \"rBCM\"]:\n", + " for g in [1, 2, 4, 8, 16, 32]:\n", + " with suite.runs(f\"nested {agg}, {g} groups, fixed theta: fit + predict\"):\n", + " k, t = timed(lambda: lk.NestedKriging(y, X, \"matern5_2\", nb_groups=g, aggregation=agg, seed=1,\n", + " optim=\"none\", parameters=fixed))\n", + " m = pred(k, Xt)[0]\n", + " rows.append(dict(aggregation=agg, groups=g, time_s=t,\n", + " mean_gap=np.sqrt(np.mean((m - m_e) ** 2)) / np.std(m_e),\n", + " q2=metrics.q2(yt, m)))\n", + " conv = pd.DataFrame(rows)\n", + " nk = conv[conv[\"aggregation\"] == \"NK\"]\n", + " suite.less(\"nested NK, 1 group == exact predictor (RMS gap / sd)\", nk[\"mean_gap\"].iloc[0], 1e-6)\n", + " suite.less(\"nested NK, 32 groups: RMS gap to the exact predictor / sd\", nk[\"mean_gap\"].iloc[-1], 0.1,\n", + " severity=\"warn\")\n", + " rb = conv[conv[\"aggregation\"] == \"rBCM\"]\n", + " suite.true(\"nested NK closer to the exact predictor than rBCM (2 to 32 groups)\",\n", + " bool(np.all(nk[\"mean_gap\"].to_numpy()[1:] <= rb[\"mean_gap\"].to_numpy()[1:])), severity=\"warn\")\n", + " fig, axes = plt.subplots(1, 2, figsize=(10, 3.8))\n", + " for agg, g in conv.groupby(\"aggregation\"):\n", + " axes[0].semilogy(g[\"groups\"], np.maximum(g[\"mean_gap\"], 1e-12), \"o-\", label=agg)\n", + " axes[1].plot(g[\"groups\"], g[\"time_s\"], \"o-\", label=agg)\n", + " axes[0].set_ylabel(\"RMS(mean − mean(exact)) / sd\"); axes[1].set_ylabel(\"fit time (s)\")\n", + " for ax in axes:\n", + " ax.set_xscale(\"log\", base=2); ax.set_xlabel(\"number of groups\"); ax.grid(True, which=\"both\", alpha=.3)\n", + " axes[0].legend(); fig.suptitle(\"Hartmann 3, n = 300, θ fixed: NestedKriging vs the exact predictor\"); plt.show()\n", + " display(conv.round(5))" + ] + }, + { + "cell_type": "markdown", + "id": "c005", "metadata": {}, "source": [ "## Accuracy and cost against the exact fit" @@ -69,7 +162,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c003", + "id": "c006", "metadata": {}, "outputs": [], "source": [ @@ -103,7 +196,7 @@ }, { "cell_type": "markdown", - "id": "c004", + "id": "c007", "metadata": {}, "source": [ "## `NestedKriging` aggregations: interpolation and variance" @@ -112,7 +205,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c005", + "id": "c008", "metadata": {}, "outputs": [], "source": [ @@ -137,7 +230,7 @@ }, { "cell_type": "markdown", - "id": "c006", + "id": "c009", "metadata": {}, "source": [ "## save / load, pickle, copy\n", @@ -150,7 +243,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c007", + "id": "c010", "metadata": {}, "outputs": [], "source": [ @@ -163,7 +256,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c008", + "id": "c011", "metadata": {}, "outputs": [], "source": [ diff --git a/aggregations_NestedKriging_R.ipynb b/aggregations_NestedKriging_R.ipynb deleted file mode 100644 index ff6fd14..0000000 --- a/aggregations_NestedKriging_R.ipynb +++ /dev/null @@ -1,125 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "c000", - "metadata": {}, - "source": [ - "# R · `NestedKriging` — `NK` and `PoE` aggregations, 4 groups of 100 points\n", - "\n", - "Branin on [0,1]² (n = 30 LHS points unless stated, Q² on 500 uniform test\n", - "points), through rlibkriging. Checks (error-level unless stated):\n", - "\n", - "* stdev ≥ 0, Q² > 0.95 (warn-level)\n", - "* `NK` interpolates the data (it is a proper kriging predictor)\n", - "* `save` then `load`: the same predicted mean\n", - "\n", - "A class missing from the installed rlibkriging (release channel) is recorded as SKIP." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c001", - "metadata": {}, - "outputs": [], - "source": [ - "suppressPackageStartupMessages(library(rlibkriging))\n", - "source(\"tools/r/checks.R\")\n", - "s <- new_suite(\"aggregations_NestedKriging_R\")\n", - "cat(\"rlibkriging\", as.character(packageVersion(\"rlibkriging\")), \"\\n\")\n", - "\n", - "branin <- function(U) { # on [0,1]^2\n", - " x1 <- 15 * U[, 1] - 5; x2 <- 15 * U[, 2]\n", - " (x2 - 5.1 / (4 * pi^2) * x1^2 + 5 / pi * x1 - 6)^2 + 10 * (1 - 1 / (8 * pi)) * cos(x1) + 10\n", - "}\n", - "set.seed(1)\n", - "lhs <- function(n, d) sapply(1:d, function(j) (sample(n) - runif(n)) / n)\n", - "X <- lhs(30, 2); y <- branin(X)\n", - "Xt <- matrix(runif(1000), ncol = 2); yt <- branin(Xt)\n", - "tmp <- tempdir()\n", - "has_class <- function(name) exists(name, mode = \"function\")\n", - "skip <- function(s, check, reason) {\n", - " s$checks[[length(s$checks) + 1]] <- data.frame(suite = s$name, lang = \"R\", check = check, severity = \"info\",\n", - " status = \"SKIP\", value = \"\", expected = \"\", detail = reason, stringsAsFactors = FALSE)\n", - " message(\"[SKIP] \", check, \": \", reason)\n", - "}\n", - "roundtrip_same_mean <- function(s, label, k, Xn) {\n", - " f <- file.path(tmp, paste0(gsub(\"[^A-Za-z0-9]\", \"_\", label), \".json\"))\n", - " check_runs(s, paste0(label, \": save + load\"), {\n", - " save(k, f)\n", - " k2 <- rlibkriging::load(f)\n", - " check_close(s, paste0(label, \": load(save(k)) predicts the same mean\"),\n", - " predict(k2, Xn, return_stdev = FALSE)$mean, predict(k, Xn, return_stdev = FALSE)$mean, rtol = 1e-10)\n", - " })\n", - "}" - ] - }, - { - "cell_type": "markdown", - "id": "c002", - "metadata": {}, - "source": [ - "## `NestedKriging`: 4 groups of 100 points, `NK` and `PoE` aggregations" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c003", - "metadata": {}, - "outputs": [], - "source": [ - "Xl <- lhs(400, 2); yl <- branin(Xl)\n", - "if (has_class(\"NestedKriging\")) {\n", - " for (agg in c(\"NK\", \"PoE\")) {\n", - " check_runs(s, paste0(\"NestedKriging(\", agg, \"): fit + predict\"), {\n", - " kk <- NestedKriging(yl, Xl, kernel = \"matern5_2\", nb_groups = 4, aggregation = agg, seed = 3)\n", - " pk <- predict(kk, Xt, return_stdev = TRUE)\n", - " check_less(s, paste0(\"NestedKriging(\", agg, \"): stdev < 0\"), sum(pk$stdev < 0), 1)\n", - " add_metric(s, \"branin n=400\", paste(\"NestedKriging\", agg), q2 = q2(yt, pk$mean))\n", - " check_greater(s, paste0(\"NestedKriging(\", agg, \"): Q2\"), q2(yt, pk$mean), 0.95, severity = \"warn\")\n", - " if (agg == \"NK\")\n", - " check_less(s, \"NestedKriging(NK): interpolates (max |m(X) - y| / sd(y))\",\n", - " max(abs(predict(kk, Xl, return_stdev = FALSE)$mean - yl)) / sd(yl), 1e-3)\n", - " roundtrip_same_mean(s, paste0(\"NestedKriging(\", agg, \")\"), kk, Xt)\n", - " })\n", - " }\n", - "} else skip(s, \"NestedKriging\", \"not in this rlibkriging\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c004", - "metadata": {}, - "outputs": [], - "source": [ - "me <- do.call(rbind, s$metrics)\n", - "print(me[me$metric == \"q2\", c(\"case\", \"package\", \"value\")], row.names = FALSE)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c005", - "metadata": {}, - "outputs": [], - "source": [ - "finalize(s)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "R", - "language": "R", - "name": "ir" - }, - "language_info": { - "name": "R" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/aggregations_NestedKriging_julia.ipynb b/aggregations_NestedKriging_julia.ipynb deleted file mode 100644 index f49993a..0000000 --- a/aggregations_NestedKriging_julia.ipynb +++ /dev/null @@ -1,128 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "c000", - "metadata": {}, - "source": [ - "# Julia · `NestedKriging` — `NK` and `PoE` aggregations, 4 groups of 100 points\n", - "\n", - "Branin on [0,1]² (n = 30 LHS points unless stated, Q² on 500 uniform test\n", - "points), through JLibKriging. Checks (error-level unless stated):\n", - "\n", - "* stdev ≥ 0, Q² > 0.95 (warn-level)\n", - "* `NK` interpolates the data (it is a proper kriging predictor)\n", - "* `save` then `load`: the same predicted mean\n", - "\n", - "The same checks as `aggregations_NestedKriging_R.ipynb`, with the Julia binding." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c001", - "metadata": {}, - "outputs": [], - "source": [ - "import JLibKriging as LK\n", - "using Random\n", - "include(\"tools/julia/checks.jl\")\n", - "s = Suite(\"aggregations_NestedKriging_julia\")\n", - "\n", - "branin(U) = begin\n", - " x1 = 15 .* U[:, 1] .- 5; x2 = 15 .* U[:, 2]\n", - " (x2 .- 5.1 / (4π^2) .* x1 .^ 2 .+ 5 / π .* x1 .- 6) .^ 2 .+ 10 * (1 - 1 / (8π)) .* cos.(x1) .+ 10\n", - "end\n", - "rng = MersenneTwister(1)\n", - "lhs(n, d) = hcat([(randperm(rng, n) .- rand(rng, n)) ./ n for _ in 1:d]...)\n", - "X = lhs(30, 2); y = branin(X)\n", - "Xt = rand(rng, 500, 2); yt = branin(Xt)\n", - "sd(v) = sqrt(sum((v .- sum(v) / length(v)) .^ 2) / (length(v) - 1))\n", - "finite_count(v...) = count(!isfinite, vcat(v...))\n", - "\n", - "function check_runs!(f, s, check)\n", - " try\n", - " f()\n", - " catch e\n", - " add_check!(s, check, false; value=string(typeof(e)), expected=\"no error\", detail=first(sprint(showerror, e), 200))\n", - " end\n", - "end\n", - "\n", - "function roundtrip!(s, label, k, Xn)\n", - " check_runs!(s, \"$label: save + load\") do\n", - " f = joinpath(mktempdir(), \"k.json\")\n", - " LK.save(k, f)\n", - " k2 = LK.load(f)\n", - " check_close!(s, \"$label: load(save(k)) predicts the same mean\",\n", - " LK.predict(k2, Xn; return_stdev=false).mean, LK.predict(k, Xn; return_stdev=false).mean; rtol=1e-10)\n", - " end\n", - "end" - ] - }, - { - "cell_type": "markdown", - "id": "c002", - "metadata": {}, - "source": [ - "## `NestedKriging`: 4 groups of 100 points, `NK` and `PoE` aggregations" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c003", - "metadata": {}, - "outputs": [], - "source": [ - "Xl = lhs(400, 2); yl = branin(Xl)\n", - "for agg in (\"NK\", \"PoE\")\n", - " label = \"NestedKriging($agg)\"\n", - " check_runs!(s, \"$label: fit + predict\") do\n", - " kk = LK.NestedKriging(yl, Xl, \"matern5_2\", 4; aggregation=agg, seed=3)\n", - " pk = LK.predict(kk, Xt; return_stdev=true)\n", - " add_check!(s, \"$label: stdev >= 0\", all(pk.stdev .>= 0); value=minimum(pk.stdev), expected=\">= 0\")\n", - " add_metric!(s, \"branin n=400\", \"NestedKriging $agg\"; q2=q2(yt, pk.mean))\n", - " add_check!(s, \"$label: Q2\", q2(yt, pk.mean) > 0.95; value=q2(yt, pk.mean), expected=\"> 0.95\", severity=\"warn\")\n", - " if agg == \"NK\"\n", - " e = maximum(abs.(LK.predict(kk, Xl; return_stdev=false).mean .- yl)) / sd(yl)\n", - " add_check!(s, \"NestedKriging(NK): interpolates (max |m(X) - y| / sd(y))\", e < 1e-3; value=e, expected=\"< 1e-3\")\n", - " end\n", - " roundtrip!(s, label, kk, Xt)\n", - " end\n", - "end" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c004", - "metadata": {}, - "outputs": [], - "source": [ - "foreach(r -> r[5] == \"q2\" && println(rpad(r[3], 24), rpad(r[4], 28), r[6]), s.metrics)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c005", - "metadata": {}, - "outputs": [], - "source": [ - "finalize!(s)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Julia 1.11", - "language": "julia", - "name": "julia-1.11" - }, - "language_info": { - "name": "julia" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/aggregations_NestedKriging_octave.ipynb b/aggregations_NestedKriging_octave.ipynb deleted file mode 100644 index 53c7c3a..0000000 --- a/aggregations_NestedKriging_octave.ipynb +++ /dev/null @@ -1,138 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "c000", - "metadata": {}, - "source": [ - "# Octave · `NestedKriging` — `NK` and `PoE` aggregations, 4 groups of 100 points\n", - "\n", - "Branin on [0,1]² (n = 30 LHS points unless stated, Q² on 500 uniform test\n", - "points), through mLibKriging. Checks (error-level unless stated):\n", - "\n", - "* stdev ≥ 0, Q² > 0.95 (warn-level)\n", - "* `NK` interpolates the data (it is a proper kriging predictor)\n", - "* `save` then `load`: the same predicted mean\n", - "\n", - "The same checks as `aggregations_NestedKriging_R.ipynb`, with the Octave binding.\n", - "\n", - "A class missing from the installed mLibKriging (release channel) is recorded as SKIP." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c001", - "metadata": {}, - "outputs": [], - "source": [ - "addpath(fullfile(pwd, \"tools\", \"octave\"));\n", - "s = lkb_suite(\"aggregations_NestedKriging_octave\");\n", - "q2 = @(y, m) 1 - sum((y - m).^2) / sum((y - mean(y)).^2);\n", - "branin = @(U) ((15 * U(:, 2)) - 5.1 / (4 * pi^2) * (15 * U(:, 1) - 5).^2 + 5 / pi * (15 * U(:, 1) - 5) - 6).^2 ...\n", - " + 10 * (1 - 1 / (8 * pi)) * cos(15 * U(:, 1) - 5) + 10;\n", - "rand(\"seed\", 1); randn(\"seed\", 1);\n", - "function U = lhs(n, d)\n", - " U = zeros(n, d);\n", - " for j = 1:d, U(:, j) = (randperm(n)' - rand(n, 1)) / n; end\n", - "end\n", - "X = lhs(30, 2); y = branin(X);\n", - "Xt = rand(500, 2); yt = branin(Xt);\n", - "tmp = tempname(); mkdir(tmp);\n", - "has_class = @(name) exist(name) == 2; % a class file on the path\n", - "\n", - "function s = roundtrip(s, label, k, Xn, loader)\n", - " f = fullfile(tempdir(), [regexprep(label, \"[^A-Za-z0-9]\", \"_\") \".json\"]);\n", - " try\n", - " k.save(f);\n", - " k2 = loader(f);\n", - " m1 = k.predict(Xn, false, false, false);\n", - " m2 = k2.predict(Xn, false, false, false);\n", - " s = lkb_close(s, [label \": load(save(k)) predicts the same mean\"], m2, m1, 1e-10);\n", - " catch err\n", - " s = lkb_check(s, [label \": save + load\"], false, \"error\", \"no error\", err.message);\n", - " end\n", - "end\n", - "\n", - "function s = skip(s, check, reason)\n", - " s.checks(end + 1, :) = {s.name, \"octave\", check, \"info\", \"SKIP\", \"\", \"\", reason};\n", - " printf(\"[SKIP] %s: %s\\n\", check, reason);\n", - "end" - ] - }, - { - "cell_type": "markdown", - "id": "c002", - "metadata": {}, - "source": [ - "## `NestedKriging`: 4 groups of 100 points, `NK` and `PoE` aggregations" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c003", - "metadata": {}, - "outputs": [], - "source": [ - "Xl = lhs(400, 2); yl = branin(Xl);\n", - "if has_class(\"NestedKriging\")\n", - " for agg = {\"NK\", \"PoE\"}\n", - " agg = agg{1}; label = [\"NestedKriging(\" agg \")\"];\n", - " try\n", - " kk = NestedKriging(yl, Xl, \"matern5_2\", 4, agg, \"kmeans\", 3);\n", - " [mk, sk] = kk.predict(Xt); % stdev returned when asked for (2 outputs)\n", - " s = lkb_check(s, [label \": stdev >= 0\"], all(sk >= 0), min(sk), \">= 0\");\n", - " s = lkb_metric(s, \"branin n=400\", [\"NestedKriging \" agg], \"q2\", q2(yt, mk));\n", - " s = lkb_check(s, [label \": Q2\"], q2(yt, mk) > 0.95, q2(yt, mk), \"> 0.95\", \"\", \"warn\");\n", - " if strcmp(agg, \"NK\")\n", - " e = max(abs(kk.predict(Xl) - yl)) / std(yl);\n", - " s = lkb_check(s, \"NestedKriging(NK): interpolates (max |m(X) - y| / sd(y))\", e < 1e-3, e, \"< 1e-3\");\n", - " end\n", - " f = fullfile(tmp, [\"nested_\" agg \".json\"]);\n", - " kk.save(f); k2 = load_kriging(f);\n", - " s = lkb_close(s, [label \": load(save(k)) predicts the same mean\"], k2.predict(Xt), kk.predict(Xt), 1e-10);\n", - " catch err\n", - " s = lkb_check(s, [label \": fit + predict\"], false, \"error\", \"no error\", err.message);\n", - " end\n", - " end\n", - "else\n", - " s = skip(s, \"NestedKriging\", \"not in this mLibKriging\");\n", - "end" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c004", - "metadata": {}, - "outputs": [], - "source": [ - "q = strcmp(s.metrics(:, 5), \"q2\");\n", - "disp(s.metrics(q, [3, 4, 6]))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c005", - "metadata": {}, - "outputs": [], - "source": [ - "lkb_finalize(s);" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Octave", - "language": "octave", - "name": "octave" - }, - "language_info": { - "name": "octave" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/cross-binding-reference_Kriging_python.ipynb b/cross-binding-reference_Kriging_python.ipynb deleted file mode 100644 index 203884e..0000000 --- a/cross-binding-reference_Kriging_python.ipynb +++ /dev/null @@ -1,121 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "c000", - "metadata": {}, - "source": [ - "# Python · Kriging — cross-binding consistency: the Python reference\n", - "\n", - "All bindings (Python, R, Octave, Julia) wrap the same C++ library: on the\n", - "same data with the same options they must return the same numbers. Each\n", - "language runs the protocol below on the shared datasets of `data/` and writes\n", - "`results/xbind_.csv` (`lang,dataset,quantity,index,value`);\n", - "`tools/report.py` then compares every language to Python.\n", - "\n", - "Protocol, for each dataset (Branin n=25, Hartmann 3 n=40, Borehole n=80, on\n", - "the unit cube), kernel `matern5_2`, trend `constant`, θ₀ = 0.3 in every\n", - "dimension:\n", - "\n", - "| quantity | how | tolerance in the report |\n", - "|---|---|---|\n", - "| `fixed_sigma2`, `fixed_beta`, `fixed_ll`, `fixed_loo` | `optim=\"none\"`, θ = θ₀ fixed | rel. 1e-8 (error) |\n", - "| `fixed_mean`, `fixed_stdev` | predict at the first 50 test points | rel. 1e-8 (error) |\n", - "| `llfun`, `llfun_grad` | `logLikelihoodFun(θ₀)` with gradient | rel. 1e-8 (error) |\n", - "| `fit_theta`, `fit_ll`, `fit_mean` | default fit (`BFGS`) | rel. 1e-4 (warn: optimizer paths may differ) |\n", - "| `sim` | `simulate(nsim=3, seed=123)` at the first 5 test points (5 × 3, column-major) | rel. 1e-8 (warn: RNG may be binding-specific) |" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c001", - "metadata": {}, - "outputs": [], - "source": [ - "import csv\n", - "\n", - "import numpy as np\n", - "import pylibkriging as lk\n", - "\n", - "import lkbench\n", - "from lkbench import compat\n", - "from lkbench.checks import DATA_DIR as DATA, RESULTS_DIR\n", - "\n", - "suite = lkbench.Suite(\"cross-binding-reference_Kriging_python\")\n", - "print(\"pylibkriging\", compat.version())\n", - "DATASETS = [\"branin\", \"hartmann3\", \"borehole\"]\n", - "rows = []\n", - "\n", - "\n", - "def put(dataset, quantity, values):\n", - " for i, v in enumerate(np.ravel(values), start=1):\n", - " rows.append((\"python\", dataset, quantity, i, f\"{float(v):.17g}\"))\n", - "\n", - "\n", - "def load(name, stem):\n", - " return np.loadtxt(DATA / f\"{name}_{stem}.csv\", delimiter=\",\", ndmin=2)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c002", - "metadata": {}, - "outputs": [], - "source": [ - "for name in DATASETS:\n", - " X, y, Xt = load(name, \"X\"), load(name, \"y\").ravel(), load(name, \"Xt\")[:50]\n", - " d = X.shape[1]\n", - " theta0 = np.full((1, d), 0.3)\n", - " kf = lk.Kriging(y, X, \"matern5_2\", regmodel=\"constant\", optim=\"none\",\n", - " parameters={\"theta\": theta0, \"is_theta_estim\": False})\n", - " put(name, \"fixed_sigma2\", kf.sigma2())\n", - " put(name, \"fixed_beta\", kf.beta())\n", - " put(name, \"fixed_ll\", kf.logLikelihood())\n", - " put(name, \"fixed_loo\", kf.leaveOneOut())\n", - " m, s, *_ = kf.predict(Xt, return_stdev=True)\n", - " put(name, \"fixed_mean\", m)\n", - " put(name, \"fixed_stdev\", s)\n", - " ll, grad, _ = kf.logLikelihoodFun(theta0.ravel(), True)\n", - " put(name, \"llfun\", ll)\n", - " put(name, \"llfun_grad\", grad)\n", - " suite.close(f\"{name}: logLikelihood() == logLikelihoodFun(theta0)\", kf.logLikelihood(), ll, rtol=1e-10)\n", - " k = lk.Kriging(y, X, \"matern5_2\")\n", - " put(name, \"fit_theta\", k.theta())\n", - " put(name, \"fit_ll\", k.logLikelihood())\n", - " put(name, \"fit_mean\", k.predict(Xt, return_stdev=False)[0])\n", - " put(name, \"sim\", np.ravel(np.asarray(kf.simulate(3, 123, Xt[:5])), order=\"F\")) # column-major, as R/Octave/Julia\n", - "\n", - "RESULTS_DIR.mkdir(exist_ok=True)\n", - "with open(RESULTS_DIR / \"xbind_python.csv\", \"w\", newline=\"\") as fh:\n", - " w = csv.writer(fh)\n", - " w.writerow([\"lang\", \"dataset\", \"quantity\", \"index\", \"value\"])\n", - " w.writerows(rows)\n", - "print(len(rows), \"values written\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c003", - "metadata": {}, - "outputs": [], - "source": [ - "suite.finalize()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/cross-binding-reference_all-classes_python.ipynb b/cross-binding-reference_all-classes_python.ipynb new file mode 100644 index 0000000..49d8aea --- /dev/null +++ b/cross-binding-reference_all-classes_python.ipynb @@ -0,0 +1,223 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "c000", + "metadata": {}, + "source": [ + "# Python · all classes — cross-binding consistency: the Python reference\n", + "\n", + "All bindings (Python, R, Octave, Julia) wrap the same C++ library: on the\n", + "same data with the same options they must return the same numbers. Each\n", + "language runs the protocol below on the shared datasets of `data/` and writes\n", + "`results/xbind_.csv` (`lang,dataset,quantity,index,value`);\n", + "`tools/report.py` then compares every language to Python.\n", + "\n", + "Protocol, for each dataset (Branin n=25, Hartmann 3 n=40, Borehole n=80, on\n", + "the unit cube), kernel `matern5_2`, trend `constant`, θ₀ = 0.3 in every\n", + "dimension:\n", + "\n", + "| quantity | how | tolerance in the report |\n", + "|---|---|---|\n", + "| `fixed_sigma2`, `fixed_beta`, `fixed_ll`, `fixed_loo` | `optim=\"none\"`, θ = θ₀ fixed | rel. 1e-8 (error) |\n", + "| `fixed_mean`, `fixed_stdev` | predict at the first 50 test points | rel. 1e-8 (error) |\n", + "| `llfun`, `llfun_grad` | `logLikelihoodFun(θ₀)` with gradient | rel. 1e-8 (error) |\n", + "| `fit_theta`, `fit_ll`, `fit_mean` | default fit (`BFGS`) | rel. 1e-4 (warn: optimizer paths may differ) |\n", + "| `sim` | `simulate(nsim=3, seed=123)` at the first 5 test points (5 × 3, column-major) | rel. 1e-8 (warn: RNG may be binding-specific) |\n", + "\n", + "Then, on Branin only, every other model class and fit method, so that each\n", + "binding is checked on all of them (the class-specific notebooks are in Python\n", + "only): with fixed hyperparameters the numbers must be identical (error); a\n", + "fit is compared at warn level, as above.\n", + "\n", + "| quantity | how | tolerance in the report |\n", + "|---|---|---|\n", + "| `nugget_fixed_mean`, `nugget_fixed_stdev` | `noise=\"nugget\"`, θ = θ₀, σ² = var(y), nugget = 0.01 var(y), all fixed | rel. 1e-8 (error) |\n", + "| `noise_fixed_mean`, `noise_fixed_stdev` | known noise variances 0.01 var(y), θ = θ₀ and σ² fixed | rel. 1e-8 (error) |\n", + "| `nugget_fit_ll`, `nugget_fit_mean` | `noise=\"nugget\"`, default fit | rel. 1e-4 (warn) |\n", + "| `vecchia_fixed_ll`, `vecchia_fixed_mean`, `nystrom_fixed_ll`, `nystrom_fixed_mean` | `method=\"vecchia(10)\"` / `\"nystrom(10)\"`, θ = θ₀ fixed; likelihood and prediction with that method | rel. 1e-8 (error) |\n", + "| `warp_fit_mean` | `WarpKriging`, `kumaraswamy` on both inputs, default fit | rel. 1e-4 (warn) |\n", + "| `mlp_fit_mean` | `MLPKriging`, hidden layer of 8 (`tanh`), 2 features, `BFGS10` | rel. 1e-4 (warn) |\n", + "| `nested_fit_mean` | `NestedKriging`, 2 groups (`kmeans`, seed 1), `NK` | rel. 1e-4 (warn) |" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c001", + "metadata": {}, + "outputs": [], + "source": [ + "# Setup for running this notebook outside the repository (Colab, Binder, a lone copy):\n", + "# nothing to do in the repository itself (CI, local runs). Otherwise the repository\n", + "# (datasets of data/, helpers of tools/) is cloned and the released pylibkriging installed.\n", + "import importlib\n", + "import os\n", + "import subprocess\n", + "import sys\n", + "from pathlib import Path\n", + "\n", + "if not Path(\"tools/known_issues.csv\").is_file():\n", + " if not Path(\"benchmarks/tools/known_issues.csv\").is_file():\n", + " subprocess.run([\"git\", \"clone\", \"--depth\", \"1\", \"https://github.com/libKriging/benchmarks\"], check=True)\n", + " os.chdir(\"benchmarks\")\n", + " try:\n", + " import pylibkriging # noqa: F401\n", + " except ImportError:\n", + " r = subprocess.run([sys.executable, \"-m\", \"pip\", \"install\", \"--quiet\", \"pylibkriging\"],\n", + " capture_output=True, text=True)\n", + " if r.returncode != 0:\n", + " raise RuntimeError(\n", + " f\"pylibkriging could not be installed for Python {sys.version_info.major}.{sys.version_info.minor} \"\n", + " \"(PyPI has wheels for CPython 3.10 to 3.12 only). On Colab: Runtime > Change runtime version > \"\n", + " \"2026.07 (Python 3.12), then run again.\\n\" + r.stderr[-500:])\n", + " importlib.invalidate_caches()\n", + "sys.path.insert(0, str(Path(\"tools\").resolve())) # lkbench, when not installed (Colab, Binder)\n", + "print(\"working directory:\", os.getcwd())" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c002", + "metadata": {}, + "outputs": [], + "source": [ + "import csv\n", + "\n", + "import numpy as np\n", + "import pylibkriging as lk\n", + "\n", + "import lkbench\n", + "from lkbench import compat\n", + "from lkbench.checks import DATA_DIR as DATA, RESULTS_DIR\n", + "\n", + "suite = lkbench.Suite(\"cross-binding-reference_all-classes_python\")\n", + "print(\"pylibkriging\", compat.version())\n", + "DATASETS = [\"branin\", \"hartmann3\", \"borehole\"]\n", + "rows = []\n", + "\n", + "\n", + "def put(dataset, quantity, values):\n", + " for i, v in enumerate(np.ravel(values), start=1):\n", + " rows.append((\"python\", dataset, quantity, i, f\"{float(v):.17g}\"))\n", + "\n", + "\n", + "def load(name, stem):\n", + " return np.loadtxt(DATA / f\"{name}_{stem}.csv\", delimiter=\",\", ndmin=2)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c003", + "metadata": {}, + "outputs": [], + "source": [ + "for name in DATASETS:\n", + " X, y, Xt = load(name, \"X\"), load(name, \"y\").ravel(), load(name, \"Xt\")[:50]\n", + " d = X.shape[1]\n", + " theta0 = np.full((1, d), 0.3)\n", + " kf = lk.Kriging(y, X, \"matern5_2\", regmodel=\"constant\", optim=\"none\",\n", + " parameters={\"theta\": theta0, \"is_theta_estim\": False})\n", + " put(name, \"fixed_sigma2\", kf.sigma2())\n", + " put(name, \"fixed_beta\", kf.beta())\n", + " put(name, \"fixed_ll\", kf.logLikelihood())\n", + " put(name, \"fixed_loo\", kf.leaveOneOut())\n", + " m, s, *_ = kf.predict(Xt, return_stdev=True)\n", + " put(name, \"fixed_mean\", m)\n", + " put(name, \"fixed_stdev\", s)\n", + " ll, grad, _ = kf.logLikelihoodFun(theta0.ravel(), True)\n", + " put(name, \"llfun\", ll)\n", + " put(name, \"llfun_grad\", grad)\n", + " suite.close(f\"{name}: logLikelihood() == logLikelihoodFun(theta0)\", kf.logLikelihood(), ll, rtol=1e-10)\n", + " k = lk.Kriging(y, X, \"matern5_2\")\n", + " put(name, \"fit_theta\", k.theta())\n", + " put(name, \"fit_ll\", k.logLikelihood())\n", + " put(name, \"fit_mean\", k.predict(Xt, return_stdev=False)[0])\n", + " put(name, \"sim\", np.ravel(np.asarray(kf.simulate(3, 123, Xt[:5])), order=\"F\")) # column-major, as R/Octave/Julia" + ] + }, + { + "cell_type": "markdown", + "id": "c004", + "metadata": {}, + "source": [ + "## Other classes and fit methods (Branin)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c005", + "metadata": {}, + "outputs": [], + "source": [ + "name = \"branin\"\n", + "X, y, Xt = load(name, \"X\"), load(name, \"y\").ravel(), load(name, \"Xt\")[:50]\n", + "theta0 = np.full((1, 2), 0.3)\n", + "s2 = float(np.var(y)); nug = 0.01 * s2\n", + "fixed = {\"theta\": theta0, \"is_theta_estim\": False, \"sigma2\": s2, \"is_sigma2_estim\": False}\n", + "kn = lk.Kriging(y, X, \"matern5_2\", noise=\"nugget\", optim=\"none\",\n", + " parameters={**fixed, \"nugget\": nug, \"is_nugget_estim\": False})\n", + "m, s, *_ = kn.predict(Xt, return_stdev=True)\n", + "put(name, \"nugget_fixed_mean\", m); put(name, \"nugget_fixed_stdev\", s)\n", + "kh = lk.Kriging(y, X, \"matern5_2\", noise=np.full(len(y), nug), optim=\"none\", parameters=fixed)\n", + "m, s, *_ = kh.predict(Xt, return_stdev=True)\n", + "put(name, \"noise_fixed_mean\", m); put(name, \"noise_fixed_stdev\", s)\n", + "kf = lk.Kriging(y, X, \"matern5_2\", noise=\"nugget\")\n", + "put(name, \"nugget_fit_ll\", kf.logLikelihood()); put(name, \"nugget_fit_mean\", kf.predict(Xt, return_stdev=False)[0])\n", + "for meth in [\"vecchia\", \"nystrom\"] if compat.has(\"method\") else []:\n", + " ka = lk.Kriging(y, X, \"matern5_2\", optim=\"none\", parameters={\"theta\": theta0, \"is_theta_estim\": False},\n", + " method=f\"{meth}(10)\")\n", + " put(name, f\"{meth}_fixed_ll\", ka.logLikelihood(method=meth))\n", + " put(name, f\"{meth}_fixed_mean\", ka.predict(Xt, return_stdev=False, method=meth)[0])\n", + "if not compat.has(\"method\"):\n", + " suite.skip(\"vecchia / nystrom quantities\", \"pylibkriging without the method= argument\")\n", + "kw = lk.WarpKriging(y, X, [\"kumaraswamy\", \"kumaraswamy\"], \"matern5_2\")\n", + "put(name, \"warp_fit_mean\", kw.predict(Xt, return_stdev=False)[0])\n", + "km = lk.MLPKriging(y, X, hidden_dims=[8], d_out=2, activation=\"tanh\", kernel=\"matern5_2\", optim=\"BFGS10\")\n", + "put(name, \"mlp_fit_mean\", km.predict(Xt, return_stdev=False)[0])\n", + "kk = lk.NestedKriging(y, X, \"matern5_2\", nb_groups=2, aggregation=\"NK\", seed=1)\n", + "put(name, \"nested_fit_mean\", kk.predict(Xt, return_stdev=False)[0])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c006", + "metadata": {}, + "outputs": [], + "source": [ + "RESULTS_DIR.mkdir(exist_ok=True)\n", + "with open(RESULTS_DIR / \"xbind_python.csv\", \"w\", newline=\"\") as fh:\n", + " w = csv.writer(fh)\n", + " w.writerow([\"lang\", \"dataset\", \"quantity\", \"index\", \"value\"])\n", + " w.writerows(rows)\n", + "print(len(rows), \"values written\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c007", + "metadata": {}, + "outputs": [], + "source": [ + "suite.finalize()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/cross-binding_Kriging_R.ipynb b/cross-binding_all-classes_R.ipynb similarity index 50% rename from cross-binding_Kriging_R.ipynb rename to cross-binding_all-classes_R.ipynb index 3fde9ef..b5e59fd 100644 --- a/cross-binding_Kriging_R.ipynb +++ b/cross-binding_all-classes_R.ipynb @@ -5,9 +5,9 @@ "id": "c000", "metadata": {}, "source": [ - "# R · Kriging — cross-binding consistency: the R side\n", + "# R · all classes — cross-binding consistency: the R side\n", "\n", - "Same protocol as `cross-binding-reference_Kriging_python.ipynb` (see there), on the\n", + "Same protocol as `cross-binding-reference_all-classes_python.ipynb` (see there), on the\n", "shared datasets of `data/`; writes `results/xbind_R.csv`, compared with the\n", "Python values by `tools/report.py`." ] @@ -21,7 +21,7 @@ "source": [ "suppressPackageStartupMessages(library(rlibkriging))\n", "source(\"tools/r/checks.R\")\n", - "s <- new_suite(\"cross-binding_Kriging_R\")\n", + "s <- new_suite(\"cross-binding_all-classes_R\")\n", "cat(\"rlibkriging\", as.character(packageVersion(\"rlibkriging\")), \"\\n\")\n", "rows <- list()\n", "put <- function(dataset, quantity, values) {\n", @@ -60,7 +60,59 @@ " put(name, \"fit_ll\", logLikelihood(k))\n", " put(name, \"fit_mean\", predict(k, Xt, return_stdev = FALSE)$mean)\n", " put(name, \"sim\", simulate(kf, nsim = 3, seed = 123, x = Xt[1:5, , drop = FALSE]))\n", - "}\n", + "}" + ] + }, + { + "cell_type": "markdown", + "id": "c003", + "metadata": {}, + "source": [ + "## Other classes and fit methods (Branin)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c004", + "metadata": {}, + "outputs": [], + "source": [ + "name <- \"branin\"\n", + "X <- read_data(name, \"X\"); y <- as.numeric(read_data(name, \"y\")); Xt <- read_data(name, \"Xt\")[1:50, , drop = FALSE]\n", + "theta0 <- matrix(0.3, nrow = 1, ncol = 2)\n", + "s2 <- mean((y - mean(y))^2); nug <- 0.01 * s2 # population variance, as numpy.var\n", + "fixed <- list(theta = theta0, is_theta_estim = FALSE, sigma2 = s2, is_sigma2_estim = FALSE)\n", + "kn <- Kriging(y, X, \"matern5_2\", noise = \"nugget\", optim = \"none\",\n", + " parameters = c(fixed, list(nugget = nug, is_nugget_estim = FALSE)))\n", + "p <- predict(kn, Xt, return_stdev = TRUE)\n", + "put(name, \"nugget_fixed_mean\", p$mean); put(name, \"nugget_fixed_stdev\", p$stdev)\n", + "kh <- Kriging(y, X, \"matern5_2\", noise = rep(nug, length(y)), optim = \"none\", parameters = fixed)\n", + "p <- predict(kh, Xt, return_stdev = TRUE)\n", + "put(name, \"noise_fixed_mean\", p$mean); put(name, \"noise_fixed_stdev\", p$stdev)\n", + "kf <- Kriging(y, X, \"matern5_2\", noise = \"nugget\")\n", + "put(name, \"nugget_fit_ll\", logLikelihood(kf)); put(name, \"nugget_fit_mean\", predict(kf, Xt, return_stdev = FALSE)$mean)\n", + "for (meth in c(\"vecchia\", \"nystrom\")) tryCatch({ # method= is absent from older releases\n", + " ka <- Kriging(y, X, \"matern5_2\", optim = \"none\", parameters = list(theta = theta0, is_theta_estim = FALSE),\n", + " method = paste0(meth, \"(10)\"))\n", + " put(name, paste0(meth, \"_fixed_ll\"), ka$logLikelihood(method = meth))\n", + " put(name, paste0(meth, \"_fixed_mean\"), predict(ka, Xt, return_stdev = FALSE, method = meth)$mean)\n", + "}, error = function(e) message(meth, \" skipped (reported as missing): \", conditionMessage(e)))\n", + "kw <- WarpKriging(y, X, warping = c(\"kumaraswamy\", \"kumaraswamy\"), kernel = \"matern5_2\")\n", + "put(name, \"warp_fit_mean\", predict(kw, Xt, return_stdev = FALSE)$mean)\n", + "km <- MLPKriging(y, X, hidden_dims = c(8), d_out = 2, activation = \"tanh\", kernel = \"matern5_2\", optim = \"BFGS10\")\n", + "put(name, \"mlp_fit_mean\", predict(km, Xt, return_stdev = FALSE)$mean)\n", + "kk <- NestedKriging(y, X, kernel = \"matern5_2\", nb_groups = 2, aggregation = \"NK\", seed = 1)\n", + "put(name, \"nested_fit_mean\", predict(kk, Xt, return_stdev = FALSE)$mean)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c005", + "metadata": {}, + "outputs": [], + "source": [ "out <- do.call(rbind, rows)\n", "write.csv(out, file.path(results_dir, \"xbind_R.csv\"), row.names = FALSE, quote = FALSE)\n", "cat(nrow(out), \"values written\\n\")" @@ -69,7 +121,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c003", + "id": "c006", "metadata": {}, "outputs": [], "source": [ diff --git a/cross-binding_Kriging_julia.ipynb b/cross-binding_all-classes_julia.ipynb similarity index 53% rename from cross-binding_Kriging_julia.ipynb rename to cross-binding_all-classes_julia.ipynb index 17e83c0..bb9c03d 100644 --- a/cross-binding_Kriging_julia.ipynb +++ b/cross-binding_all-classes_julia.ipynb @@ -5,9 +5,9 @@ "id": "c000", "metadata": {}, "source": [ - "# Julia · Kriging — cross-binding consistency: the Julia side\n", + "# Julia · all classes — cross-binding consistency: the Julia side\n", "\n", - "Same protocol as `cross-binding-reference_Kriging_python.ipynb` (see there), on\n", + "Same protocol as `cross-binding-reference_all-classes_python.ipynb` (see there), on\n", "the shared datasets of `data/`; writes `results/xbind_julia.csv`, compared with\n", "the Python values by `tools/report.py`.\n", "\n", @@ -25,7 +25,7 @@ "source": [ "import JLibKriging as LK\n", "include(\"tools/julia/checks.jl\")\n", - "s = Suite(\"cross-binding_Kriging_julia\")\n", + "s = Suite(\"cross-binding_all-classes_julia\")\n", "rows = Vector{Vector{String}}()\n", "record!(dataset, quantity, values) =\n", " for (i, v) in enumerate(vec(Float64.(collect(values))))\n", @@ -61,7 +61,59 @@ " record!(name, \"fit_ll\", LK.log_likelihood(k))\n", " record!(name, \"fit_mean\", LK.predict(k, Xt; return_stdev=false).mean)\n", " record!(name, \"sim\", LK.simulate(kf, 3, 123, Xt[1:5, :]))\n", + "end" + ] + }, + { + "cell_type": "markdown", + "id": "c003", + "metadata": {}, + "source": [ + "## Other classes and fit methods (Branin)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c004", + "metadata": {}, + "outputs": [], + "source": [ + "name = \"branin\"\n", + "Xd = read_data(name, \"X\"); yd = vec(read_data(name, \"y\")); Xt = read_data(name, \"Xt\")[1:50, :]\n", + "theta0 = [0.3, 0.3]\n", + "s2 = sum((yd .- sum(yd) / length(yd)) .^ 2) / length(yd); nug = 0.01 * s2 # population variance, as numpy.var\n", + "kn = LK.Kriging(yd, Xd, \"matern5_2\"; noise=\"nugget\", optim=\"none\", theta=theta0, is_theta_estim=false,\n", + " sigma2=s2, is_sigma2_estim=false, nugget=nug, is_nugget_estim=false)\n", + "p = LK.predict(kn, Xt; return_stdev=true)\n", + "record!(name, \"nugget_fixed_mean\", p.mean); record!(name, \"nugget_fixed_stdev\", p.stdev)\n", + "kh = LK.Kriging(yd, Xd, \"matern5_2\"; noise=fill(nug, length(yd)), optim=\"none\", theta=theta0, is_theta_estim=false,\n", + " sigma2=s2, is_sigma2_estim=false)\n", + "p = LK.predict(kh, Xt; return_stdev=true)\n", + "record!(name, \"noise_fixed_mean\", p.mean); record!(name, \"noise_fixed_stdev\", p.stdev)\n", + "kf = LK.Kriging(yd, Xd, \"matern5_2\"; noise=\"nugget\")\n", + "record!(name, \"nugget_fit_ll\", LK.log_likelihood(kf))\n", + "record!(name, \"nugget_fit_mean\", LK.predict(kf, Xt; return_stdev=false).mean)\n", + "for meth in (\"vecchia\", \"nystrom\")\n", + " ka = LK.Kriging(yd, Xd, \"matern5_2\"; optim=\"none\", theta=theta0, is_theta_estim=false, method=\"$meth(10)\")\n", + " record!(name, \"$(meth)_fixed_ll\", LK.log_likelihood(ka; method=meth))\n", + " record!(name, \"$(meth)_fixed_mean\", LK.predict(ka, Xt; return_stdev=false, method=meth).mean)\n", "end\n", + "kw = LK.WarpKriging(yd, Xd, [\"kumaraswamy\", \"kumaraswamy\"], \"matern5_2\")\n", + "record!(name, \"warp_fit_mean\", LK.predict(kw, Xt; return_stdev=false).mean)\n", + "km = LK.MLPKriging(yd, Xd, [8], 2; activation=\"tanh\", kernel=\"matern5_2\", optim=\"BFGS10\")\n", + "record!(name, \"mlp_fit_mean\", LK.predict(km, Xt; return_stdev=false).mean)\n", + "kk = LK.NestedKriging(yd, Xd, \"matern5_2\", 2; aggregation=\"NK\", seed=1)\n", + "record!(name, \"nested_fit_mean\", LK.predict(kk, Xt; return_stdev=false).mean)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c005", + "metadata": {}, + "outputs": [], + "source": [ "open(joinpath(RESULTS_DIR, \"xbind_julia.csv\"), \"w\") do io\n", " println(io, \"lang,dataset,quantity,index,value\")\n", " for r in rows; println(io, join(r, \",\")); end\n", @@ -72,7 +124,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c003", + "id": "c006", "metadata": {}, "outputs": [], "source": [ diff --git a/cross-binding_Kriging_octave.ipynb b/cross-binding_all-classes_octave.ipynb similarity index 50% rename from cross-binding_Kriging_octave.ipynb rename to cross-binding_all-classes_octave.ipynb index 64fc325..b77ce9a 100644 --- a/cross-binding_Kriging_octave.ipynb +++ b/cross-binding_all-classes_octave.ipynb @@ -5,9 +5,9 @@ "id": "c000", "metadata": {}, "source": [ - "# Octave · Kriging — cross-binding consistency: the Octave side\n", + "# Octave · all classes — cross-binding consistency: the Octave side\n", "\n", - "Same protocol as `cross-binding-reference_Kriging_python.ipynb` (see there), on the\n", + "Same protocol as `cross-binding-reference_all-classes_python.ipynb` (see there), on the\n", "shared datasets of `data/`; writes `results/xbind_octave.csv`, compared with\n", "the Python values by `tools/report.py`. mLibKriging must be on the path\n", "(`OCTAVE_PATH` or `addpath`)." @@ -21,7 +21,7 @@ "outputs": [], "source": [ "addpath(fullfile(pwd, \"tools\", \"octave\"));\n", - "s = lkb_suite(\"cross-binding_Kriging_octave\");\n", + "s = lkb_suite(\"cross-binding_all-classes_octave\");\n", "out = {};\n", "function out = put(out, dataset, quantity, values)\n", " v = double(values(:));\n", @@ -65,7 +65,67 @@ " out = put(out, name, \"fit_mean\", mk);\n", " S = kf.simulate(int32(3), int32(123), Xt(1:5, :), false);\n", " out = put(out, name, \"sim\", S);\n", + "end" + ] + }, + { + "cell_type": "markdown", + "id": "c003", + "metadata": {}, + "source": [ + "## Other classes and fit methods (Branin)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c004", + "metadata": {}, + "outputs": [], + "source": [ + "name = \"branin\";\n", + "X = read_data(name, \"X\"); y = read_data(name, \"y\"); Xt = read_data(name, \"Xt\")(1:50, :);\n", + "theta0 = [0.3, 0.3];\n", + "s2 = mean((y - mean(y)).^2); nug = 0.01 * s2; % population variance, as numpy.var\n", + "kn = Kriging(y, X, \"matern5_2\", \"constant\", false, \"none\", \"LL\", ...\n", + " Params(\"theta\", theta0, \"is_theta_estim\", false, \"sigma2\", s2, \"is_sigma2_estim\", false, ...\n", + " \"nugget\", nug, \"is_nugget_estim\", false), \"nugget\");\n", + "[m, sd] = kn.predict(Xt, true, false, false);\n", + "out = put(out, name, \"nugget_fixed_mean\", m); out = put(out, name, \"nugget_fixed_stdev\", sd);\n", + "kh = Kriging(y, X, \"matern5_2\", \"constant\", false, \"none\", \"LL\", ...\n", + " Params(\"theta\", theta0, \"is_theta_estim\", false, \"sigma2\", s2, \"is_sigma2_estim\", false), ...\n", + " \"heterogeneous\", nug * ones(size(y)));\n", + "[m, sd] = kh.predict(Xt, true, false, false);\n", + "out = put(out, name, \"noise_fixed_mean\", m); out = put(out, name, \"noise_fixed_stdev\", sd);\n", + "kf = Kriging(y, X, \"matern5_2\", \"constant\", false, \"BFGS\", \"LL\", [], \"nugget\");\n", + "ll = kf.logLikelihood(); out = put(out, name, \"nugget_fit_ll\", ll);\n", + "m = kf.predict(Xt, false, false, false); out = put(out, name, \"nugget_fit_mean\", m);\n", + "for meth = {\"vecchia\", \"nystrom\"}\n", + " meth = meth{1};\n", + " try % method is absent from older releases\n", + " ka = Kriging(y, X, \"matern5_2\", \"constant\", false, \"none\", \"LL\", Params(\"theta\", theta0, \"is_theta_estim\", false), ...\n", + " \"none\", [], [meth \"(10)\"]);\n", + " ll = ka.logLikelihood(meth); out = put(out, name, [meth \"_fixed_ll\"], ll);\n", + " m = ka.predict(Xt, false, false, false, meth); out = put(out, name, [meth \"_fixed_mean\"], m);\n", + " catch err\n", + " printf(\"%s skipped (reported as missing): %s\\n\", meth, err.message);\n", + " end\n", "end\n", + "kw = WarpKriging(y, X, {\"kumaraswamy\", \"kumaraswamy\"}, \"matern5_2\");\n", + "m = kw.predict(Xt, false, false, false); out = put(out, name, \"warp_fit_mean\", m);\n", + "km = MLPKriging(y, X, [8], 2, \"tanh\", \"matern5_2\", \"constant\", false, \"BFGS10\");\n", + "m = km.predict(Xt, false, false, false); out = put(out, name, \"mlp_fit_mean\", m);\n", + "kk = NestedKriging(y, X, \"matern5_2\", 2, \"NK\", \"kmeans\", 1);\n", + "m = kk.predict(Xt); out = put(out, name, \"nested_fit_mean\", m);" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c005", + "metadata": {}, + "outputs": [], + "source": [ "d = getenv(\"LKBENCH_RESULTS\"); if isempty(d), d = \"results\"; end\n", "if ~exist(d, \"dir\"), mkdir(d); end\n", "fid = fopen(fullfile(d, \"xbind_octave.csv\"), \"w\");\n", @@ -80,7 +140,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c003", + "id": "c006", "metadata": {}, "outputs": [], "source": [ diff --git a/deep-kernel-projected-branin_MLPKriging_python.ipynb b/deep-kernel-projected-branin_MLPKriging_python.ipynb index 2039666..22c966d 100644 --- a/deep-kernel-projected-branin_MLPKriging_python.ipynb +++ b/deep-kernel-projected-branin_MLPKriging_python.ipynb @@ -24,9 +24,45 @@ "id": "c001", "metadata": {}, "outputs": [], + "source": [ + "# Setup for running this notebook outside the repository (Colab, Binder, a lone copy):\n", + "# nothing to do in the repository itself (CI, local runs). Otherwise the repository\n", + "# (datasets of data/, helpers of tools/) is cloned and the released pylibkriging installed.\n", + "import importlib\n", + "import os\n", + "import subprocess\n", + "import sys\n", + "from pathlib import Path\n", + "\n", + "if not Path(\"tools/known_issues.csv\").is_file():\n", + " if not Path(\"benchmarks/tools/known_issues.csv\").is_file():\n", + " subprocess.run([\"git\", \"clone\", \"--depth\", \"1\", \"https://github.com/libKriging/benchmarks\"], check=True)\n", + " os.chdir(\"benchmarks\")\n", + " try:\n", + " import pylibkriging # noqa: F401\n", + " except ImportError:\n", + " r = subprocess.run([sys.executable, \"-m\", \"pip\", \"install\", \"--quiet\", \"pylibkriging\"],\n", + " capture_output=True, text=True)\n", + " if r.returncode != 0:\n", + " raise RuntimeError(\n", + " f\"pylibkriging could not be installed for Python {sys.version_info.major}.{sys.version_info.minor} \"\n", + " \"(PyPI has wheels for CPython 3.10 to 3.12 only). On Colab: Runtime > Change runtime version > \"\n", + " \"2026.07 (Python 3.12), then run again.\\n\" + r.stderr[-500:])\n", + " importlib.invalidate_caches()\n", + "sys.path.insert(0, str(Path(\"tools\").resolve())) # lkbench, when not installed (Colab, Binder)\n", + "print(\"working directory:\", os.getcwd())" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c002", + "metadata": {}, + "outputs": [], "source": [ "import time\n", "\n", + "import matplotlib.pyplot as plt\n", "import numpy as np\n", "import pylibkriging as lk\n", "\n", @@ -45,7 +81,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c002", + "id": "c003", "metadata": {}, "outputs": [], "source": [ @@ -82,14 +118,17 @@ " suite.greater(f\"{case}: Q2\", res[\"q2\"], 0.8)\n", " else:\n", " suite.greater(f\"{case}: Q2 - Q2(Kriging)\", res[\"q2\"] - res_k[\"q2\"], -0.1, severity=\"warn\")\n", - " display(suite.table().loc[\"projected_branin_10d\"].round(4))\n", + " tab = suite.table().loc[\"projected_branin_10d\"]\n", + " tab[\"q2\"].plot.barh(figsize=(6, 3)); plt.xlabel(\"Q² (test set)\"); plt.xlim(min(0, tab[\"q2\"].min()), 1)\n", + " plt.title(\"Branin(Wx), x in [0,1]^10\"); plt.show()\n", + " display(tab.round(4))\n", "else:\n", " suite.skip(\"MLPKriging\", \"not available in this pylibkriging\")" ] }, { "cell_type": "markdown", - "id": "c003", + "id": "c004", "metadata": {}, "source": [ "## save / load, pickle, copy\n", @@ -102,7 +141,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c004", + "id": "c005", "metadata": {}, "outputs": [], "source": [ @@ -116,7 +155,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c005", + "id": "c006", "metadata": {}, "outputs": [], "source": [ diff --git a/deep-kernel_MLPKriging_R.ipynb b/deep-kernel_MLPKriging_R.ipynb deleted file mode 100644 index 8aa9d10..0000000 --- a/deep-kernel_MLPKriging_R.ipynb +++ /dev/null @@ -1,124 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "c000", - "metadata": {}, - "source": [ - "# R · `MLPKriging` — a joint feature map ℝ² → ℝ² before the kernel\n", - "\n", - "Branin on [0,1]² (n = 30 LHS points unless stated, Q² on 500 uniform test\n", - "points), through rlibkriging. Checks (error-level unless stated):\n", - "\n", - "* 10 restarts (`BFGS10`): finite predictions, Q² > 0.9 (warn-level)\n", - "* `save` then `load`: the same predicted mean\n", - "* the default optimizer as good as `BFGS10` (warn-level; see libKriging/libKriging#382)\n", - "\n", - "A class missing from the installed rlibkriging (release channel) is recorded as SKIP." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c001", - "metadata": {}, - "outputs": [], - "source": [ - "suppressPackageStartupMessages(library(rlibkriging))\n", - "source(\"tools/r/checks.R\")\n", - "s <- new_suite(\"deep-kernel_MLPKriging_R\")\n", - "cat(\"rlibkriging\", as.character(packageVersion(\"rlibkriging\")), \"\\n\")\n", - "\n", - "branin <- function(U) { # on [0,1]^2\n", - " x1 <- 15 * U[, 1] - 5; x2 <- 15 * U[, 2]\n", - " (x2 - 5.1 / (4 * pi^2) * x1^2 + 5 / pi * x1 - 6)^2 + 10 * (1 - 1 / (8 * pi)) * cos(x1) + 10\n", - "}\n", - "set.seed(1)\n", - "lhs <- function(n, d) sapply(1:d, function(j) (sample(n) - runif(n)) / n)\n", - "X <- lhs(30, 2); y <- branin(X)\n", - "Xt <- matrix(runif(1000), ncol = 2); yt <- branin(Xt)\n", - "tmp <- tempdir()\n", - "has_class <- function(name) exists(name, mode = \"function\")\n", - "skip <- function(s, check, reason) {\n", - " s$checks[[length(s$checks) + 1]] <- data.frame(suite = s$name, lang = \"R\", check = check, severity = \"info\",\n", - " status = \"SKIP\", value = \"\", expected = \"\", detail = reason, stringsAsFactors = FALSE)\n", - " message(\"[SKIP] \", check, \": \", reason)\n", - "}\n", - "roundtrip_same_mean <- function(s, label, k, Xn) {\n", - " f <- file.path(tmp, paste0(gsub(\"[^A-Za-z0-9]\", \"_\", label), \".json\"))\n", - " check_runs(s, paste0(label, \": save + load\"), {\n", - " save(k, f)\n", - " k2 <- rlibkriging::load(f)\n", - " check_close(s, paste0(label, \": load(save(k)) predicts the same mean\"),\n", - " predict(k2, Xn, return_stdev = FALSE)$mean, predict(k, Xn, return_stdev = FALSE)$mean, rtol = 1e-10)\n", - " })\n", - "}" - ] - }, - { - "cell_type": "markdown", - "id": "c002", - "metadata": {}, - "source": [ - "## `MLPKriging`: a joint feature map ℝ² → ℝ² before the kernel" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c003", - "metadata": {}, - "outputs": [], - "source": [ - "if (has_class(\"MLPKriging\")) {\n", - " check_runs(s, \"MLPKriging: fit + predict\", {\n", - " km <- MLPKriging(y, X, hidden_dims = c(8), d_out = 2, activation = \"tanh\", kernel = \"matern5_2\", optim = \"BFGS10\")\n", - " pm <- predict(km, Xt, return_stdev = TRUE)\n", - " check_less(s, \"MLPKriging: non-finite predictions\", sum(!is.finite(c(pm$mean, pm$stdev))), 1)\n", - " add_metric(s, \"branin\", \"MLPKriging tanh [8] -> 2, BFGS10\", q2 = q2(yt, pm$mean))\n", - " check_greater(s, \"MLPKriging: Q2\", q2(yt, pm$mean), 0.9, severity = \"warn\")\n", - " roundtrip_same_mean(s, \"MLPKriging\", km, Xt)\n", - " # the default optimizer (\"BFGS+Adam\")\n", - " kd <- MLPKriging(y, X, hidden_dims = c(8), d_out = 2, activation = \"tanh\", kernel = \"matern5_2\")\n", - " qd <- q2(yt, predict(kd, Xt, return_stdev = FALSE)$mean)\n", - " add_metric(s, \"branin\", \"MLPKriging tanh [8] -> 2, default optim\", q2 = qd)\n", - " check_greater(s, \"MLPKriging: Q2(default optim) - Q2(BFGS10)\", qd - q2(yt, pm$mean), -0.05, severity = \"warn\")\n", - " })\n", - "} else skip(s, \"MLPKriging\", \"not in this rlibkriging\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c004", - "metadata": {}, - "outputs": [], - "source": [ - "me <- do.call(rbind, s$metrics)\n", - "print(me[me$metric == \"q2\", c(\"case\", \"package\", \"value\")], row.names = FALSE)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c005", - "metadata": {}, - "outputs": [], - "source": [ - "finalize(s)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "R", - "language": "R", - "name": "ir" - }, - "language_info": { - "name": "R" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/deep-kernel_MLPKriging_julia.ipynb b/deep-kernel_MLPKriging_julia.ipynb deleted file mode 100644 index cd8cbef..0000000 --- a/deep-kernel_MLPKriging_julia.ipynb +++ /dev/null @@ -1,126 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "c000", - "metadata": {}, - "source": [ - "# Julia · `MLPKriging` — a joint feature map ℝ² → ℝ² before the kernel\n", - "\n", - "Branin on [0,1]² (n = 30 LHS points unless stated, Q² on 500 uniform test\n", - "points), through JLibKriging. Checks (error-level unless stated):\n", - "\n", - "* 10 restarts (`BFGS10`): finite predictions, Q² > 0.9 (warn-level)\n", - "* `save` then `load`: the same predicted mean\n", - "* the default optimizer as good as `BFGS10` (warn-level; see libKriging/libKriging#382)\n", - "\n", - "The same checks as `deep-kernel_MLPKriging_R.ipynb`, with the Julia binding." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c001", - "metadata": {}, - "outputs": [], - "source": [ - "import JLibKriging as LK\n", - "using Random\n", - "include(\"tools/julia/checks.jl\")\n", - "s = Suite(\"deep-kernel_MLPKriging_julia\")\n", - "\n", - "branin(U) = begin\n", - " x1 = 15 .* U[:, 1] .- 5; x2 = 15 .* U[:, 2]\n", - " (x2 .- 5.1 / (4π^2) .* x1 .^ 2 .+ 5 / π .* x1 .- 6) .^ 2 .+ 10 * (1 - 1 / (8π)) .* cos.(x1) .+ 10\n", - "end\n", - "rng = MersenneTwister(1)\n", - "lhs(n, d) = hcat([(randperm(rng, n) .- rand(rng, n)) ./ n for _ in 1:d]...)\n", - "X = lhs(30, 2); y = branin(X)\n", - "Xt = rand(rng, 500, 2); yt = branin(Xt)\n", - "sd(v) = sqrt(sum((v .- sum(v) / length(v)) .^ 2) / (length(v) - 1))\n", - "finite_count(v...) = count(!isfinite, vcat(v...))\n", - "\n", - "function check_runs!(f, s, check)\n", - " try\n", - " f()\n", - " catch e\n", - " add_check!(s, check, false; value=string(typeof(e)), expected=\"no error\", detail=first(sprint(showerror, e), 200))\n", - " end\n", - "end\n", - "\n", - "function roundtrip!(s, label, k, Xn)\n", - " check_runs!(s, \"$label: save + load\") do\n", - " f = joinpath(mktempdir(), \"k.json\")\n", - " LK.save(k, f)\n", - " k2 = LK.load(f)\n", - " check_close!(s, \"$label: load(save(k)) predicts the same mean\",\n", - " LK.predict(k2, Xn; return_stdev=false).mean, LK.predict(k, Xn; return_stdev=false).mean; rtol=1e-10)\n", - " end\n", - "end" - ] - }, - { - "cell_type": "markdown", - "id": "c002", - "metadata": {}, - "source": [ - "## `MLPKriging`: a joint feature map ℝ² → ℝ² before the kernel" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c003", - "metadata": {}, - "outputs": [], - "source": [ - "check_runs!(s, \"MLPKriging: fit + predict\") do\n", - " km = LK.MLPKriging(y, X, [8], 2; activation=\"tanh\", kernel=\"matern5_2\", optim=\"BFGS10\")\n", - " pm = LK.predict(km, Xt; return_stdev=true)\n", - " add_check!(s, \"MLPKriging: finite predictions\", finite_count(pm.mean, pm.stdev) == 0;\n", - " value=finite_count(pm.mean, pm.stdev), expected=\"0 non-finite\")\n", - " add_metric!(s, \"branin\", \"MLPKriging tanh [8] -> 2, BFGS10\"; q2=q2(yt, pm.mean))\n", - " add_check!(s, \"MLPKriging: Q2\", q2(yt, pm.mean) > 0.9; value=q2(yt, pm.mean), expected=\"> 0.9\", severity=\"warn\")\n", - " roundtrip!(s, \"MLPKriging\", km, Xt)\n", - " kd = LK.MLPKriging(y, X, [8], 2; activation=\"tanh\", kernel=\"matern5_2\") # default optimizer\n", - " qd = q2(yt, LK.predict(kd, Xt; return_stdev=false).mean)\n", - " add_metric!(s, \"branin\", \"MLPKriging tanh [8] -> 2, default optim\"; q2=qd)\n", - " add_check!(s, \"MLPKriging: Q2(default optim) - Q2(BFGS10)\", qd - q2(yt, pm.mean) > -0.05;\n", - " value=qd - q2(yt, pm.mean), expected=\"> -0.05\", severity=\"warn\")\n", - "end" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c004", - "metadata": {}, - "outputs": [], - "source": [ - "foreach(r -> r[5] == \"q2\" && println(rpad(r[3], 24), rpad(r[4], 28), r[6]), s.metrics)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c005", - "metadata": {}, - "outputs": [], - "source": [ - "finalize!(s)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Julia 1.11", - "language": "julia", - "name": "julia-1.11" - }, - "language_info": { - "name": "julia" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/deep-kernel_MLPKriging_octave.ipynb b/deep-kernel_MLPKriging_octave.ipynb deleted file mode 100644 index fcf944b..0000000 --- a/deep-kernel_MLPKriging_octave.ipynb +++ /dev/null @@ -1,132 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "c000", - "metadata": {}, - "source": [ - "# Octave · `MLPKriging` — a joint feature map ℝ² → ℝ² before the kernel\n", - "\n", - "Branin on [0,1]² (n = 30 LHS points unless stated, Q² on 500 uniform test\n", - "points), through mLibKriging. Checks (error-level unless stated):\n", - "\n", - "* 10 restarts (`BFGS10`): finite predictions, Q² > 0.9 (warn-level)\n", - "* `save` then `load`: the same predicted mean\n", - "* the default optimizer as good as `BFGS10` (warn-level; see libKriging/libKriging#382)\n", - "\n", - "The same checks as `deep-kernel_MLPKriging_R.ipynb`, with the Octave binding.\n", - "\n", - "A class missing from the installed mLibKriging (release channel) is recorded as SKIP." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c001", - "metadata": {}, - "outputs": [], - "source": [ - "addpath(fullfile(pwd, \"tools\", \"octave\"));\n", - "s = lkb_suite(\"deep-kernel_MLPKriging_octave\");\n", - "q2 = @(y, m) 1 - sum((y - m).^2) / sum((y - mean(y)).^2);\n", - "branin = @(U) ((15 * U(:, 2)) - 5.1 / (4 * pi^2) * (15 * U(:, 1) - 5).^2 + 5 / pi * (15 * U(:, 1) - 5) - 6).^2 ...\n", - " + 10 * (1 - 1 / (8 * pi)) * cos(15 * U(:, 1) - 5) + 10;\n", - "rand(\"seed\", 1); randn(\"seed\", 1);\n", - "function U = lhs(n, d)\n", - " U = zeros(n, d);\n", - " for j = 1:d, U(:, j) = (randperm(n)' - rand(n, 1)) / n; end\n", - "end\n", - "X = lhs(30, 2); y = branin(X);\n", - "Xt = rand(500, 2); yt = branin(Xt);\n", - "tmp = tempname(); mkdir(tmp);\n", - "has_class = @(name) exist(name) == 2; % a class file on the path\n", - "\n", - "function s = roundtrip(s, label, k, Xn, loader)\n", - " f = fullfile(tempdir(), [regexprep(label, \"[^A-Za-z0-9]\", \"_\") \".json\"]);\n", - " try\n", - " k.save(f);\n", - " k2 = loader(f);\n", - " m1 = k.predict(Xn, false, false, false);\n", - " m2 = k2.predict(Xn, false, false, false);\n", - " s = lkb_close(s, [label \": load(save(k)) predicts the same mean\"], m2, m1, 1e-10);\n", - " catch err\n", - " s = lkb_check(s, [label \": save + load\"], false, \"error\", \"no error\", err.message);\n", - " end\n", - "end\n", - "\n", - "function s = skip(s, check, reason)\n", - " s.checks(end + 1, :) = {s.name, \"octave\", check, \"info\", \"SKIP\", \"\", \"\", reason};\n", - " printf(\"[SKIP] %s: %s\\n\", check, reason);\n", - "end" - ] - }, - { - "cell_type": "markdown", - "id": "c002", - "metadata": {}, - "source": [ - "## `MLPKriging`: a joint feature map ℝ² → ℝ² before the kernel" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c003", - "metadata": {}, - "outputs": [], - "source": [ - "if has_class(\"MLPKriging\")\n", - " try\n", - " km = MLPKriging(y, X, [8], 2, \"tanh\", \"matern5_2\", \"constant\", false, \"BFGS10\");\n", - " [mm, sm] = km.predict(Xt, true, false, false);\n", - " s = lkb_check(s, \"MLPKriging: finite predictions\", all(isfinite([mm; sm])), sum(~isfinite([mm; sm])), \"0 non-finite\");\n", - " s = lkb_metric(s, \"branin\", \"MLPKriging tanh [8] -> 2, BFGS10\", \"q2\", q2(yt, mm));\n", - " s = lkb_check(s, \"MLPKriging: Q2\", q2(yt, mm) > 0.9, q2(yt, mm), \"> 0.9\", \"\", \"warn\");\n", - " s = roundtrip(s, \"MLPKriging\", km, Xt, @load_kriging);\n", - " kd = MLPKriging(y, X, [8], 2, \"tanh\", \"matern5_2\"); % default optimizer\n", - " qd = q2(yt, kd.predict(Xt, false, false, false));\n", - " s = lkb_metric(s, \"branin\", \"MLPKriging tanh [8] -> 2, default optim\", \"q2\", qd);\n", - " s = lkb_check(s, \"MLPKriging: Q2(default optim) - Q2(BFGS10)\", qd - q2(yt, mm) > -0.05, qd - q2(yt, mm), \"> -0.05\", \"\", \"warn\");\n", - " catch err\n", - " s = lkb_check(s, \"MLPKriging: fit + predict\", false, \"error\", \"no error\", err.message);\n", - " end\n", - "else\n", - " s = skip(s, \"MLPKriging\", \"not in this mLibKriging\");\n", - "end" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c004", - "metadata": {}, - "outputs": [], - "source": [ - "q = strcmp(s.metrics(:, 5), \"q2\");\n", - "disp(s.metrics(q, [3, 4, 6]))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c005", - "metadata": {}, - "outputs": [], - "source": [ - "lkb_finalize(s);" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Octave", - "language": "octave", - "name": "octave" - }, - "language_info": { - "name": "octave" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/high-dimension-normalize-screening_Kriging_python.ipynb b/high-dimension-normalize-screening_Kriging_python.ipynb index 74da769..c77f611 100644 --- a/high-dimension-normalize-screening_Kriging_python.ipynb +++ b/high-dimension-normalize-screening_Kriging_python.ipynb @@ -35,9 +35,45 @@ "id": "c001", "metadata": {}, "outputs": [], + "source": [ + "# Setup for running this notebook outside the repository (Colab, Binder, a lone copy):\n", + "# nothing to do in the repository itself (CI, local runs). Otherwise the repository\n", + "# (datasets of data/, helpers of tools/) is cloned and the released pylibkriging installed.\n", + "import importlib\n", + "import os\n", + "import subprocess\n", + "import sys\n", + "from pathlib import Path\n", + "\n", + "if not Path(\"tools/known_issues.csv\").is_file():\n", + " if not Path(\"benchmarks/tools/known_issues.csv\").is_file():\n", + " subprocess.run([\"git\", \"clone\", \"--depth\", \"1\", \"https://github.com/libKriging/benchmarks\"], check=True)\n", + " os.chdir(\"benchmarks\")\n", + " try:\n", + " import pylibkriging # noqa: F401\n", + " except ImportError:\n", + " r = subprocess.run([sys.executable, \"-m\", \"pip\", \"install\", \"--quiet\", \"pylibkriging\"],\n", + " capture_output=True, text=True)\n", + " if r.returncode != 0:\n", + " raise RuntimeError(\n", + " f\"pylibkriging could not be installed for Python {sys.version_info.major}.{sys.version_info.minor} \"\n", + " \"(PyPI has wheels for CPython 3.10 to 3.12 only). On Colab: Runtime > Change runtime version > \"\n", + " \"2026.07 (Python 3.12), then run again.\\n\" + r.stderr[-500:])\n", + " importlib.invalidate_caches()\n", + "sys.path.insert(0, str(Path(\"tools\").resolve())) # lkbench, when not installed (Colab, Binder)\n", + "print(\"working directory:\", os.getcwd())" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c002", + "metadata": {}, + "outputs": [], "source": [ "import time\n", "\n", + "import matplotlib.pyplot as plt\n", "import numpy as np\n", "import pandas as pd\n", "import pylibkriging as lk\n", @@ -56,7 +92,7 @@ }, { "cell_type": "markdown", - "id": "c002", + "id": "c003", "metadata": {}, "source": [ "## Accuracy, unit-cube inputs vs native inputs with `normalize=True`" @@ -65,7 +101,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c003", + "id": "c004", "metadata": {}, "outputs": [], "source": [ @@ -100,7 +136,7 @@ }, { "cell_type": "markdown", - "id": "c004", + "id": "c005", "metadata": {}, "source": [ "## Screening: Welch et al. (1992)\n", @@ -113,7 +149,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c005", + "id": "c006", "metadata": {}, "outputs": [], "source": [ @@ -125,13 +161,18 @@ "active = [0, 3, 4, 11, 18, 19] # x1, x4, x5, x12, x19, x20\n", "suite.greater(\"welch: min theta(inactive) / max theta(active)\", theta[inactive].min() / theta[active].max(), 1.0,\n", " severity=\"warn\")\n", + "colors = [\"tab:red\" if i in active else \"tab:gray\" if i in inactive else \"tab:blue\" for i in range(20)]\n", + "plt.figure(figsize=(9, 3))\n", + "plt.bar([f\"x{i + 1}\" for i in range(20)], theta, color=colors)\n", + "plt.yscale(\"log\"); plt.ylabel(\"fitted θ (log)\")\n", + "plt.title(\"Welch (20D): red = most influential inputs, gray = inactive (x8, x16)\"); plt.show()\n", "pd.Series(theta, index=[f\"x{i + 1}\" for i in range(20)]).round(3).to_frame(\"theta\").T" ] }, { "cell_type": "code", "execution_count": null, - "id": "c006", + "id": "c007", "metadata": {}, "outputs": [], "source": [ diff --git a/interpolation-simulate-update-save_Kriging_R.ipynb b/interpolation-simulate-update-save_Kriging_R.ipynb deleted file mode 100644 index 90eb449..0000000 --- a/interpolation-simulate-update-save_Kriging_R.ipynb +++ /dev/null @@ -1,131 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "c000", - "metadata": {}, - "source": [ - "# R · `Kriging`, noise-free — interpolation, conditional simulation, update, save/load\n", - "\n", - "Branin on [0,1]² (n = 30 LHS points unless stated, Q² on 500 uniform test\n", - "points), through rlibkriging. Checks (error-level unless stated):\n", - "\n", - "* interpolation (max |m(X) − y| / sd(y) < 1e-6) and Q² > 0.95 on 500 test points\n", - "* conditional simulations through the data (relative to σ)\n", - "* stdev > 0 off the design\n", - "* `update` without refit = a fit on all the points at the same θ, σ²\n", - "* `save` then `load`: the same predicted mean" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c001", - "metadata": {}, - "outputs": [], - "source": [ - "suppressPackageStartupMessages(library(rlibkriging))\n", - "source(\"tools/r/checks.R\")\n", - "s <- new_suite(\"interpolation-simulate-update-save_Kriging_R\")\n", - "cat(\"rlibkriging\", as.character(packageVersion(\"rlibkriging\")), \"\\n\")\n", - "\n", - "branin <- function(U) { # on [0,1]^2\n", - " x1 <- 15 * U[, 1] - 5; x2 <- 15 * U[, 2]\n", - " (x2 - 5.1 / (4 * pi^2) * x1^2 + 5 / pi * x1 - 6)^2 + 10 * (1 - 1 / (8 * pi)) * cos(x1) + 10\n", - "}\n", - "set.seed(1)\n", - "lhs <- function(n, d) sapply(1:d, function(j) (sample(n) - runif(n)) / n)\n", - "X <- lhs(30, 2); y <- branin(X)\n", - "Xt <- matrix(runif(1000), ncol = 2); yt <- branin(Xt)\n", - "tmp <- tempdir()\n", - "has_class <- function(name) exists(name, mode = \"function\")\n", - "skip <- function(s, check, reason) {\n", - " s$checks[[length(s$checks) + 1]] <- data.frame(suite = s$name, lang = \"R\", check = check, severity = \"info\",\n", - " status = \"SKIP\", value = \"\", expected = \"\", detail = reason, stringsAsFactors = FALSE)\n", - " message(\"[SKIP] \", check, \": \", reason)\n", - "}\n", - "roundtrip_same_mean <- function(s, label, k, Xn) {\n", - " f <- file.path(tmp, paste0(gsub(\"[^A-Za-z0-9]\", \"_\", label), \".json\"))\n", - " check_runs(s, paste0(label, \": save + load\"), {\n", - " save(k, f)\n", - " k2 <- rlibkriging::load(f)\n", - " check_close(s, paste0(label, \": load(save(k)) predicts the same mean\"),\n", - " predict(k2, Xn, return_stdev = FALSE)$mean, predict(k, Xn, return_stdev = FALSE)$mean, rtol = 1e-10)\n", - " })\n", - "}" - ] - }, - { - "cell_type": "markdown", - "id": "c002", - "metadata": {}, - "source": [ - "## `Kriging`, noise-free" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c003", - "metadata": {}, - "outputs": [], - "source": [ - "k <- Kriging(y, X, \"matern5_2\")\n", - "print(k)\n", - "p <- predict(k, Xt, return_stdev = TRUE)\n", - "add_metric(s, \"branin\", \"Kriging\", q2 = q2(yt, p$mean), rmse = rmse(yt, p$mean))\n", - "check_greater(s, \"Kriging: Q2\", q2(yt, p$mean), 0.95)\n", - "check_less(s, \"Kriging: interpolates (max |m(X) - y| / sd(y))\",\n", - " max(abs(predict(k, X, return_stdev = FALSE)$mean - y)) / sd(y), 1e-6)\n", - "sims <- simulate(k, nsim = 5, seed = 2, x = X)\n", - "check_less(s, \"Kriging: conditional simulations go through the data (max |sim(X) - y| / sigma)\",\n", - " max(abs(sims - y)) / sqrt(k$sigma2()), 1e-4)\n", - "check_greater(s, \"Kriging: stdev > 0 off the design\", min(p$stdev), 0)\n", - "# update with 5 new points (no refit) = a fit on the 35 points at the same theta\n", - "Xu <- lhs(5, 2); yu <- branin(Xu)\n", - "ku <- copy(k)\n", - "update(ku, yu, Xu, refit = FALSE)\n", - "k35 <- Kriging(c(y, yu), rbind(X, Xu), \"matern5_2\", optim = \"none\",\n", - " parameters = list(theta = matrix(k$theta(), nrow = 1), is_theta_estim = FALSE,\n", - " sigma2 = k$sigma2(), is_sigma2_estim = FALSE))\n", - "check_close(s, \"Kriging: update(refit = FALSE) == fit on all points (same theta, sigma2)\",\n", - " predict(ku, Xt, return_stdev = FALSE)$mean, predict(k35, Xt, return_stdev = FALSE)$mean,\n", - " rtol = 1e-6, atol = 1e-8 * sd(y))\n", - "roundtrip_same_mean(s, \"Kriging\", k, Xt)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c004", - "metadata": {}, - "outputs": [], - "source": [ - "me <- do.call(rbind, s$metrics)\n", - "print(me[me$metric == \"q2\", c(\"case\", \"package\", \"value\")], row.names = FALSE)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c005", - "metadata": {}, - "outputs": [], - "source": [ - "finalize(s)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "R", - "language": "R", - "name": "ir" - }, - "language_info": { - "name": "R" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/interpolation-simulate-update-save_Kriging_julia.ipynb b/interpolation-simulate-update-save_Kriging_julia.ipynb deleted file mode 100644 index 3968d6f..0000000 --- a/interpolation-simulate-update-save_Kriging_julia.ipynb +++ /dev/null @@ -1,134 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "c000", - "metadata": {}, - "source": [ - "# Julia · `Kriging`, noise-free — interpolation, conditional simulation, update, save/load\n", - "\n", - "Branin on [0,1]² (n = 30 LHS points unless stated, Q² on 500 uniform test\n", - "points), through JLibKriging. Checks (error-level unless stated):\n", - "\n", - "* interpolation (max |m(X) − y| / sd(y) < 1e-6) and Q² > 0.95 on 500 test points\n", - "* conditional simulations through the data (relative to σ)\n", - "* stdev > 0 off the design\n", - "* `update` without refit = a fit on all the points at the same θ, σ²\n", - "* `save` then `load`: the same predicted mean\n", - "\n", - "The same checks as `interpolation-simulate-update-save_Kriging_R.ipynb`, with the Julia binding." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c001", - "metadata": {}, - "outputs": [], - "source": [ - "import JLibKriging as LK\n", - "using Random\n", - "include(\"tools/julia/checks.jl\")\n", - "s = Suite(\"interpolation-simulate-update-save_Kriging_julia\")\n", - "\n", - "branin(U) = begin\n", - " x1 = 15 .* U[:, 1] .- 5; x2 = 15 .* U[:, 2]\n", - " (x2 .- 5.1 / (4π^2) .* x1 .^ 2 .+ 5 / π .* x1 .- 6) .^ 2 .+ 10 * (1 - 1 / (8π)) .* cos.(x1) .+ 10\n", - "end\n", - "rng = MersenneTwister(1)\n", - "lhs(n, d) = hcat([(randperm(rng, n) .- rand(rng, n)) ./ n for _ in 1:d]...)\n", - "X = lhs(30, 2); y = branin(X)\n", - "Xt = rand(rng, 500, 2); yt = branin(Xt)\n", - "sd(v) = sqrt(sum((v .- sum(v) / length(v)) .^ 2) / (length(v) - 1))\n", - "finite_count(v...) = count(!isfinite, vcat(v...))\n", - "\n", - "function check_runs!(f, s, check)\n", - " try\n", - " f()\n", - " catch e\n", - " add_check!(s, check, false; value=string(typeof(e)), expected=\"no error\", detail=first(sprint(showerror, e), 200))\n", - " end\n", - "end\n", - "\n", - "function roundtrip!(s, label, k, Xn)\n", - " check_runs!(s, \"$label: save + load\") do\n", - " f = joinpath(mktempdir(), \"k.json\")\n", - " LK.save(k, f)\n", - " k2 = LK.load(f)\n", - " check_close!(s, \"$label: load(save(k)) predicts the same mean\",\n", - " LK.predict(k2, Xn; return_stdev=false).mean, LK.predict(k, Xn; return_stdev=false).mean; rtol=1e-10)\n", - " end\n", - "end" - ] - }, - { - "cell_type": "markdown", - "id": "c002", - "metadata": {}, - "source": [ - "## `Kriging`, noise-free" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c003", - "metadata": {}, - "outputs": [], - "source": [ - "k = LK.Kriging(y, X, \"matern5_2\")\n", - "p = LK.predict(k, Xt; return_stdev=true)\n", - "add_metric!(s, \"branin\", \"Kriging\"; q2=q2(yt, p.mean))\n", - "add_check!(s, \"Kriging: Q2\", q2(yt, p.mean) > 0.95; value=q2(yt, p.mean), expected=\"> 0.95\")\n", - "e = maximum(abs.(LK.predict(k, X; return_stdev=false).mean .- y)) / sd(y)\n", - "add_check!(s, \"Kriging: interpolates (max |m(X) - y| / sd(y))\", e < 1e-6; value=e, expected=\"< 1e-6\")\n", - "S = LK.simulate(k, 5, 2, X)\n", - "e = maximum(abs.(S .- y)) / sqrt(LK.sigma2(k))\n", - "add_check!(s, \"Kriging: conditional simulations go through the data (max |sim(X) - y| / sigma)\", e < 1e-4;\n", - " value=e, expected=\"< 1e-4\")\n", - "add_check!(s, \"Kriging: stdev > 0 off the design\", minimum(p.stdev) > 0; value=minimum(p.stdev), expected=\"> 0\")\n", - "Xu = lhs(5, 2); yu = branin(Xu)\n", - "ku = copy(k)\n", - "LK.update!(ku, yu, Xu; refit=false)\n", - "k35 = LK.Kriging(vcat(y, yu), vcat(X, Xu), \"matern5_2\"; optim=\"none\",\n", - " theta=vec(LK.theta(k)), is_theta_estim=false, sigma2=LK.sigma2(k), is_sigma2_estim=false)\n", - "check_close!(s, \"Kriging: update!(refit=false) == fit on all points (same theta, sigma2)\",\n", - " LK.predict(ku, Xt; return_stdev=false).mean, LK.predict(k35, Xt; return_stdev=false).mean;\n", - " rtol=1e-6, atol=1e-8 * sd(y))\n", - "roundtrip!(s, \"Kriging\", k, Xt)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c004", - "metadata": {}, - "outputs": [], - "source": [ - "foreach(r -> r[5] == \"q2\" && println(rpad(r[3], 24), rpad(r[4], 28), r[6]), s.metrics)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c005", - "metadata": {}, - "outputs": [], - "source": [ - "finalize!(s)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Julia 1.11", - "language": "julia", - "name": "julia-1.11" - }, - "language_info": { - "name": "julia" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/interpolation-simulate-update-save_Kriging_octave.ipynb b/interpolation-simulate-update-save_Kriging_octave.ipynb deleted file mode 100644 index e82db02..0000000 --- a/interpolation-simulate-update-save_Kriging_octave.ipynb +++ /dev/null @@ -1,134 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "c000", - "metadata": {}, - "source": [ - "# Octave · `Kriging`, noise-free — interpolation, conditional simulation, update, save/load\n", - "\n", - "Branin on [0,1]² (n = 30 LHS points unless stated, Q² on 500 uniform test\n", - "points), through mLibKriging. Checks (error-level unless stated):\n", - "\n", - "* interpolation (max |m(X) − y| / sd(y) < 1e-6) and Q² > 0.95 on 500 test points\n", - "* conditional simulations through the data (relative to σ)\n", - "* stdev > 0 off the design\n", - "* `update` without refit = a fit on all the points at the same θ, σ²\n", - "* `save` then `load`: the same predicted mean\n", - "\n", - "The same checks as `interpolation-simulate-update-save_Kriging_R.ipynb`, with the Octave binding." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c001", - "metadata": {}, - "outputs": [], - "source": [ - "addpath(fullfile(pwd, \"tools\", \"octave\"));\n", - "s = lkb_suite(\"interpolation-simulate-update-save_Kriging_octave\");\n", - "q2 = @(y, m) 1 - sum((y - m).^2) / sum((y - mean(y)).^2);\n", - "branin = @(U) ((15 * U(:, 2)) - 5.1 / (4 * pi^2) * (15 * U(:, 1) - 5).^2 + 5 / pi * (15 * U(:, 1) - 5) - 6).^2 ...\n", - " + 10 * (1 - 1 / (8 * pi)) * cos(15 * U(:, 1) - 5) + 10;\n", - "rand(\"seed\", 1); randn(\"seed\", 1);\n", - "function U = lhs(n, d)\n", - " U = zeros(n, d);\n", - " for j = 1:d, U(:, j) = (randperm(n)' - rand(n, 1)) / n; end\n", - "end\n", - "X = lhs(30, 2); y = branin(X);\n", - "Xt = rand(500, 2); yt = branin(Xt);\n", - "tmp = tempname(); mkdir(tmp);\n", - "has_class = @(name) exist(name) == 2; % a class file on the path\n", - "\n", - "function s = roundtrip(s, label, k, Xn, loader)\n", - " f = fullfile(tempdir(), [regexprep(label, \"[^A-Za-z0-9]\", \"_\") \".json\"]);\n", - " try\n", - " k.save(f);\n", - " k2 = loader(f);\n", - " m1 = k.predict(Xn, false, false, false);\n", - " m2 = k2.predict(Xn, false, false, false);\n", - " s = lkb_close(s, [label \": load(save(k)) predicts the same mean\"], m2, m1, 1e-10);\n", - " catch err\n", - " s = lkb_check(s, [label \": save + load\"], false, \"error\", \"no error\", err.message);\n", - " end\n", - "end\n", - "\n", - "function s = skip(s, check, reason)\n", - " s.checks(end + 1, :) = {s.name, \"octave\", check, \"info\", \"SKIP\", \"\", \"\", reason};\n", - " printf(\"[SKIP] %s: %s\\n\", check, reason);\n", - "end" - ] - }, - { - "cell_type": "markdown", - "id": "c002", - "metadata": {}, - "source": [ - "## `Kriging`, noise-free" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c003", - "metadata": {}, - "outputs": [], - "source": [ - "k = Kriging(y, X, \"matern5_2\");\n", - "[m, sd] = k.predict(Xt, true, false, false);\n", - "s = lkb_metric(s, \"branin\", \"Kriging\", \"q2\", q2(yt, m));\n", - "s = lkb_check(s, \"Kriging: Q2\", q2(yt, m) > 0.95, q2(yt, m), \"> 0.95\");\n", - "mX = k.predict(X, false, false, false);\n", - "e = max(abs(mX - y)) / std(y);\n", - "s = lkb_check(s, \"Kriging: interpolates (max |m(X) - y| / sd(y))\", e < 1e-6, e, \"< 1e-6\");\n", - "S = k.simulate(int32(5), int32(2), X, false);\n", - "e = max(max(abs(S - y))) / sqrt(k.sigma2());\n", - "s = lkb_check(s, \"Kriging: conditional simulations go through the data (max |sim(X) - y| / sigma)\", e < 1e-4, e, \"< 1e-4\");\n", - "s = lkb_check(s, \"Kriging: stdev > 0 off the design\", min(sd) > 0, min(sd), \"> 0\");\n", - "Xu = lhs(5, 2); yu = branin(Xu);\n", - "ku = k.copy();\n", - "ku.update(yu, Xu, false);\n", - "th = k.theta();\n", - "k35 = Kriging([y; yu], [X; Xu], \"matern5_2\", \"constant\", false, \"none\", \"LL\", ...\n", - " Params(\"theta\", th(:)', \"is_theta_estim\", false, \"sigma2\", k.sigma2(), \"is_sigma2_estim\", false));\n", - "s = lkb_close(s, \"Kriging: update(refit = false) == fit on all points (same theta, sigma2)\", ...\n", - " ku.predict(Xt, false, false, false), k35.predict(Xt, false, false, false), 1e-6, 1e-8 * std(y));\n", - "s = roundtrip(s, \"Kriging\", k, Xt, @load_kriging);" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c004", - "metadata": {}, - "outputs": [], - "source": [ - "q = strcmp(s.metrics(:, 5), \"q2\");\n", - "disp(s.metrics(q, [3, 4, 6]))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c005", - "metadata": {}, - "outputs": [], - "source": [ - "lkb_finalize(s);" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Octave", - "language": "octave", - "name": "octave" - }, - "language_info": { - "name": "octave" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/kernels-trends-interpolation_Kriging_python.ipynb b/kernels-trends-interpolation_Kriging_python.ipynb index a04e7d4..1b1f150 100644 --- a/kernels-trends-interpolation_Kriging_python.ipynb +++ b/kernels-trends-interpolation_Kriging_python.ipynb @@ -46,6 +46,41 @@ "id": "c001", "metadata": {}, "outputs": [], + "source": [ + "# Setup for running this notebook outside the repository (Colab, Binder, a lone copy):\n", + "# nothing to do in the repository itself (CI, local runs). Otherwise the repository\n", + "# (datasets of data/, helpers of tools/) is cloned and the released pylibkriging installed.\n", + "import importlib\n", + "import os\n", + "import subprocess\n", + "import sys\n", + "from pathlib import Path\n", + "\n", + "if not Path(\"tools/known_issues.csv\").is_file():\n", + " if not Path(\"benchmarks/tools/known_issues.csv\").is_file():\n", + " subprocess.run([\"git\", \"clone\", \"--depth\", \"1\", \"https://github.com/libKriging/benchmarks\"], check=True)\n", + " os.chdir(\"benchmarks\")\n", + " try:\n", + " import pylibkriging # noqa: F401\n", + " except ImportError:\n", + " r = subprocess.run([sys.executable, \"-m\", \"pip\", \"install\", \"--quiet\", \"pylibkriging\"],\n", + " capture_output=True, text=True)\n", + " if r.returncode != 0:\n", + " raise RuntimeError(\n", + " f\"pylibkriging could not be installed for Python {sys.version_info.major}.{sys.version_info.minor} \"\n", + " \"(PyPI has wheels for CPython 3.10 to 3.12 only). On Colab: Runtime > Change runtime version > \"\n", + " \"2026.07 (Python 3.12), then run again.\\n\" + r.stderr[-500:])\n", + " importlib.invalidate_caches()\n", + "sys.path.insert(0, str(Path(\"tools\").resolve())) # lkbench, when not installed (Colab, Binder)\n", + "print(\"working directory:\", os.getcwd())" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c002", + "metadata": {}, + "outputs": [], "source": [ "import time\n", "\n", @@ -81,7 +116,7 @@ }, { "cell_type": "markdown", - "id": "c002", + "id": "c003", "metadata": {}, "source": [ "## Kernel × trend grid" @@ -90,7 +125,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c003", + "id": "c004", "metadata": {}, "outputs": [], "source": [ @@ -137,7 +172,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c004", + "id": "c005", "metadata": {}, "outputs": [], "source": [ @@ -150,7 +185,7 @@ }, { "cell_type": "markdown", - "id": "c005", + "id": "c006", "metadata": {}, "source": [ "Expectations that should hold whatever the version (warn-level: they are\n", @@ -162,7 +197,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c006", + "id": "c007", "metadata": {}, "outputs": [], "source": [ @@ -177,7 +212,7 @@ }, { "cell_type": "markdown", - "id": "c007", + "id": "c008", "metadata": {}, "source": [ "## Consistency of `predict`" @@ -186,7 +221,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c008", + "id": "c009", "metadata": {}, "outputs": [], "source": [ @@ -209,7 +244,7 @@ }, { "cell_type": "markdown", - "id": "c009", + "id": "c010", "metadata": {}, "source": [ "### Derivatives of the predictor\n", @@ -221,7 +256,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c010", + "id": "c011", "metadata": {}, "outputs": [], "source": [ @@ -241,7 +276,7 @@ }, { "cell_type": "markdown", - "id": "c011", + "id": "c012", "metadata": {}, "source": [ "## A look at the 1D fits" @@ -250,7 +285,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c012", + "id": "c013", "metadata": {}, "outputs": [], "source": [ @@ -274,7 +309,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c013", + "id": "c014", "metadata": {}, "outputs": [], "source": [ diff --git a/large-n_Kriging-vecchia-nystrom_R.ipynb b/large-n_Kriging-vecchia-nystrom_R.ipynb deleted file mode 100644 index 4240136..0000000 --- a/large-n_Kriging-vecchia-nystrom_R.ipynb +++ /dev/null @@ -1,120 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "c000", - "metadata": {}, - "source": [ - "# R · `Kriging` with an approximate fit — Vecchia, Nyström (n = 400)\n", - "\n", - "Branin on [0,1]² (n = 30 LHS points unless stated, Q² on 500 uniform test\n", - "points), through rlibkriging. Checks (error-level unless stated):\n", - "\n", - "* the fit method is reported by `method()`\n", - "* `predict` with the fit method: finite mean and stdev\n", - "* Q² > 0.95 (warn-level)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c001", - "metadata": {}, - "outputs": [], - "source": [ - "suppressPackageStartupMessages(library(rlibkriging))\n", - "source(\"tools/r/checks.R\")\n", - "s <- new_suite(\"large-n_Kriging-vecchia-nystrom_R\")\n", - "cat(\"rlibkriging\", as.character(packageVersion(\"rlibkriging\")), \"\\n\")\n", - "\n", - "branin <- function(U) { # on [0,1]^2\n", - " x1 <- 15 * U[, 1] - 5; x2 <- 15 * U[, 2]\n", - " (x2 - 5.1 / (4 * pi^2) * x1^2 + 5 / pi * x1 - 6)^2 + 10 * (1 - 1 / (8 * pi)) * cos(x1) + 10\n", - "}\n", - "set.seed(1)\n", - "lhs <- function(n, d) sapply(1:d, function(j) (sample(n) - runif(n)) / n)\n", - "X <- lhs(30, 2); y <- branin(X)\n", - "Xt <- matrix(runif(1000), ncol = 2); yt <- branin(Xt)\n", - "tmp <- tempdir()\n", - "has_class <- function(name) exists(name, mode = \"function\")\n", - "skip <- function(s, check, reason) {\n", - " s$checks[[length(s$checks) + 1]] <- data.frame(suite = s$name, lang = \"R\", check = check, severity = \"info\",\n", - " status = \"SKIP\", value = \"\", expected = \"\", detail = reason, stringsAsFactors = FALSE)\n", - " message(\"[SKIP] \", check, \": \", reason)\n", - "}\n", - "roundtrip_same_mean <- function(s, label, k, Xn) {\n", - " f <- file.path(tmp, paste0(gsub(\"[^A-Za-z0-9]\", \"_\", label), \".json\"))\n", - " check_runs(s, paste0(label, \": save + load\"), {\n", - " save(k, f)\n", - " k2 <- rlibkriging::load(f)\n", - " check_close(s, paste0(label, \": load(save(k)) predicts the same mean\"),\n", - " predict(k2, Xn, return_stdev = FALSE)$mean, predict(k, Xn, return_stdev = FALSE)$mean, rtol = 1e-10)\n", - " })\n", - "}" - ] - }, - { - "cell_type": "markdown", - "id": "c002", - "metadata": {}, - "source": [ - "## `Kriging` with an approximate fit: Vecchia, Nyström (n = 400)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c003", - "metadata": {}, - "outputs": [], - "source": [ - "Xl <- lhs(400, 2); yl <- branin(Xl)\n", - "for (m in c(\"vecchia(20)\", \"nystrom(60)\")) {\n", - " short <- sub(\"\\\\(.*\", \"\", m)\n", - " check_runs(s, paste0(\"Kriging(method = '\", m, \"'): fit + predict\"), {\n", - " ka <- Kriging(yl, Xl, \"matern5_2\", method = m)\n", - " add_check(s, paste0(\"Kriging(method = '\", m, \"'): method()\"), identical(ka$method(), m), ka$method(), m)\n", - " pa <- predict(ka, Xt, return_stdev = TRUE, method = short)\n", - " check_less(s, paste0(\"Kriging(method = '\", m, \"'): non-finite predictions\"),\n", - " sum(!is.finite(c(pa$mean, pa$stdev))), 1)\n", - " add_metric(s, \"branin n=400\", paste(\"Kriging\", m), q2 = q2(yt, pa$mean))\n", - " check_greater(s, paste0(\"Kriging(method = '\", m, \"'): Q2\"), q2(yt, pa$mean), 0.95, severity = \"warn\")\n", - " })\n", - "}" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c004", - "metadata": {}, - "outputs": [], - "source": [ - "me <- do.call(rbind, s$metrics)\n", - "print(me[me$metric == \"q2\", c(\"case\", \"package\", \"value\")], row.names = FALSE)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c005", - "metadata": {}, - "outputs": [], - "source": [ - "finalize(s)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "R", - "language": "R", - "name": "ir" - }, - "language_info": { - "name": "R" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/large-n_Kriging-vecchia-nystrom_julia.ipynb b/large-n_Kriging-vecchia-nystrom_julia.ipynb deleted file mode 100644 index 5240454..0000000 --- a/large-n_Kriging-vecchia-nystrom_julia.ipynb +++ /dev/null @@ -1,126 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "c000", - "metadata": {}, - "source": [ - "# Julia · `Kriging` with an approximate fit — Vecchia, Nyström (n = 400)\n", - "\n", - "Branin on [0,1]² (n = 30 LHS points unless stated, Q² on 500 uniform test\n", - "points), through JLibKriging. Checks (error-level unless stated):\n", - "\n", - "* the fit method is reported by `method()`\n", - "* `predict` with the fit method: finite mean and stdev\n", - "* Q² > 0.95 (warn-level)\n", - "\n", - "The same checks as `large-n_Kriging-vecchia-nystrom_R.ipynb`, with the Julia binding." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c001", - "metadata": {}, - "outputs": [], - "source": [ - "import JLibKriging as LK\n", - "using Random\n", - "include(\"tools/julia/checks.jl\")\n", - "s = Suite(\"large-n_Kriging-vecchia-nystrom_julia\")\n", - "\n", - "branin(U) = begin\n", - " x1 = 15 .* U[:, 1] .- 5; x2 = 15 .* U[:, 2]\n", - " (x2 .- 5.1 / (4π^2) .* x1 .^ 2 .+ 5 / π .* x1 .- 6) .^ 2 .+ 10 * (1 - 1 / (8π)) .* cos.(x1) .+ 10\n", - "end\n", - "rng = MersenneTwister(1)\n", - "lhs(n, d) = hcat([(randperm(rng, n) .- rand(rng, n)) ./ n for _ in 1:d]...)\n", - "X = lhs(30, 2); y = branin(X)\n", - "Xt = rand(rng, 500, 2); yt = branin(Xt)\n", - "sd(v) = sqrt(sum((v .- sum(v) / length(v)) .^ 2) / (length(v) - 1))\n", - "finite_count(v...) = count(!isfinite, vcat(v...))\n", - "\n", - "function check_runs!(f, s, check)\n", - " try\n", - " f()\n", - " catch e\n", - " add_check!(s, check, false; value=string(typeof(e)), expected=\"no error\", detail=first(sprint(showerror, e), 200))\n", - " end\n", - "end\n", - "\n", - "function roundtrip!(s, label, k, Xn)\n", - " check_runs!(s, \"$label: save + load\") do\n", - " f = joinpath(mktempdir(), \"k.json\")\n", - " LK.save(k, f)\n", - " k2 = LK.load(f)\n", - " check_close!(s, \"$label: load(save(k)) predicts the same mean\",\n", - " LK.predict(k2, Xn; return_stdev=false).mean, LK.predict(k, Xn; return_stdev=false).mean; rtol=1e-10)\n", - " end\n", - "end" - ] - }, - { - "cell_type": "markdown", - "id": "c002", - "metadata": {}, - "source": [ - "## `Kriging` with an approximate fit: Vecchia, Nyström (n = 400)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c003", - "metadata": {}, - "outputs": [], - "source": [ - "Xl = lhs(400, 2); yl = branin(Xl)\n", - "for m in (\"vecchia(20)\", \"nystrom(60)\")\n", - " short = split(m, \"(\")[1]\n", - " label = \"Kriging(method = '$m')\"\n", - " check_runs!(s, \"$label: fit + predict\") do\n", - " ka = LK.Kriging(yl, Xl, \"matern5_2\"; method=m)\n", - " add_check!(s, \"$label: method()\", LK.method(ka) == m; value=LK.method(ka), expected=m)\n", - " pa = LK.predict(ka, Xt; return_stdev=true, method=String(short))\n", - " add_check!(s, \"$label: finite predictions\", finite_count(pa.mean, pa.stdev) == 0;\n", - " value=finite_count(pa.mean, pa.stdev), expected=\"0 non-finite\")\n", - " add_metric!(s, \"branin n=400\", \"Kriging $m\"; q2=q2(yt, pa.mean))\n", - " add_check!(s, \"$label: Q2\", q2(yt, pa.mean) > 0.95; value=q2(yt, pa.mean), expected=\"> 0.95\", severity=\"warn\")\n", - " end\n", - "end" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c004", - "metadata": {}, - "outputs": [], - "source": [ - "foreach(r -> r[5] == \"q2\" && println(rpad(r[3], 24), rpad(r[4], 28), r[6]), s.metrics)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c005", - "metadata": {}, - "outputs": [], - "source": [ - "finalize!(s)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Julia 1.11", - "language": "julia", - "name": "julia-1.11" - }, - "language_info": { - "name": "julia" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/large-n_Kriging-vecchia-nystrom_octave.ipynb b/large-n_Kriging-vecchia-nystrom_octave.ipynb deleted file mode 100644 index 3e5a3a3..0000000 --- a/large-n_Kriging-vecchia-nystrom_octave.ipynb +++ /dev/null @@ -1,128 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "c000", - "metadata": {}, - "source": [ - "# Octave · `Kriging` with an approximate fit — Vecchia, Nyström (n = 400)\n", - "\n", - "Branin on [0,1]² (n = 30 LHS points unless stated, Q² on 500 uniform test\n", - "points), through mLibKriging. Checks (error-level unless stated):\n", - "\n", - "* the fit method is reported by `method()`\n", - "* `predict` with the fit method: finite mean and stdev\n", - "* Q² > 0.95 (warn-level)\n", - "\n", - "The same checks as `large-n_Kriging-vecchia-nystrom_R.ipynb`, with the Octave binding." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c001", - "metadata": {}, - "outputs": [], - "source": [ - "addpath(fullfile(pwd, \"tools\", \"octave\"));\n", - "s = lkb_suite(\"large-n_Kriging-vecchia-nystrom_octave\");\n", - "q2 = @(y, m) 1 - sum((y - m).^2) / sum((y - mean(y)).^2);\n", - "branin = @(U) ((15 * U(:, 2)) - 5.1 / (4 * pi^2) * (15 * U(:, 1) - 5).^2 + 5 / pi * (15 * U(:, 1) - 5) - 6).^2 ...\n", - " + 10 * (1 - 1 / (8 * pi)) * cos(15 * U(:, 1) - 5) + 10;\n", - "rand(\"seed\", 1); randn(\"seed\", 1);\n", - "function U = lhs(n, d)\n", - " U = zeros(n, d);\n", - " for j = 1:d, U(:, j) = (randperm(n)' - rand(n, 1)) / n; end\n", - "end\n", - "X = lhs(30, 2); y = branin(X);\n", - "Xt = rand(500, 2); yt = branin(Xt);\n", - "tmp = tempname(); mkdir(tmp);\n", - "has_class = @(name) exist(name) == 2; % a class file on the path\n", - "\n", - "function s = roundtrip(s, label, k, Xn, loader)\n", - " f = fullfile(tempdir(), [regexprep(label, \"[^A-Za-z0-9]\", \"_\") \".json\"]);\n", - " try\n", - " k.save(f);\n", - " k2 = loader(f);\n", - " m1 = k.predict(Xn, false, false, false);\n", - " m2 = k2.predict(Xn, false, false, false);\n", - " s = lkb_close(s, [label \": load(save(k)) predicts the same mean\"], m2, m1, 1e-10);\n", - " catch err\n", - " s = lkb_check(s, [label \": save + load\"], false, \"error\", \"no error\", err.message);\n", - " end\n", - "end\n", - "\n", - "function s = skip(s, check, reason)\n", - " s.checks(end + 1, :) = {s.name, \"octave\", check, \"info\", \"SKIP\", \"\", \"\", reason};\n", - " printf(\"[SKIP] %s: %s\\n\", check, reason);\n", - "end" - ] - }, - { - "cell_type": "markdown", - "id": "c002", - "metadata": {}, - "source": [ - "## `Kriging` with an approximate fit: Vecchia, Nyström (n = 400)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c003", - "metadata": {}, - "outputs": [], - "source": [ - "Xl = lhs(400, 2); yl = branin(Xl);\n", - "for mth = {\"vecchia(20)\", \"nystrom(60)\"}\n", - " mth = mth{1}; short = regexprep(mth, \"\\\\(.*\", \"\");\n", - " label = [\"Kriging(method = '\" mth \"')\"];\n", - " try\n", - " ka = Kriging(yl, Xl, \"matern5_2\", \"constant\", false, \"BFGS\", \"LL\", [], \"none\", [], mth);\n", - " got = ka.method();\n", - " s = lkb_check(s, [label \": method()\"], strcmp(got, mth), got, mth);\n", - " [ma, sa] = ka.predict(Xt, true, false, false, short);\n", - " s = lkb_check(s, [label \": finite predictions\"], all(isfinite([ma; sa])), sum(~isfinite([ma; sa])), \"0 non-finite\");\n", - " s = lkb_metric(s, \"branin n=400\", [\"Kriging \" mth], \"q2\", q2(yt, ma));\n", - " s = lkb_check(s, [label \": Q2\"], q2(yt, ma) > 0.95, q2(yt, ma), \"> 0.95\", \"\", \"warn\");\n", - " catch err\n", - " s = lkb_check(s, [label \": fit + predict\"], false, \"error\", \"no error\", err.message);\n", - " end\n", - "end" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c004", - "metadata": {}, - "outputs": [], - "source": [ - "q = strcmp(s.metrics(:, 5), \"q2\");\n", - "disp(s.metrics(q, [3, 4, 6]))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c005", - "metadata": {}, - "outputs": [], - "source": [ - "lkb_finalize(s);" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Octave", - "language": "octave", - "name": "octave" - }, - "language_info": { - "name": "octave" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/large-n_Kriging-vecchia-nystrom_python.ipynb b/large-n_Kriging-vecchia-nystrom_python.ipynb index e414b18..4d8d3d1 100644 --- a/large-n_Kriging-vecchia-nystrom_python.ipynb +++ b/large-n_Kriging-vecchia-nystrom_python.ipynb @@ -23,6 +23,10 @@ "for reference) and n = 5000 (approximations only).\n", "\n", "Checks:\n", + "* **convergence to the exact model** at fixed θ (Hartmann 3, n = 300): the gap\n", + " between the approximate and the exact log-likelihood and predicted mean,\n", + " as m (Vecchia) and k (Nyström) grow, down to the limit cases m = n − 1 and\n", + " k = n where the approximation *is* the exact model;\n", "* **limit cases**: Vecchia with m = n − 1 neighbours *is* the exact likelihood;\n", "* the method is explicit at prediction time: `predict(method=\"exact\")` on a\n", " Nyström/Vecchia fit raises, `method=` gives finite predictions;\n", @@ -36,9 +40,45 @@ "id": "c001", "metadata": {}, "outputs": [], + "source": [ + "# Setup for running this notebook outside the repository (Colab, Binder, a lone copy):\n", + "# nothing to do in the repository itself (CI, local runs). Otherwise the repository\n", + "# (datasets of data/, helpers of tools/) is cloned and the released pylibkriging installed.\n", + "import importlib\n", + "import os\n", + "import subprocess\n", + "import sys\n", + "from pathlib import Path\n", + "\n", + "if not Path(\"tools/known_issues.csv\").is_file():\n", + " if not Path(\"benchmarks/tools/known_issues.csv\").is_file():\n", + " subprocess.run([\"git\", \"clone\", \"--depth\", \"1\", \"https://github.com/libKriging/benchmarks\"], check=True)\n", + " os.chdir(\"benchmarks\")\n", + " try:\n", + " import pylibkriging # noqa: F401\n", + " except ImportError:\n", + " r = subprocess.run([sys.executable, \"-m\", \"pip\", \"install\", \"--quiet\", \"pylibkriging\"],\n", + " capture_output=True, text=True)\n", + " if r.returncode != 0:\n", + " raise RuntimeError(\n", + " f\"pylibkriging could not be installed for Python {sys.version_info.major}.{sys.version_info.minor} \"\n", + " \"(PyPI has wheels for CPython 3.10 to 3.12 only). On Colab: Runtime > Change runtime version > \"\n", + " \"2026.07 (Python 3.12), then run again.\\n\" + r.stderr[-500:])\n", + " importlib.invalidate_caches()\n", + "sys.path.insert(0, str(Path(\"tools\").resolve())) # lkbench, when not installed (Colab, Binder)\n", + "print(\"working directory:\", os.getcwd())" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c002", + "metadata": {}, + "outputs": [], "source": [ "import time\n", "\n", + "import matplotlib.pyplot as plt\n", "import numpy as np\n", "import pandas as pd\n", "import pylibkriging as lk\n", @@ -66,7 +106,7 @@ }, { "cell_type": "markdown", - "id": "c002", + "id": "c003", "metadata": {}, "source": [ "## Limit case: Vecchia with all the previous points as neighbours is exact" @@ -75,7 +115,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c003", + "id": "c004", "metadata": {}, "outputs": [], "source": [ @@ -101,7 +141,60 @@ }, { "cell_type": "markdown", - "id": "c004", + "id": "c005", + "metadata": {}, + "source": [ + "## Convergence to the exact model, at fixed θ\n", + "\n", + "Hartmann 3, n = 300, θ of the exact fit kept fixed: the reference is the exact\n", + "model with the same hyperparameters, so the gap measures the approximation\n", + "alone (no optimizer effect). Relative gap of the log-likelihood, and RMS gap\n", + "of the predicted mean on 500 test points relative to its standard deviation." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c006", + "metadata": {}, + "outputs": [], + "source": [ + "if HAS_METHOD:\n", + " tf = TESTS[\"hartmann3\"]\n", + " X, y, Xt, yt = design.dataset(tf, 300, n_test=500, seed=54)\n", + " n = len(y)\n", + " theta = np.asarray(lk.Kriging(y, X, \"matern5_2\").theta()).reshape(1, -1)\n", + " fixed = {\"theta\": theta, \"is_theta_estim\": False}\n", + " ke = lk.Kriging(y, X, \"matern5_2\", optim=\"none\", parameters=fixed)\n", + " ll_e, m_e = ke.logLikelihood(), pred(ke, Xt)[0]\n", + " rows = []\n", + " for meth, sizes in [(\"vecchia\", [2, 5, 10, 20, 40, 80, 160, n - 1]), (\"nystrom\", [10, 25, 50, 100, 200, n])]:\n", + " for size in sizes:\n", + " with suite.runs(f\"{meth}({size}) at fixed theta: fit + predict\"):\n", + " ka = lk.Kriging(y, X, \"matern5_2\", optim=\"none\", parameters=fixed, method=f\"{meth}({size})\")\n", + " rows.append(dict(method=meth, size=size,\n", + " ll_gap=abs(ka.logLikelihood(method=meth) - ll_e) / abs(ll_e),\n", + " mean_gap=np.sqrt(np.mean((pred(ka, Xt, method=meth)[0] - m_e) ** 2)) / np.std(m_e)))\n", + " conv = pd.DataFrame(rows)\n", + " for meth, g in conv.groupby(\"method\"):\n", + " suite.metric(f\"hartmann3/n={n}/fixed theta\", meth, **{f\"ll_gap_{s}\": v for s, v in zip(g[\"size\"], g[\"ll_gap\"])})\n", + " for col, lim in [(\"ll_gap\", 1e-6), (\"mean_gap\", 1e-2)]:\n", + " suite.less(f\"{meth}: {col} at the limit size ({g['size'].iloc[-1]})\", g[col].iloc[-1], lim)\n", + " suite.true(f\"{meth}: {col} decreases with the size\", bool(np.all(np.diff(g[col].to_numpy()) < 0)),\n", + " detail=\", \".join(f\"{v:.2g}\" for v in g[col]), severity=\"warn\")\n", + " fig, axes = plt.subplots(1, 2, figsize=(10, 3.8))\n", + " for ax, col, label in [(axes[0], \"ll_gap\", \"|LL − LL(exact)| / |LL(exact)|\"),\n", + " (axes[1], \"mean_gap\", \"RMS(mean − mean(exact)) / sd\")]:\n", + " for meth, g in conv.groupby(\"method\"):\n", + " ax.loglog(g[\"size\"], g[col], \"o-\", label=f\"{meth}(m)\" if meth == \"vecchia\" else f\"{meth}(k)\")\n", + " ax.set_xlabel(\"m (neighbours) or k (landmarks)\"); ax.set_ylabel(label); ax.grid(True, which=\"both\", alpha=.3)\n", + " axes[0].legend(); fig.suptitle(f\"Hartmann 3, n = {n}, θ fixed: approximation → exact model\"); plt.show()\n", + " display(conv.pivot(index=\"size\", columns=\"method\", values=[\"ll_gap\", \"mean_gap\"]).round(6))" + ] + }, + { + "cell_type": "markdown", + "id": "c007", "metadata": {}, "source": [ "## Accuracy and cost" @@ -110,7 +203,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c005", + "id": "c008", "metadata": {}, "outputs": [], "source": [ @@ -156,7 +249,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c006", + "id": "c009", "metadata": {}, "outputs": [], "source": [ diff --git a/noisy-data_Kriging-nugget-noise_R.ipynb b/noisy-data_Kriging-nugget-noise_R.ipynb deleted file mode 100644 index 65c0196..0000000 --- a/noisy-data_Kriging-nugget-noise_R.ipynb +++ /dev/null @@ -1,137 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "c000", - "metadata": {}, - "source": [ - "# R · `Kriging` with noise — estimated nugget, known variances\n", - "\n", - "Branin on [0,1]² (n = 30 LHS points unless stated, Q² on 500 uniform test\n", - "points), through rlibkriging. Checks (error-level unless stated):\n", - "\n", - "* `noise = \"nugget\"` (10 restarts): nugget > 0, Q² > 0.95 on the noise-free function, smoothing next to the design points\n", - "* the default single start reaches the multistart likelihood (warn-level)\n", - "* `save` then `load`: the same predicted mean\n", - "* `noise = `: the stored variances, finite predictions" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c001", - "metadata": {}, - "outputs": [], - "source": [ - "suppressPackageStartupMessages(library(rlibkriging))\n", - "source(\"tools/r/checks.R\")\n", - "s <- new_suite(\"noisy-data_Kriging-nugget-noise_R\")\n", - "cat(\"rlibkriging\", as.character(packageVersion(\"rlibkriging\")), \"\\n\")\n", - "\n", - "branin <- function(U) { # on [0,1]^2\n", - " x1 <- 15 * U[, 1] - 5; x2 <- 15 * U[, 2]\n", - " (x2 - 5.1 / (4 * pi^2) * x1^2 + 5 / pi * x1 - 6)^2 + 10 * (1 - 1 / (8 * pi)) * cos(x1) + 10\n", - "}\n", - "set.seed(1)\n", - "lhs <- function(n, d) sapply(1:d, function(j) (sample(n) - runif(n)) / n)\n", - "X <- lhs(30, 2); y <- branin(X)\n", - "Xt <- matrix(runif(1000), ncol = 2); yt <- branin(Xt)\n", - "tmp <- tempdir()\n", - "has_class <- function(name) exists(name, mode = \"function\")\n", - "skip <- function(s, check, reason) {\n", - " s$checks[[length(s$checks) + 1]] <- data.frame(suite = s$name, lang = \"R\", check = check, severity = \"info\",\n", - " status = \"SKIP\", value = \"\", expected = \"\", detail = reason, stringsAsFactors = FALSE)\n", - " message(\"[SKIP] \", check, \": \", reason)\n", - "}\n", - "roundtrip_same_mean <- function(s, label, k, Xn) {\n", - " f <- file.path(tmp, paste0(gsub(\"[^A-Za-z0-9]\", \"_\", label), \".json\"))\n", - " check_runs(s, paste0(label, \": save + load\"), {\n", - " save(k, f)\n", - " k2 <- rlibkriging::load(f)\n", - " check_close(s, paste0(label, \": load(save(k)) predicts the same mean\"),\n", - " predict(k2, Xn, return_stdev = FALSE)$mean, predict(k, Xn, return_stdev = FALSE)$mean, rtol = 1e-10)\n", - " })\n", - "}" - ] - }, - { - "cell_type": "markdown", - "id": "c002", - "metadata": {}, - "source": [ - "## `Kriging` with noise: estimated nugget, known variances\n", - "\n", - "Noise of standard deviation 5% of sd(y). The nugget model is fitted with 10\n", - "restarts (`optim = \"BFGS10\"`); the default single start is compared with it\n", - "(warn-level: a lower likelihood means a local optimum). As DiceKriging's\n", - "nugget, the predictor returns y exactly *at* the design points (the nugget is\n", - "a micro-scale variation): smoothing is checked next to them." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c003", - "metadata": {}, - "outputs": [], - "source": [ - "sd_noise <- 0.05 * sd(y)\n", - "yn <- y + rnorm(length(y), sd = sd_noise)\n", - "kn <- Kriging(yn, X, \"matern5_2\", noise = \"nugget\", optim = \"BFGS10\")\n", - "pn <- predict(kn, Xt, return_stdev = TRUE)\n", - "add_metric(s, \"branin noisy\", \"Kriging nugget BFGS10\", q2 = q2(yt, pn$mean), nugget = kn$nugget(),\n", - " ll = logLikelihood(kn))\n", - "check_greater(s, \"Kriging(nugget): estimated nugget > 0\", kn$nugget(), 0)\n", - "check_greater(s, \"Kriging(nugget): Q2\", q2(yt, pn$mean), 0.95)\n", - "near <- pmin(X + 1e-7, 1)\n", - "check_greater(s, \"Kriging(nugget): smooths the noisy data next to the design points (rms residual / noise sd)\",\n", - " rmse(yn, predict(kn, near, return_stdev = FALSE)$mean) / sd_noise, 0.3)\n", - "kn1 <- Kriging(yn, X, \"matern5_2\", noise = \"nugget\")\n", - "add_metric(s, \"branin noisy\", \"Kriging nugget BFGS\", q2 = q2(yt, predict(kn1, Xt, return_stdev = FALSE)$mean),\n", - " nugget = kn1$nugget(), ll = logLikelihood(kn1))\n", - "check_greater(s, \"Kriging(nugget): LL(single start) - LL(10 restarts)\", logLikelihood(kn1) - logLikelihood(kn), -1,\n", - " severity = \"warn\")\n", - "roundtrip_same_mean(s, \"Kriging(nugget)\", kn, Xt)\n", - "kh <- Kriging(yn, X, \"matern5_2\", noise = rep(sd_noise^2, length(yn)))\n", - "check_close(s, \"Kriging(noise = variances): noise() returns them\", kh$noise(), rep(sd_noise^2, length(yn)),\n", - " rtol = 1e-12)\n", - "ph <- predict(kh, Xt, return_stdev = TRUE)\n", - "check_less(s, \"Kriging(noise = variances): non-finite predictions\", sum(!is.finite(c(ph$mean, ph$stdev))), 1)\n", - "add_metric(s, \"branin noisy\", \"Kriging known noise\", q2 = q2(yt, ph$mean))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c004", - "metadata": {}, - "outputs": [], - "source": [ - "me <- do.call(rbind, s$metrics)\n", - "print(me[me$metric == \"q2\", c(\"case\", \"package\", \"value\")], row.names = FALSE)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c005", - "metadata": {}, - "outputs": [], - "source": [ - "finalize(s)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "R", - "language": "R", - "name": "ir" - }, - "language_info": { - "name": "R" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/noisy-data_Kriging-nugget-noise_julia.ipynb b/noisy-data_Kriging-nugget-noise_julia.ipynb deleted file mode 100644 index 77c1ad7..0000000 --- a/noisy-data_Kriging-nugget-noise_julia.ipynb +++ /dev/null @@ -1,141 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "c000", - "metadata": {}, - "source": [ - "# Julia · `Kriging` with noise — estimated nugget, known variances\n", - "\n", - "Branin on [0,1]² (n = 30 LHS points unless stated, Q² on 500 uniform test\n", - "points), through JLibKriging. Checks (error-level unless stated):\n", - "\n", - "* `noise = \"nugget\"` (10 restarts): nugget > 0, Q² > 0.95 on the noise-free function, smoothing next to the design points\n", - "* the default single start reaches the multistart likelihood (warn-level)\n", - "* `save` then `load`: the same predicted mean\n", - "* `noise = `: the stored variances, finite predictions\n", - "\n", - "The same checks as `noisy-data_Kriging-nugget-noise_R.ipynb`, with the Julia binding." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c001", - "metadata": {}, - "outputs": [], - "source": [ - "import JLibKriging as LK\n", - "using Random\n", - "include(\"tools/julia/checks.jl\")\n", - "s = Suite(\"noisy-data_Kriging-nugget-noise_julia\")\n", - "\n", - "branin(U) = begin\n", - " x1 = 15 .* U[:, 1] .- 5; x2 = 15 .* U[:, 2]\n", - " (x2 .- 5.1 / (4π^2) .* x1 .^ 2 .+ 5 / π .* x1 .- 6) .^ 2 .+ 10 * (1 - 1 / (8π)) .* cos.(x1) .+ 10\n", - "end\n", - "rng = MersenneTwister(1)\n", - "lhs(n, d) = hcat([(randperm(rng, n) .- rand(rng, n)) ./ n for _ in 1:d]...)\n", - "X = lhs(30, 2); y = branin(X)\n", - "Xt = rand(rng, 500, 2); yt = branin(Xt)\n", - "sd(v) = sqrt(sum((v .- sum(v) / length(v)) .^ 2) / (length(v) - 1))\n", - "finite_count(v...) = count(!isfinite, vcat(v...))\n", - "\n", - "function check_runs!(f, s, check)\n", - " try\n", - " f()\n", - " catch e\n", - " add_check!(s, check, false; value=string(typeof(e)), expected=\"no error\", detail=first(sprint(showerror, e), 200))\n", - " end\n", - "end\n", - "\n", - "function roundtrip!(s, label, k, Xn)\n", - " check_runs!(s, \"$label: save + load\") do\n", - " f = joinpath(mktempdir(), \"k.json\")\n", - " LK.save(k, f)\n", - " k2 = LK.load(f)\n", - " check_close!(s, \"$label: load(save(k)) predicts the same mean\",\n", - " LK.predict(k2, Xn; return_stdev=false).mean, LK.predict(k, Xn; return_stdev=false).mean; rtol=1e-10)\n", - " end\n", - "end" - ] - }, - { - "cell_type": "markdown", - "id": "c002", - "metadata": {}, - "source": [ - "## `Kriging` with noise: estimated nugget, known variances\n", - "\n", - "Noise of standard deviation 5% of sd(y). The nugget model is fitted with 10\n", - "restarts (`optim=\"BFGS10\"`); the default single start is compared with it\n", - "(warn-level: a lower likelihood means a local optimum). As DiceKriging's\n", - "nugget, the predictor returns y exactly *at* the design points (the nugget is\n", - "a micro-scale variation): smoothing is checked next to them." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c003", - "metadata": {}, - "outputs": [], - "source": [ - "sd_noise = 0.05 * sd(y)\n", - "yn = y .+ sd_noise .* randn(rng, length(y))\n", - "kn = LK.Kriging(yn, X, \"matern5_2\"; noise=\"nugget\", optim=\"BFGS10\")\n", - "mn = LK.predict(kn, Xt; return_stdev=false).mean\n", - "add_metric!(s, \"branin noisy\", \"Kriging nugget BFGS10\"; q2=q2(yt, mn))\n", - "add_check!(s, \"Kriging(nugget): estimated nugget > 0\", LK.nugget(kn) > 0; value=LK.nugget(kn), expected=\"> 0\")\n", - "add_check!(s, \"Kriging(nugget): Q2\", q2(yt, mn) > 0.95; value=q2(yt, mn), expected=\"> 0.95\")\n", - "r = sqrt(sum((LK.predict(kn, min.(X .+ 1e-7, 1.0); return_stdev=false).mean .- yn) .^ 2) / length(yn)) / sd_noise\n", - "add_check!(s, \"Kriging(nugget): smooths the noisy data next to the design points (rms residual / noise sd)\", r > 0.3;\n", - " value=r, expected=\"> 0.3\")\n", - "kn1 = LK.Kriging(yn, X, \"matern5_2\"; noise=\"nugget\")\n", - "add_metric!(s, \"branin noisy\", \"Kriging nugget BFGS\"; q2=q2(yt, LK.predict(kn1, Xt; return_stdev=false).mean))\n", - "dll = LK.log_likelihood(kn1) - LK.log_likelihood(kn)\n", - "add_check!(s, \"Kriging(nugget): LL(single start) - LL(10 restarts)\", dll > -1; value=dll, expected=\"> -1\", severity=\"warn\")\n", - "roundtrip!(s, \"Kriging(nugget)\", kn, Xt)\n", - "v = fill(sd_noise^2, length(yn))\n", - "kh = LK.Kriging(yn, X, \"matern5_2\"; noise=v)\n", - "check_close!(s, \"Kriging(noise = variances): noise() returns them\", LK.noise(kh), v; rtol=1e-12)\n", - "ph = LK.predict(kh, Xt; return_stdev=true)\n", - "add_check!(s, \"Kriging(noise = variances): finite predictions\", finite_count(ph.mean, ph.stdev) == 0;\n", - " value=finite_count(ph.mean, ph.stdev), expected=\"0 non-finite\")\n", - "add_metric!(s, \"branin noisy\", \"Kriging known noise\"; q2=q2(yt, ph.mean))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c004", - "metadata": {}, - "outputs": [], - "source": [ - "foreach(r -> r[5] == \"q2\" && println(rpad(r[3], 24), rpad(r[4], 28), r[6]), s.metrics)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c005", - "metadata": {}, - "outputs": [], - "source": [ - "finalize!(s)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Julia 1.11", - "language": "julia", - "name": "julia-1.11" - }, - "language_info": { - "name": "julia" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/noisy-data_Kriging-nugget-noise_octave.ipynb b/noisy-data_Kriging-nugget-noise_octave.ipynb deleted file mode 100644 index 0741b79..0000000 --- a/noisy-data_Kriging-nugget-noise_octave.ipynb +++ /dev/null @@ -1,140 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "c000", - "metadata": {}, - "source": [ - "# Octave · `Kriging` with noise — estimated nugget, known variances\n", - "\n", - "Branin on [0,1]² (n = 30 LHS points unless stated, Q² on 500 uniform test\n", - "points), through mLibKriging. Checks (error-level unless stated):\n", - "\n", - "* `noise = \"nugget\"` (10 restarts): nugget > 0, Q² > 0.95 on the noise-free function, smoothing next to the design points\n", - "* the default single start reaches the multistart likelihood (warn-level)\n", - "* `save` then `load`: the same predicted mean\n", - "* `noise = `: the stored variances, finite predictions\n", - "\n", - "The same checks as `noisy-data_Kriging-nugget-noise_R.ipynb`, with the Octave binding." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c001", - "metadata": {}, - "outputs": [], - "source": [ - "addpath(fullfile(pwd, \"tools\", \"octave\"));\n", - "s = lkb_suite(\"noisy-data_Kriging-nugget-noise_octave\");\n", - "q2 = @(y, m) 1 - sum((y - m).^2) / sum((y - mean(y)).^2);\n", - "branin = @(U) ((15 * U(:, 2)) - 5.1 / (4 * pi^2) * (15 * U(:, 1) - 5).^2 + 5 / pi * (15 * U(:, 1) - 5) - 6).^2 ...\n", - " + 10 * (1 - 1 / (8 * pi)) * cos(15 * U(:, 1) - 5) + 10;\n", - "rand(\"seed\", 1); randn(\"seed\", 1);\n", - "function U = lhs(n, d)\n", - " U = zeros(n, d);\n", - " for j = 1:d, U(:, j) = (randperm(n)' - rand(n, 1)) / n; end\n", - "end\n", - "X = lhs(30, 2); y = branin(X);\n", - "Xt = rand(500, 2); yt = branin(Xt);\n", - "tmp = tempname(); mkdir(tmp);\n", - "has_class = @(name) exist(name) == 2; % a class file on the path\n", - "\n", - "function s = roundtrip(s, label, k, Xn, loader)\n", - " f = fullfile(tempdir(), [regexprep(label, \"[^A-Za-z0-9]\", \"_\") \".json\"]);\n", - " try\n", - " k.save(f);\n", - " k2 = loader(f);\n", - " m1 = k.predict(Xn, false, false, false);\n", - " m2 = k2.predict(Xn, false, false, false);\n", - " s = lkb_close(s, [label \": load(save(k)) predicts the same mean\"], m2, m1, 1e-10);\n", - " catch err\n", - " s = lkb_check(s, [label \": save + load\"], false, \"error\", \"no error\", err.message);\n", - " end\n", - "end\n", - "\n", - "function s = skip(s, check, reason)\n", - " s.checks(end + 1, :) = {s.name, \"octave\", check, \"info\", \"SKIP\", \"\", \"\", reason};\n", - " printf(\"[SKIP] %s: %s\\n\", check, reason);\n", - "end" - ] - }, - { - "cell_type": "markdown", - "id": "c002", - "metadata": {}, - "source": [ - "## `Kriging` with noise: estimated nugget, known variances\n", - "\n", - "Noise of standard deviation 5% of sd(y). The nugget model is fitted with 10\n", - "restarts (`\"BFGS10\"`); the default single start is compared with it\n", - "(warn-level: a lower likelihood means a local optimum). As DiceKriging's\n", - "nugget, the predictor returns y exactly *at* the design points (the nugget is\n", - "a micro-scale variation): smoothing is checked next to them." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c003", - "metadata": {}, - "outputs": [], - "source": [ - "sd_noise = 0.05 * std(y);\n", - "yn = y + sd_noise * randn(size(y));\n", - "kn = Kriging(yn, X, \"matern5_2\", \"constant\", false, \"BFGS10\", \"LL\", [], \"nugget\");\n", - "mn = kn.predict(Xt, false, false, false);\n", - "nug = kn.nugget(); ll10 = kn.logLikelihood();\n", - "s = lkb_metric(s, \"branin noisy\", \"Kriging nugget BFGS10\", \"q2\", q2(yt, mn));\n", - "s = lkb_check(s, \"Kriging(nugget): estimated nugget > 0\", nug > 0, nug, \"> 0\");\n", - "s = lkb_check(s, \"Kriging(nugget): Q2\", q2(yt, mn) > 0.95, q2(yt, mn), \"> 0.95\");\n", - "r = sqrt(mean((kn.predict(min(X + 1e-7, 1), false, false, false) - yn).^2)) / sd_noise;\n", - "s = lkb_check(s, \"Kriging(nugget): smooths the noisy data next to the design points (rms residual / noise sd)\", r > 0.3, r, \"> 0.3\");\n", - "kn1 = Kriging(yn, X, \"matern5_2\", \"constant\", false, \"BFGS\", \"LL\", [], \"nugget\");\n", - "ll1 = kn1.logLikelihood();\n", - "s = lkb_metric(s, \"branin noisy\", \"Kriging nugget BFGS\", \"q2\", q2(yt, kn1.predict(Xt, false, false, false)));\n", - "s = lkb_check(s, \"Kriging(nugget): LL(single start) - LL(10 restarts)\", ll1 - ll10 > -1, ll1 - ll10, \"> -1\", \"\", \"warn\");\n", - "s = roundtrip(s, \"Kriging(nugget)\", kn, Xt, @load_kriging);\n", - "v = sd_noise^2 * ones(size(yn));\n", - "kh = Kriging(yn, X, \"matern5_2\", \"constant\", false, \"BFGS\", \"LL\", [], \"heterogeneous\", v);\n", - "s = lkb_close(s, \"Kriging(noise = variances): noise() returns them\", kh.noise(), v, 1e-12);\n", - "[mh, sh] = kh.predict(Xt, true, false, false);\n", - "s = lkb_check(s, \"Kriging(noise = variances): finite predictions\", all(isfinite([mh; sh])), sum(~isfinite([mh; sh])), \"0 non-finite\");\n", - "s = lkb_metric(s, \"branin noisy\", \"Kriging known noise\", \"q2\", q2(yt, mh));" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c004", - "metadata": {}, - "outputs": [], - "source": [ - "q = strcmp(s.metrics(:, 5), \"q2\");\n", - "disp(s.metrics(q, [3, 4, 6]))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c005", - "metadata": {}, - "outputs": [], - "source": [ - "lkb_finalize(s);" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Octave", - "language": "octave", - "name": "octave" - }, - "language_info": { - "name": "octave" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/noisy-data_Kriging-nugget-noise_python.ipynb b/noisy-data_Kriging-nugget-noise_python.ipynb index e06bc56..c895332 100644 --- a/noisy-data_Kriging-nugget-noise_python.ipynb +++ b/noisy-data_Kriging-nugget-noise_python.ipynb @@ -39,6 +39,41 @@ "id": "c001", "metadata": {}, "outputs": [], + "source": [ + "# Setup for running this notebook outside the repository (Colab, Binder, a lone copy):\n", + "# nothing to do in the repository itself (CI, local runs). Otherwise the repository\n", + "# (datasets of data/, helpers of tools/) is cloned and the released pylibkriging installed.\n", + "import importlib\n", + "import os\n", + "import subprocess\n", + "import sys\n", + "from pathlib import Path\n", + "\n", + "if not Path(\"tools/known_issues.csv\").is_file():\n", + " if not Path(\"benchmarks/tools/known_issues.csv\").is_file():\n", + " subprocess.run([\"git\", \"clone\", \"--depth\", \"1\", \"https://github.com/libKriging/benchmarks\"], check=True)\n", + " os.chdir(\"benchmarks\")\n", + " try:\n", + " import pylibkriging # noqa: F401\n", + " except ImportError:\n", + " r = subprocess.run([sys.executable, \"-m\", \"pip\", \"install\", \"--quiet\", \"pylibkriging\"],\n", + " capture_output=True, text=True)\n", + " if r.returncode != 0:\n", + " raise RuntimeError(\n", + " f\"pylibkriging could not be installed for Python {sys.version_info.major}.{sys.version_info.minor} \"\n", + " \"(PyPI has wheels for CPython 3.10 to 3.12 only). On Colab: Runtime > Change runtime version > \"\n", + " \"2026.07 (Python 3.12), then run again.\\n\" + r.stderr[-500:])\n", + " importlib.invalidate_caches()\n", + "sys.path.insert(0, str(Path(\"tools\").resolve())) # lkbench, when not installed (Colab, Binder)\n", + "print(\"working directory:\", os.getcwd())" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c002", + "metadata": {}, + "outputs": [], "source": [ "import numpy as np\n", "import pandas as pd\n", @@ -59,7 +94,7 @@ }, { "cell_type": "markdown", - "id": "c002", + "id": "c003", "metadata": {}, "source": [ "## Homoscedastic noise, estimated nugget" @@ -68,7 +103,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c003", + "id": "c004", "metadata": {}, "outputs": [], "source": [ @@ -81,6 +116,11 @@ " res = metrics.all_metrics(yt, m, s)\n", " suite.metric(fname, \"nugget\", nugget=nugget, true_var=sd ** 2, **res)\n", " suite.between(f\"{fname}: nugget / true noise variance\", nugget / sd ** 2, 1 / 3, 3)\n", + " k1 = lk.Kriging(y, X, \"matern5_2\", noise=\"nugget\") # default single start\n", + " suite.metric(fname, \"nugget, single start\", nugget=float(np.ravel(k1.nugget())[0]), ll=float(k1.logLikelihood()),\n", + " **metrics.all_metrics(yt, *pred(k1, Xt)))\n", + " suite.greater(f\"{fname}: LL(single start) - LL(10 restarts)\", k1.logLikelihood() - k.logLikelihood(), -1,\n", + " severity=\"warn\")\n", " suite.greater(f\"{fname}: Q2 vs noise-free truth\", res[\"q2\"], q2_floor)\n", " mX, _ = pred(k, X)\n", " mX_near, _ = pred(k, np.clip(X + 1e-7, 0, 1))\n", @@ -101,7 +141,7 @@ }, { "cell_type": "markdown", - "id": "c004", + "id": "c005", "metadata": {}, "source": [ "## A zero noise vector is the noise-free model" @@ -110,7 +150,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c005", + "id": "c006", "metadata": {}, "outputs": [], "source": [ @@ -133,7 +173,7 @@ }, { "cell_type": "markdown", - "id": "c006", + "id": "c007", "metadata": {}, "source": [ "## Heteroscedastic noise: known variances vs a single nugget" @@ -142,7 +182,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c007", + "id": "c008", "metadata": {}, "outputs": [], "source": [ @@ -175,7 +215,7 @@ }, { "cell_type": "markdown", - "id": "c008", + "id": "c009", "metadata": {}, "source": [ "## Replicates\n", @@ -188,7 +228,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c009", + "id": "c010", "metadata": {}, "outputs": [], "source": [ @@ -212,7 +252,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c010", + "id": "c011", "metadata": {}, "outputs": [], "source": [ diff --git a/objectives-LL-LOO-LMP-gradients_Kriging_python.ipynb b/objectives-LL-LOO-LMP-gradients_Kriging_python.ipynb index 6961b48..09656f1 100644 --- a/objectives-LL-LOO-LMP-gradients_Kriging_python.ipynb +++ b/objectives-LL-LOO-LMP-gradients_Kriging_python.ipynb @@ -37,6 +37,41 @@ "id": "c001", "metadata": {}, "outputs": [], + "source": [ + "# Setup for running this notebook outside the repository (Colab, Binder, a lone copy):\n", + "# nothing to do in the repository itself (CI, local runs). Otherwise the repository\n", + "# (datasets of data/, helpers of tools/) is cloned and the released pylibkriging installed.\n", + "import importlib\n", + "import os\n", + "import subprocess\n", + "import sys\n", + "from pathlib import Path\n", + "\n", + "if not Path(\"tools/known_issues.csv\").is_file():\n", + " if not Path(\"benchmarks/tools/known_issues.csv\").is_file():\n", + " subprocess.run([\"git\", \"clone\", \"--depth\", \"1\", \"https://github.com/libKriging/benchmarks\"], check=True)\n", + " os.chdir(\"benchmarks\")\n", + " try:\n", + " import pylibkriging # noqa: F401\n", + " except ImportError:\n", + " r = subprocess.run([sys.executable, \"-m\", \"pip\", \"install\", \"--quiet\", \"pylibkriging\"],\n", + " capture_output=True, text=True)\n", + " if r.returncode != 0:\n", + " raise RuntimeError(\n", + " f\"pylibkriging could not be installed for Python {sys.version_info.major}.{sys.version_info.minor} \"\n", + " \"(PyPI has wheels for CPython 3.10 to 3.12 only). On Colab: Runtime > Change runtime version > \"\n", + " \"2026.07 (Python 3.12), then run again.\\n\" + r.stderr[-500:])\n", + " importlib.invalidate_caches()\n", + "sys.path.insert(0, str(Path(\"tools\").resolve())) # lkbench, when not installed (Colab, Binder)\n", + "print(\"working directory:\", os.getcwd())" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c002", + "metadata": {}, + "outputs": [], "source": [ "import numpy as np\n", "import pandas as pd\n", @@ -63,7 +98,7 @@ }, { "cell_type": "markdown", - "id": "c002", + "id": "c003", "metadata": {}, "source": [ "## Analytic gradients vs finite differences" @@ -72,7 +107,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c003", + "id": "c004", "metadata": {}, "outputs": [], "source": [ @@ -100,7 +135,7 @@ }, { "cell_type": "markdown", - "id": "c004", + "id": "c005", "metadata": {}, "source": [ "## Optimality of the fitted θ, objective by objective" @@ -109,7 +144,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c005", + "id": "c006", "metadata": {}, "outputs": [], "source": [ @@ -140,7 +175,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c006", + "id": "c007", "metadata": {}, "outputs": [], "source": [ @@ -151,7 +186,7 @@ }, { "cell_type": "markdown", - "id": "c007", + "id": "c008", "metadata": {}, "source": [ "## `logLikelihood()` == `logLikelihoodFun(theta())` (and the same for LOO, LMP)" @@ -160,7 +195,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c008", + "id": "c009", "metadata": {}, "outputs": [], "source": [ @@ -174,7 +209,7 @@ }, { "cell_type": "markdown", - "id": "c009", + "id": "c010", "metadata": {}, "source": [ "## Closed-form leave-one-out vs explicit refits\n", @@ -188,7 +223,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c010", + "id": "c011", "metadata": {}, "outputs": [], "source": [ @@ -220,7 +255,7 @@ }, { "cell_type": "markdown", - "id": "c011", + "id": "c012", "metadata": {}, "source": [ "## Multistart\n", @@ -232,7 +267,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c012", + "id": "c013", "metadata": {}, "outputs": [], "source": [ @@ -254,7 +289,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c013", + "id": "c014", "metadata": {}, "outputs": [], "source": [ diff --git a/robustness-invalid-inputs_Kriging_python.ipynb b/robustness-invalid-inputs_Kriging_python.ipynb index b18b41d..b1cc8a0 100644 --- a/robustness-invalid-inputs_Kriging_python.ipynb +++ b/robustness-invalid-inputs_Kriging_python.ipynb @@ -31,9 +31,45 @@ "id": "c001", "metadata": {}, "outputs": [], + "source": [ + "# Setup for running this notebook outside the repository (Colab, Binder, a lone copy):\n", + "# nothing to do in the repository itself (CI, local runs). Otherwise the repository\n", + "# (datasets of data/, helpers of tools/) is cloned and the released pylibkriging installed.\n", + "import importlib\n", + "import os\n", + "import subprocess\n", + "import sys\n", + "from pathlib import Path\n", + "\n", + "if not Path(\"tools/known_issues.csv\").is_file():\n", + " if not Path(\"benchmarks/tools/known_issues.csv\").is_file():\n", + " subprocess.run([\"git\", \"clone\", \"--depth\", \"1\", \"https://github.com/libKriging/benchmarks\"], check=True)\n", + " os.chdir(\"benchmarks\")\n", + " try:\n", + " import pylibkriging # noqa: F401\n", + " except ImportError:\n", + " r = subprocess.run([sys.executable, \"-m\", \"pip\", \"install\", \"--quiet\", \"pylibkriging\"],\n", + " capture_output=True, text=True)\n", + " if r.returncode != 0:\n", + " raise RuntimeError(\n", + " f\"pylibkriging could not be installed for Python {sys.version_info.major}.{sys.version_info.minor} \"\n", + " \"(PyPI has wheels for CPython 3.10 to 3.12 only). On Colab: Runtime > Change runtime version > \"\n", + " \"2026.07 (Python 3.12), then run again.\\n\" + r.stderr[-500:])\n", + " importlib.invalidate_caches()\n", + "sys.path.insert(0, str(Path(\"tools\").resolve())) # lkbench, when not installed (Colab, Binder)\n", + "print(\"working directory:\", os.getcwd())" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c002", + "metadata": {}, + "outputs": [], "source": [ "import time\n", "\n", + "import matplotlib.pyplot as plt\n", "import numpy as np\n", "import pylibkriging as lk\n", "\n", @@ -67,7 +103,7 @@ }, { "cell_type": "markdown", - "id": "c002", + "id": "c003", "metadata": {}, "source": [ "## Invalid inputs are rejected" @@ -76,7 +112,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c003", + "id": "c004", "metadata": {}, "outputs": [], "source": [ @@ -104,7 +140,7 @@ }, { "cell_type": "markdown", - "id": "c004", + "id": "c005", "metadata": {}, "source": [ "## Degenerate designs and responses" @@ -113,7 +149,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c005", + "id": "c006", "metadata": {}, "outputs": [], "source": [ @@ -138,7 +174,7 @@ }, { "cell_type": "markdown", - "id": "c006", + "id": "c007", "metadata": {}, "source": [ "## Scale invariance and equivariance" @@ -147,7 +183,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c007", + "id": "c008", "metadata": {}, "outputs": [], "source": [ @@ -175,7 +211,7 @@ }, { "cell_type": "markdown", - "id": "c008", + "id": "c009", "metadata": {}, "source": [ "## Far from the data" @@ -184,7 +220,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c009", + "id": "c010", "metadata": {}, "outputs": [], "source": [ @@ -200,7 +236,7 @@ }, { "cell_type": "markdown", - "id": "c010", + "id": "c011", "metadata": {}, "source": [ "## Many prediction points\n", @@ -215,7 +251,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c011", + "id": "c012", "metadata": {}, "outputs": [], "source": [ @@ -231,13 +267,20 @@ " suite.metric(f\"predict_q={q}\", \"pylibkriging\", seconds=best)\n", "expo = np.log2(times[8000] / times[4000])\n", "suite.less(\"predict(stdev) time growth exponent in q (8000 vs 4000)\", expo, 1.5,\n", - " detail=\", \".join(f\"q={q}: {t:.3f}s\" for q, t in times.items()))" + " detail=\", \".join(f\"q={q}: {t:.3f}s\" for q, t in times.items()))\n", + "qs = np.array(list(times)); ts = np.array(list(times.values()))\n", + "plt.figure(figsize=(5, 3.6))\n", + "plt.loglog(qs, ts, \"o-\", label=\"predict(return_stdev=True)\")\n", + "plt.loglog(qs, ts[0] * qs / qs[0], \"k--\", lw=.8, label=\"O(q) reference\")\n", + "plt.loglog(qs, ts[0] * (qs / qs[0]) ** 2, \"k:\", lw=.8, label=\"O(q²) reference\")\n", + "plt.xlabel(\"q (prediction points)\"); plt.ylabel(\"time (s)\"); plt.legend(); plt.grid(True, which=\"both\", alpha=.3)\n", + "plt.show()" ] }, { "cell_type": "code", "execution_count": null, - "id": "c012", + "id": "c013", "metadata": {}, "outputs": [], "source": [ diff --git a/save-load-pickle-copy-accessors_Kriging_python.ipynb b/save-load-pickle-copy-accessors_Kriging_python.ipynb index 3a6f3ae..349830f 100644 --- a/save-load-pickle-copy-accessors_Kriging_python.ipynb +++ b/save-load-pickle-copy-accessors_Kriging_python.ipynb @@ -28,6 +28,41 @@ "id": "c001", "metadata": {}, "outputs": [], + "source": [ + "# Setup for running this notebook outside the repository (Colab, Binder, a lone copy):\n", + "# nothing to do in the repository itself (CI, local runs). Otherwise the repository\n", + "# (datasets of data/, helpers of tools/) is cloned and the released pylibkriging installed.\n", + "import importlib\n", + "import os\n", + "import subprocess\n", + "import sys\n", + "from pathlib import Path\n", + "\n", + "if not Path(\"tools/known_issues.csv\").is_file():\n", + " if not Path(\"benchmarks/tools/known_issues.csv\").is_file():\n", + " subprocess.run([\"git\", \"clone\", \"--depth\", \"1\", \"https://github.com/libKriging/benchmarks\"], check=True)\n", + " os.chdir(\"benchmarks\")\n", + " try:\n", + " import pylibkriging # noqa: F401\n", + " except ImportError:\n", + " r = subprocess.run([sys.executable, \"-m\", \"pip\", \"install\", \"--quiet\", \"pylibkriging\"],\n", + " capture_output=True, text=True)\n", + " if r.returncode != 0:\n", + " raise RuntimeError(\n", + " f\"pylibkriging could not be installed for Python {sys.version_info.major}.{sys.version_info.minor} \"\n", + " \"(PyPI has wheels for CPython 3.10 to 3.12 only). On Colab: Runtime > Change runtime version > \"\n", + " \"2026.07 (Python 3.12), then run again.\\n\" + r.stderr[-500:])\n", + " importlib.invalidate_caches()\n", + "sys.path.insert(0, str(Path(\"tools\").resolve())) # lkbench, when not installed (Colab, Binder)\n", + "print(\"working directory:\", os.getcwd())" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c002", + "metadata": {}, + "outputs": [], "source": [ "import copy\n", "\n", @@ -71,7 +106,7 @@ }, { "cell_type": "markdown", - "id": "c002", + "id": "c003", "metadata": {}, "source": [ "## save / load, pickle, copy" @@ -80,7 +115,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c003", + "id": "c004", "metadata": {}, "outputs": [], "source": [ @@ -91,7 +126,7 @@ }, { "cell_type": "markdown", - "id": "c004", + "id": "c005", "metadata": {}, "source": [ "A copy is independent: updating it leaves the original model unchanged." @@ -100,7 +135,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c005", + "id": "c006", "metadata": {}, "outputs": [], "source": [ @@ -114,7 +149,7 @@ }, { "cell_type": "markdown", - "id": "c006", + "id": "c007", "metadata": {}, "source": [ "## Accessors mirror the constructor" @@ -123,7 +158,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c007", + "id": "c008", "metadata": {}, "outputs": [], "source": [ @@ -148,7 +183,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c008", + "id": "c009", "metadata": {}, "outputs": [], "source": [ diff --git a/sklearn-estimators_Kriging_python.ipynb b/sklearn-estimators_Kriging_python.ipynb index 496b4fa..a065132 100644 --- a/sklearn-estimators_Kriging_python.ipynb +++ b/sklearn-estimators_Kriging_python.ipynb @@ -19,6 +19,41 @@ "id": "c001", "metadata": {}, "outputs": [], + "source": [ + "# Setup for running this notebook outside the repository (Colab, Binder, a lone copy):\n", + "# nothing to do in the repository itself (CI, local runs). Otherwise the repository\n", + "# (datasets of data/, helpers of tools/) is cloned and the released pylibkriging installed.\n", + "import importlib\n", + "import os\n", + "import subprocess\n", + "import sys\n", + "from pathlib import Path\n", + "\n", + "if not Path(\"tools/known_issues.csv\").is_file():\n", + " if not Path(\"benchmarks/tools/known_issues.csv\").is_file():\n", + " subprocess.run([\"git\", \"clone\", \"--depth\", \"1\", \"https://github.com/libKriging/benchmarks\"], check=True)\n", + " os.chdir(\"benchmarks\")\n", + " try:\n", + " import pylibkriging # noqa: F401\n", + " except ImportError:\n", + " r = subprocess.run([sys.executable, \"-m\", \"pip\", \"install\", \"--quiet\", \"pylibkriging\"],\n", + " capture_output=True, text=True)\n", + " if r.returncode != 0:\n", + " raise RuntimeError(\n", + " f\"pylibkriging could not be installed for Python {sys.version_info.major}.{sys.version_info.minor} \"\n", + " \"(PyPI has wheels for CPython 3.10 to 3.12 only). On Colab: Runtime > Change runtime version > \"\n", + " \"2026.07 (Python 3.12), then run again.\\n\" + r.stderr[-500:])\n", + " importlib.invalidate_caches()\n", + "sys.path.insert(0, str(Path(\"tools\").resolve())) # lkbench, when not installed (Colab, Binder)\n", + "print(\"working directory:\", os.getcwd())" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c002", + "metadata": {}, + "outputs": [], "source": [ "import numpy as np\n", "import pylibkriging as lk\n", @@ -40,7 +75,7 @@ }, { "cell_type": "markdown", - "id": "c002", + "id": "c003", "metadata": {}, "source": [ "## Installing scikit-learn\n", @@ -54,7 +89,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c003", + "id": "c004", "metadata": {}, "outputs": [], "source": [ @@ -73,7 +108,7 @@ }, { "cell_type": "markdown", - "id": "c004", + "id": "c005", "metadata": {}, "source": [ "## scikit-learn estimators" @@ -82,7 +117,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c005", + "id": "c006", "metadata": {}, "outputs": [], "source": [ @@ -125,7 +160,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c006", + "id": "c007", "metadata": {}, "outputs": [], "source": [ diff --git a/test-functions-reference-values_no-model_python.ipynb b/test-functions-reference-values_no-model_python.ipynb index 9a2a0a8..0b48e9a 100644 --- a/test-functions-reference-values_no-model_python.ipynb +++ b/test-functions-reference-values_no-model_python.ipynb @@ -20,6 +20,41 @@ "id": "c001", "metadata": {}, "outputs": [], + "source": [ + "# Setup for running this notebook outside the repository (Colab, Binder, a lone copy):\n", + "# nothing to do in the repository itself (CI, local runs). Otherwise the repository\n", + "# (datasets of data/, helpers of tools/) is cloned and the released pylibkriging installed.\n", + "import importlib\n", + "import os\n", + "import subprocess\n", + "import sys\n", + "from pathlib import Path\n", + "\n", + "if not Path(\"tools/known_issues.csv\").is_file():\n", + " if not Path(\"benchmarks/tools/known_issues.csv\").is_file():\n", + " subprocess.run([\"git\", \"clone\", \"--depth\", \"1\", \"https://github.com/libKriging/benchmarks\"], check=True)\n", + " os.chdir(\"benchmarks\")\n", + " try:\n", + " import pylibkriging # noqa: F401\n", + " except ImportError:\n", + " r = subprocess.run([sys.executable, \"-m\", \"pip\", \"install\", \"--quiet\", \"pylibkriging\"],\n", + " capture_output=True, text=True)\n", + " if r.returncode != 0:\n", + " raise RuntimeError(\n", + " f\"pylibkriging could not be installed for Python {sys.version_info.major}.{sys.version_info.minor} \"\n", + " \"(PyPI has wheels for CPython 3.10 to 3.12 only). On Colab: Runtime > Change runtime version > \"\n", + " \"2026.07 (Python 3.12), then run again.\\n\" + r.stderr[-500:])\n", + " importlib.invalidate_caches()\n", + "sys.path.insert(0, str(Path(\"tools\").resolve())) # lkbench, when not installed (Colab, Binder)\n", + "print(\"working directory:\", os.getcwd())" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c002", + "metadata": {}, + "outputs": [], "source": [ "import numpy as np\n", "import pandas as pd\n", @@ -33,7 +68,7 @@ }, { "cell_type": "markdown", - "id": "c002", + "id": "c003", "metadata": {}, "source": [ "## Reference values" @@ -42,7 +77,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c003", + "id": "c004", "metadata": {}, "outputs": [], "source": [ @@ -57,7 +92,7 @@ }, { "cell_type": "markdown", - "id": "c004", + "id": "c005", "metadata": {}, "source": [ "## Output ranges\n", @@ -70,7 +105,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c005", + "id": "c006", "metadata": {}, "outputs": [], "source": [ @@ -97,7 +132,7 @@ }, { "cell_type": "markdown", - "id": "c006", + "id": "c007", "metadata": {}, "source": [ "## Designs\n", @@ -110,7 +145,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c007", + "id": "c008", "metadata": {}, "outputs": [], "source": [ @@ -123,7 +158,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c008", + "id": "c009", "metadata": {}, "outputs": [], "source": [ @@ -145,7 +180,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c009", + "id": "c010", "metadata": {}, "outputs": [], "source": [ diff --git a/tools/known_issues.csv b/tools/known_issues.csv index 8281855..6829036 100644 --- a/tools/known_issues.csv +++ b/tools/known_issues.csv @@ -9,9 +9,3 @@ robustness-invalid-inputs_Kriging_python,Inf in y is rejected,Kriging accepts In robustness-invalid-inputs_Kriging_python,WarpKriging with 1 spec for 2 columns is rejected,WarpKriging accepts fewer warping specs than columns of X robustness-invalid-inputs_Kriging_python,predict(stdev) time growth exponent in q (8000 vs 4000),"predict(return_stdev=True, return_cov=False) is O(q^2) in time and memory (10 000 points: 1.6 GB; 30 000: out of memory)" vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,sklearn RBF == libKriging gauss: simple kriging stdev (beta fixed),With beta fixed (is_beta_estim=False) predict still adds the trend-estimation variance (universal kriging) instead of the simple kriging variance -deep-kernel_MLPKriging_R,MLPKriging: Q2(default optim) - Q2(BFGS10),https://github.com/libKriging/libKriging/issues/382 single-start MLPKriging fit stops after a few ms at a poor optimum -noisy-data_Kriging-nugget-noise_R,Kriging(nugget): LL(single start) - LL(10 restarts),https://github.com/libKriging/libKriging/issues/383 default single-start nugget fit lands far from the optimum -deep-kernel_MLPKriging_octave,MLPKriging: Q2(default optim) - Q2(BFGS10),https://github.com/libKriging/libKriging/issues/382 single-start MLPKriging fit stops after a few ms at a poor optimum -noisy-data_Kriging-nugget-noise_octave,Kriging(nugget): LL(single start) - LL(10 restarts),https://github.com/libKriging/libKriging/issues/383 default single-start nugget fit lands far from the optimum -deep-kernel_MLPKriging_julia,MLPKriging: Q2(default optim) - Q2(BFGS10),https://github.com/libKriging/libKriging/issues/382 single-start MLPKriging fit stops after a few ms at a poor optimum -noisy-data_Kriging-nugget-noise_julia,Kriging(nugget): LL(single start) - LL(10 restarts),https://github.com/libKriging/libKriging/issues/383 default single-start nugget fit lands far from the optimum diff --git a/tools/notebook_setup.py b/tools/notebook_setup.py new file mode 100644 index 0000000..43f585e --- /dev/null +++ b/tools/notebook_setup.py @@ -0,0 +1,26 @@ +# Setup for running this notebook outside the repository (Colab, Binder, a lone copy): +# nothing to do in the repository itself (CI, local runs). Otherwise the repository +# (datasets of data/, helpers of tools/) is cloned and the released pylibkriging installed. +import importlib +import os +import subprocess +import sys +from pathlib import Path + +if not Path("tools/known_issues.csv").is_file(): + if not Path("benchmarks/tools/known_issues.csv").is_file(): + subprocess.run(["git", "clone", "--depth", "1", "https://github.com/libKriging/benchmarks"], check=True) + os.chdir("benchmarks") + try: + import pylibkriging # noqa: F401 + except ImportError: + r = subprocess.run([sys.executable, "-m", "pip", "install", "--quiet", "pylibkriging"], + capture_output=True, text=True) + if r.returncode != 0: + raise RuntimeError( + f"pylibkriging could not be installed for Python {sys.version_info.major}.{sys.version_info.minor} " + "(PyPI has wheels for CPython 3.10 to 3.12 only). On Colab: Runtime > Change runtime version > " + "2026.07 (Python 3.12), then run again.\n" + r.stderr[-500:]) + importlib.invalidate_caches() +sys.path.insert(0, str(Path("tools").resolve())) # lkbench, when not installed (Colab, Binder) +print("working directory:", os.getcwd()) diff --git a/tools/py2nb.py b/tools/py2nb.py index ecf4d7f..6c27499 100644 --- a/tools/py2nb.py +++ b/tools/py2nb.py @@ -3,7 +3,8 @@ ``% %%`` in Octave) into a notebook without outputs. The notebooks of this repository are written this way (sources in ``tools/src/``) so that they review as plain diffs; the ``.ipynb`` files, at the repository root, are generated, and CI checks that they -are in sync (and that no notebook lacks its source). +are in sync (and that no notebook lacks its source). Every Python notebook gets the +cell of ``tools/notebook_setup.py`` after its title, to run on Colab or Binder too. Usage: python tools/py2nb.py [--check] [SRC...] (default: every tools/src/* file) each ``tools/src/.`` gives ``.ipynb`` at the repository root. @@ -17,6 +18,9 @@ ROOT = Path(__file__).resolve().parents[1] SRC = ROOT / "tools" / "src" +# code cell inserted after the title cell of every Python notebook, so that it also runs +# outside the repository (Colab, Binder): a no-op in the repository itself +SETUP = {".py": ROOT / "tools" / "notebook_setup.py"} KERNELS = { ".py": ({"name": "python3", "display_name": "Python 3", "language": "python"}, @@ -51,6 +55,8 @@ def flush(): elif kind is not None: buf.append(line) flush() + if src.suffix in SETUP and cells and cells[0].cell_type == "markdown": + cells.insert(1, nbformat.v4.new_code_cell(SETUP[src.suffix].read_text().strip("\n"))) nb = nbformat.v4.new_notebook(cells=cells) nb.metadata["kernelspec"] = kernelspec nb.metadata["language_info"] = lang_info diff --git a/tools/report.py b/tools/report.py index 72cbaf1..95cc9f6 100644 --- a/tools/report.py +++ b/tools/report.py @@ -15,7 +15,7 @@ 3. known issues whose check now passes ("FIXED?": remove them from tools/known_issues.csv); 4. cross-binding consistency: each language against Python, per quantity, with - the tolerances of cross-binding-reference_Kriging_python.ipynb; + the tolerances of cross-binding-reference_all-classes_python.ipynb; 5. drifts of the accuracy metrics (q2, rmse, nlpd) and of the fit times against a baseline (tools/baseline/metrics-.csv), when one is given. @@ -47,6 +47,13 @@ "llfun": (1e-8, 0, "error"), "llfun_grad": (1e-6, 1e-8, "error"), "fit_theta": (1e-4, 1e-6, "warn"), "fit_ll": (1e-4, 0, "warn"), "fit_mean": (1e-4, 1e-6, "warn"), "sim": (1e-8, 1e-10, "warn"), + "nugget_fixed_mean": (1e-8, 1e-10, "error"), "nugget_fixed_stdev": (1e-6, 1e-8, "error"), + "noise_fixed_mean": (1e-8, 1e-10, "error"), "noise_fixed_stdev": (1e-6, 1e-8, "error"), + "nugget_fit_ll": (1e-4, 0, "warn"), "nugget_fit_mean": (1e-4, 1e-6, "warn"), + "vecchia_fixed_ll": (1e-8, 0, "error"), "vecchia_fixed_mean": (1e-8, 1e-10, "error"), + "nystrom_fixed_ll": (1e-8, 0, "error"), "nystrom_fixed_mean": (1e-8, 1e-10, "error"), + "warp_fit_mean": (1e-4, 1e-6, "warn"), "mlp_fit_mean": (1e-4, 1e-6, "warn"), + "nested_fit_mean": (1e-4, 1e-6, "warn"), } TRACKED = {"q2": "higher", "rmse": "lower", "nlpd": "lower", "fit_s": "time"} @@ -88,7 +95,8 @@ def xbind_report(results): try: r = ref.loc[grp.index] except KeyError: - rows.append((lang, dataset, quantity, "missing in Python", "", sev, "FAIL" if sev == "error" else "WARN")) + # not a disagreement: a feature absent from one binding's version (release channel) + rows.append((lang, dataset, quantity, "missing in Python", "", "warn", "WARN")) continue a, b = grp.to_numpy(float), r.to_numpy(float) scale = max(np.max(np.abs(b)), 1e-300) diff --git a/tools/run_notebooks.py b/tools/run_notebooks.py index 707b830..dac0f48 100644 --- a/tools/run_notebooks.py +++ b/tools/run_notebooks.py @@ -14,7 +14,7 @@ Usage: python3 tools/run_notebooks.py *_python.ipynb - python3 tools/run_notebooks.py --inplace --timeout 3600 --kernel julia-1.11 cross-binding_Kriging_julia.ipynb + python3 tools/run_notebooks.py --inplace --timeout 3600 --kernel julia-1.11 cross-binding_all-classes_julia.ipynb python3 tools/run_notebooks.py --skip-cell '^repo_root = ' --prelude 'addpath(...)' nb.ipynb Requires nbclient, nbformat and ipykernel. diff --git a/tools/src/aggregations-large-n_NestedKriging_python.py b/tools/src/aggregations-large-n_NestedKriging_python.py index c114924..5466a74 100644 --- a/tools/src/aggregations-large-n_NestedKriging_python.py +++ b/tools/src/aggregations-large-n_NestedKriging_python.py @@ -12,6 +12,9 @@ # every aggregation on Branin. # # Checks: +# * **limit and asymptotic behaviour** at fixed θ (Hartmann 3, n = 300): with one +# group, `NK` *is* the exact kriging predictor; the gap to the exact predictor +# as the number of groups grows, against the fit and prediction cost; # * accuracy close to the exact fit (warn-level gap, error-level floor), and # faster than the exact fit at n = 1500 (warn-level); # * `NK` interpolates the data (it is a proper kriging predictor); the PoE family @@ -21,6 +24,7 @@ # %% import time +import matplotlib.pyplot as plt import numpy as np import pandas as pd import pylibkriging as lk @@ -44,6 +48,48 @@ def timed(f): out = f() return out, time.time() - t0 +# %% [markdown] +# ## From one group (the exact predictor) to many, at fixed θ +# +# Hartmann 3, n = 300, θ of the exact fit kept fixed in every sub-model: the gap +# is that of the aggregation alone. RMS gap of the predicted mean to the exact +# predictor on 500 test points, relative to its standard deviation, and fit time. + +# %% +if compat.has("NestedKriging"): + tf = TESTS["hartmann3"] + X, y, Xt, yt = design.dataset(tf, 300, n_test=500, seed=54) + theta = np.asarray(lk.Kriging(y, X, "matern5_2").theta()).reshape(1, -1) + fixed = {"theta": theta, "is_theta_estim": False} + m_e = pred(lk.Kriging(y, X, "matern5_2", optim="none", parameters=fixed), Xt)[0] + rows = [] + for agg in ["NK", "rBCM"]: + for g in [1, 2, 4, 8, 16, 32]: + with suite.runs(f"nested {agg}, {g} groups, fixed theta: fit + predict"): + k, t = timed(lambda: lk.NestedKriging(y, X, "matern5_2", nb_groups=g, aggregation=agg, seed=1, + optim="none", parameters=fixed)) + m = pred(k, Xt)[0] + rows.append(dict(aggregation=agg, groups=g, time_s=t, + mean_gap=np.sqrt(np.mean((m - m_e) ** 2)) / np.std(m_e), + q2=metrics.q2(yt, m))) + conv = pd.DataFrame(rows) + nk = conv[conv["aggregation"] == "NK"] + suite.less("nested NK, 1 group == exact predictor (RMS gap / sd)", nk["mean_gap"].iloc[0], 1e-6) + suite.less("nested NK, 32 groups: RMS gap to the exact predictor / sd", nk["mean_gap"].iloc[-1], 0.1, + severity="warn") + rb = conv[conv["aggregation"] == "rBCM"] + suite.true("nested NK closer to the exact predictor than rBCM (2 to 32 groups)", + bool(np.all(nk["mean_gap"].to_numpy()[1:] <= rb["mean_gap"].to_numpy()[1:])), severity="warn") + fig, axes = plt.subplots(1, 2, figsize=(10, 3.8)) + for agg, g in conv.groupby("aggregation"): + axes[0].semilogy(g["groups"], np.maximum(g["mean_gap"], 1e-12), "o-", label=agg) + axes[1].plot(g["groups"], g["time_s"], "o-", label=agg) + axes[0].set_ylabel("RMS(mean − mean(exact)) / sd"); axes[1].set_ylabel("fit time (s)") + for ax in axes: + ax.set_xscale("log", base=2); ax.set_xlabel("number of groups"); ax.grid(True, which="both", alpha=.3) + axes[0].legend(); fig.suptitle("Hartmann 3, n = 300, θ fixed: NestedKriging vs the exact predictor"); plt.show() + display(conv.round(5)) + # %% [markdown] # ## Accuracy and cost against the exact fit diff --git a/tools/src/aggregations_NestedKriging_R.R b/tools/src/aggregations_NestedKriging_R.R deleted file mode 100644 index 1beb97d..0000000 --- a/tools/src/aggregations_NestedKriging_R.R +++ /dev/null @@ -1,70 +0,0 @@ -# %% [markdown] -# # R · `NestedKriging` — `NK` and `PoE` aggregations, 4 groups of 100 points -# -# Branin on [0,1]² (n = 30 LHS points unless stated, Q² on 500 uniform test -# points), through rlibkriging. Checks (error-level unless stated): -# -# * stdev ≥ 0, Q² > 0.95 (warn-level) -# * `NK` interpolates the data (it is a proper kriging predictor) -# * `save` then `load`: the same predicted mean -# -# A class missing from the installed rlibkriging (release channel) is recorded as SKIP. - -# %% -suppressPackageStartupMessages(library(rlibkriging)) -source("tools/r/checks.R") -s <- new_suite("aggregations_NestedKriging_R") -cat("rlibkriging", as.character(packageVersion("rlibkriging")), "\n") - -branin <- function(U) { # on [0,1]^2 - x1 <- 15 * U[, 1] - 5; x2 <- 15 * U[, 2] - (x2 - 5.1 / (4 * pi^2) * x1^2 + 5 / pi * x1 - 6)^2 + 10 * (1 - 1 / (8 * pi)) * cos(x1) + 10 -} -set.seed(1) -lhs <- function(n, d) sapply(1:d, function(j) (sample(n) - runif(n)) / n) -X <- lhs(30, 2); y <- branin(X) -Xt <- matrix(runif(1000), ncol = 2); yt <- branin(Xt) -tmp <- tempdir() -has_class <- function(name) exists(name, mode = "function") -skip <- function(s, check, reason) { - s$checks[[length(s$checks) + 1]] <- data.frame(suite = s$name, lang = "R", check = check, severity = "info", - status = "SKIP", value = "", expected = "", detail = reason, stringsAsFactors = FALSE) - message("[SKIP] ", check, ": ", reason) -} -roundtrip_same_mean <- function(s, label, k, Xn) { - f <- file.path(tmp, paste0(gsub("[^A-Za-z0-9]", "_", label), ".json")) - check_runs(s, paste0(label, ": save + load"), { - save(k, f) - k2 <- rlibkriging::load(f) - check_close(s, paste0(label, ": load(save(k)) predicts the same mean"), - predict(k2, Xn, return_stdev = FALSE)$mean, predict(k, Xn, return_stdev = FALSE)$mean, rtol = 1e-10) - }) -} - -# %% [markdown] -# ## `NestedKriging`: 4 groups of 100 points, `NK` and `PoE` aggregations - -# %% -Xl <- lhs(400, 2); yl <- branin(Xl) -if (has_class("NestedKriging")) { - for (agg in c("NK", "PoE")) { - check_runs(s, paste0("NestedKriging(", agg, "): fit + predict"), { - kk <- NestedKriging(yl, Xl, kernel = "matern5_2", nb_groups = 4, aggregation = agg, seed = 3) - pk <- predict(kk, Xt, return_stdev = TRUE) - check_less(s, paste0("NestedKriging(", agg, "): stdev < 0"), sum(pk$stdev < 0), 1) - add_metric(s, "branin n=400", paste("NestedKriging", agg), q2 = q2(yt, pk$mean)) - check_greater(s, paste0("NestedKriging(", agg, "): Q2"), q2(yt, pk$mean), 0.95, severity = "warn") - if (agg == "NK") - check_less(s, "NestedKriging(NK): interpolates (max |m(X) - y| / sd(y))", - max(abs(predict(kk, Xl, return_stdev = FALSE)$mean - yl)) / sd(yl), 1e-3) - roundtrip_same_mean(s, paste0("NestedKriging(", agg, ")"), kk, Xt) - }) - } -} else skip(s, "NestedKriging", "not in this rlibkriging") - -# %% -me <- do.call(rbind, s$metrics) -print(me[me$metric == "q2", c("case", "package", "value")], row.names = FALSE) - -# %% -finalize(s) diff --git a/tools/src/aggregations_NestedKriging_julia.jl b/tools/src/aggregations_NestedKriging_julia.jl deleted file mode 100644 index cd4cfe5..0000000 --- a/tools/src/aggregations_NestedKriging_julia.jl +++ /dev/null @@ -1,73 +0,0 @@ -# %% [markdown] -# # Julia · `NestedKriging` — `NK` and `PoE` aggregations, 4 groups of 100 points -# -# Branin on [0,1]² (n = 30 LHS points unless stated, Q² on 500 uniform test -# points), through JLibKriging. Checks (error-level unless stated): -# -# * stdev ≥ 0, Q² > 0.95 (warn-level) -# * `NK` interpolates the data (it is a proper kriging predictor) -# * `save` then `load`: the same predicted mean -# -# The same checks as `aggregations_NestedKriging_R.ipynb`, with the Julia binding. - -# %% -import JLibKriging as LK -using Random -include("tools/julia/checks.jl") -s = Suite("aggregations_NestedKriging_julia") - -branin(U) = begin - x1 = 15 .* U[:, 1] .- 5; x2 = 15 .* U[:, 2] - (x2 .- 5.1 / (4π^2) .* x1 .^ 2 .+ 5 / π .* x1 .- 6) .^ 2 .+ 10 * (1 - 1 / (8π)) .* cos.(x1) .+ 10 -end -rng = MersenneTwister(1) -lhs(n, d) = hcat([(randperm(rng, n) .- rand(rng, n)) ./ n for _ in 1:d]...) -X = lhs(30, 2); y = branin(X) -Xt = rand(rng, 500, 2); yt = branin(Xt) -sd(v) = sqrt(sum((v .- sum(v) / length(v)) .^ 2) / (length(v) - 1)) -finite_count(v...) = count(!isfinite, vcat(v...)) - -function check_runs!(f, s, check) - try - f() - catch e - add_check!(s, check, false; value=string(typeof(e)), expected="no error", detail=first(sprint(showerror, e), 200)) - end -end - -function roundtrip!(s, label, k, Xn) - check_runs!(s, "$label: save + load") do - f = joinpath(mktempdir(), "k.json") - LK.save(k, f) - k2 = LK.load(f) - check_close!(s, "$label: load(save(k)) predicts the same mean", - LK.predict(k2, Xn; return_stdev=false).mean, LK.predict(k, Xn; return_stdev=false).mean; rtol=1e-10) - end -end - -# %% [markdown] -# ## `NestedKriging`: 4 groups of 100 points, `NK` and `PoE` aggregations - -# %% -Xl = lhs(400, 2); yl = branin(Xl) -for agg in ("NK", "PoE") - label = "NestedKriging($agg)" - check_runs!(s, "$label: fit + predict") do - kk = LK.NestedKriging(yl, Xl, "matern5_2", 4; aggregation=agg, seed=3) - pk = LK.predict(kk, Xt; return_stdev=true) - add_check!(s, "$label: stdev >= 0", all(pk.stdev .>= 0); value=minimum(pk.stdev), expected=">= 0") - add_metric!(s, "branin n=400", "NestedKriging $agg"; q2=q2(yt, pk.mean)) - add_check!(s, "$label: Q2", q2(yt, pk.mean) > 0.95; value=q2(yt, pk.mean), expected="> 0.95", severity="warn") - if agg == "NK" - e = maximum(abs.(LK.predict(kk, Xl; return_stdev=false).mean .- yl)) / sd(yl) - add_check!(s, "NestedKriging(NK): interpolates (max |m(X) - y| / sd(y))", e < 1e-3; value=e, expected="< 1e-3") - end - roundtrip!(s, label, kk, Xt) - end -end - -# %% -foreach(r -> r[5] == "q2" && println(rpad(r[3], 24), rpad(r[4], 28), r[6]), s.metrics) - -# %% -finalize!(s) diff --git a/tools/src/aggregations_NestedKriging_octave.m b/tools/src/aggregations_NestedKriging_octave.m deleted file mode 100644 index 3553028..0000000 --- a/tools/src/aggregations_NestedKriging_octave.m +++ /dev/null @@ -1,83 +0,0 @@ -% %% [markdown] -% # Octave · `NestedKriging` — `NK` and `PoE` aggregations, 4 groups of 100 points -% -% Branin on [0,1]² (n = 30 LHS points unless stated, Q² on 500 uniform test -% points), through mLibKriging. Checks (error-level unless stated): -% -% * stdev ≥ 0, Q² > 0.95 (warn-level) -% * `NK` interpolates the data (it is a proper kriging predictor) -% * `save` then `load`: the same predicted mean -% -% The same checks as `aggregations_NestedKriging_R.ipynb`, with the Octave binding. -% -% A class missing from the installed mLibKriging (release channel) is recorded as SKIP. - -% %% -addpath(fullfile(pwd, "tools", "octave")); -s = lkb_suite("aggregations_NestedKriging_octave"); -q2 = @(y, m) 1 - sum((y - m).^2) / sum((y - mean(y)).^2); -branin = @(U) ((15 * U(:, 2)) - 5.1 / (4 * pi^2) * (15 * U(:, 1) - 5).^2 + 5 / pi * (15 * U(:, 1) - 5) - 6).^2 ... - + 10 * (1 - 1 / (8 * pi)) * cos(15 * U(:, 1) - 5) + 10; -rand("seed", 1); randn("seed", 1); -function U = lhs(n, d) - U = zeros(n, d); - for j = 1:d, U(:, j) = (randperm(n)' - rand(n, 1)) / n; end -end -X = lhs(30, 2); y = branin(X); -Xt = rand(500, 2); yt = branin(Xt); -tmp = tempname(); mkdir(tmp); -has_class = @(name) exist(name) == 2; % a class file on the path - -function s = roundtrip(s, label, k, Xn, loader) - f = fullfile(tempdir(), [regexprep(label, "[^A-Za-z0-9]", "_") ".json"]); - try - k.save(f); - k2 = loader(f); - m1 = k.predict(Xn, false, false, false); - m2 = k2.predict(Xn, false, false, false); - s = lkb_close(s, [label ": load(save(k)) predicts the same mean"], m2, m1, 1e-10); - catch err - s = lkb_check(s, [label ": save + load"], false, "error", "no error", err.message); - end -end - -function s = skip(s, check, reason) - s.checks(end + 1, :) = {s.name, "octave", check, "info", "SKIP", "", "", reason}; - printf("[SKIP] %s: %s\n", check, reason); -end - -% %% [markdown] -% ## `NestedKriging`: 4 groups of 100 points, `NK` and `PoE` aggregations - -% %% -Xl = lhs(400, 2); yl = branin(Xl); -if has_class("NestedKriging") - for agg = {"NK", "PoE"} - agg = agg{1}; label = ["NestedKriging(" agg ")"]; - try - kk = NestedKriging(yl, Xl, "matern5_2", 4, agg, "kmeans", 3); - [mk, sk] = kk.predict(Xt); % stdev returned when asked for (2 outputs) - s = lkb_check(s, [label ": stdev >= 0"], all(sk >= 0), min(sk), ">= 0"); - s = lkb_metric(s, "branin n=400", ["NestedKriging " agg], "q2", q2(yt, mk)); - s = lkb_check(s, [label ": Q2"], q2(yt, mk) > 0.95, q2(yt, mk), "> 0.95", "", "warn"); - if strcmp(agg, "NK") - e = max(abs(kk.predict(Xl) - yl)) / std(yl); - s = lkb_check(s, "NestedKriging(NK): interpolates (max |m(X) - y| / sd(y))", e < 1e-3, e, "< 1e-3"); - end - f = fullfile(tmp, ["nested_" agg ".json"]); - kk.save(f); k2 = load_kriging(f); - s = lkb_close(s, [label ": load(save(k)) predicts the same mean"], k2.predict(Xt), kk.predict(Xt), 1e-10); - catch err - s = lkb_check(s, [label ": fit + predict"], false, "error", "no error", err.message); - end - end -else - s = skip(s, "NestedKriging", "not in this mLibKriging"); -end - -% %% -q = strcmp(s.metrics(:, 5), "q2"); -disp(s.metrics(q, [3, 4, 6])) - -% %% -lkb_finalize(s); diff --git a/tools/src/cross-binding-reference_Kriging_python.py b/tools/src/cross-binding-reference_Kriging_python.py deleted file mode 100644 index b8098c0..0000000 --- a/tools/src/cross-binding-reference_Kriging_python.py +++ /dev/null @@ -1,78 +0,0 @@ -# %% [markdown] -# # Python · Kriging — cross-binding consistency: the Python reference -# -# All bindings (Python, R, Octave, Julia) wrap the same C++ library: on the -# same data with the same options they must return the same numbers. Each -# language runs the protocol below on the shared datasets of `data/` and writes -# `results/xbind_.csv` (`lang,dataset,quantity,index,value`); -# `tools/report.py` then compares every language to Python. -# -# Protocol, for each dataset (Branin n=25, Hartmann 3 n=40, Borehole n=80, on -# the unit cube), kernel `matern5_2`, trend `constant`, θ₀ = 0.3 in every -# dimension: -# -# | quantity | how | tolerance in the report | -# |---|---|---| -# | `fixed_sigma2`, `fixed_beta`, `fixed_ll`, `fixed_loo` | `optim="none"`, θ = θ₀ fixed | rel. 1e-8 (error) | -# | `fixed_mean`, `fixed_stdev` | predict at the first 50 test points | rel. 1e-8 (error) | -# | `llfun`, `llfun_grad` | `logLikelihoodFun(θ₀)` with gradient | rel. 1e-8 (error) | -# | `fit_theta`, `fit_ll`, `fit_mean` | default fit (`BFGS`) | rel. 1e-4 (warn: optimizer paths may differ) | -# | `sim` | `simulate(nsim=3, seed=123)` at the first 5 test points (5 × 3, column-major) | rel. 1e-8 (warn: RNG may be binding-specific) | - -# %% -import csv - -import numpy as np -import pylibkriging as lk - -import lkbench -from lkbench import compat -from lkbench.checks import DATA_DIR as DATA, RESULTS_DIR - -suite = lkbench.Suite("cross-binding-reference_Kriging_python") -print("pylibkriging", compat.version()) -DATASETS = ["branin", "hartmann3", "borehole"] -rows = [] - - -def put(dataset, quantity, values): - for i, v in enumerate(np.ravel(values), start=1): - rows.append(("python", dataset, quantity, i, f"{float(v):.17g}")) - - -def load(name, stem): - return np.loadtxt(DATA / f"{name}_{stem}.csv", delimiter=",", ndmin=2) - -# %% -for name in DATASETS: - X, y, Xt = load(name, "X"), load(name, "y").ravel(), load(name, "Xt")[:50] - d = X.shape[1] - theta0 = np.full((1, d), 0.3) - kf = lk.Kriging(y, X, "matern5_2", regmodel="constant", optim="none", - parameters={"theta": theta0, "is_theta_estim": False}) - put(name, "fixed_sigma2", kf.sigma2()) - put(name, "fixed_beta", kf.beta()) - put(name, "fixed_ll", kf.logLikelihood()) - put(name, "fixed_loo", kf.leaveOneOut()) - m, s, *_ = kf.predict(Xt, return_stdev=True) - put(name, "fixed_mean", m) - put(name, "fixed_stdev", s) - ll, grad, _ = kf.logLikelihoodFun(theta0.ravel(), True) - put(name, "llfun", ll) - put(name, "llfun_grad", grad) - suite.close(f"{name}: logLikelihood() == logLikelihoodFun(theta0)", kf.logLikelihood(), ll, rtol=1e-10) - k = lk.Kriging(y, X, "matern5_2") - put(name, "fit_theta", k.theta()) - put(name, "fit_ll", k.logLikelihood()) - put(name, "fit_mean", k.predict(Xt, return_stdev=False)[0]) - put(name, "sim", np.ravel(np.asarray(kf.simulate(3, 123, Xt[:5])), order="F")) # column-major, as R/Octave/Julia - -RESULTS_DIR.mkdir(exist_ok=True) -with open(RESULTS_DIR / "xbind_python.csv", "w", newline="") as fh: - w = csv.writer(fh) - w.writerow(["lang", "dataset", "quantity", "index", "value"]) - w.writerows(rows) -print(len(rows), "values written") - -# %% -suite.finalize() diff --git a/tools/src/cross-binding-reference_all-classes_python.py b/tools/src/cross-binding-reference_all-classes_python.py new file mode 100644 index 0000000..66de287 --- /dev/null +++ b/tools/src/cross-binding-reference_all-classes_python.py @@ -0,0 +1,126 @@ +# %% [markdown] +# # Python · all classes — cross-binding consistency: the Python reference +# +# All bindings (Python, R, Octave, Julia) wrap the same C++ library: on the +# same data with the same options they must return the same numbers. Each +# language runs the protocol below on the shared datasets of `data/` and writes +# `results/xbind_.csv` (`lang,dataset,quantity,index,value`); +# `tools/report.py` then compares every language to Python. +# +# Protocol, for each dataset (Branin n=25, Hartmann 3 n=40, Borehole n=80, on +# the unit cube), kernel `matern5_2`, trend `constant`, θ₀ = 0.3 in every +# dimension: +# +# | quantity | how | tolerance in the report | +# |---|---|---| +# | `fixed_sigma2`, `fixed_beta`, `fixed_ll`, `fixed_loo` | `optim="none"`, θ = θ₀ fixed | rel. 1e-8 (error) | +# | `fixed_mean`, `fixed_stdev` | predict at the first 50 test points | rel. 1e-8 (error) | +# | `llfun`, `llfun_grad` | `logLikelihoodFun(θ₀)` with gradient | rel. 1e-8 (error) | +# | `fit_theta`, `fit_ll`, `fit_mean` | default fit (`BFGS`) | rel. 1e-4 (warn: optimizer paths may differ) | +# | `sim` | `simulate(nsim=3, seed=123)` at the first 5 test points (5 × 3, column-major) | rel. 1e-8 (warn: RNG may be binding-specific) | +# +# Then, on Branin only, every other model class and fit method, so that each +# binding is checked on all of them (the class-specific notebooks are in Python +# only): with fixed hyperparameters the numbers must be identical (error); a +# fit is compared at warn level, as above. +# +# | quantity | how | tolerance in the report | +# |---|---|---| +# | `nugget_fixed_mean`, `nugget_fixed_stdev` | `noise="nugget"`, θ = θ₀, σ² = var(y), nugget = 0.01 var(y), all fixed | rel. 1e-8 (error) | +# | `noise_fixed_mean`, `noise_fixed_stdev` | known noise variances 0.01 var(y), θ = θ₀ and σ² fixed | rel. 1e-8 (error) | +# | `nugget_fit_ll`, `nugget_fit_mean` | `noise="nugget"`, default fit | rel. 1e-4 (warn) | +# | `vecchia_fixed_ll`, `vecchia_fixed_mean`, `nystrom_fixed_ll`, `nystrom_fixed_mean` | `method="vecchia(10)"` / `"nystrom(10)"`, θ = θ₀ fixed; likelihood and prediction with that method | rel. 1e-8 (error) | +# | `warp_fit_mean` | `WarpKriging`, `kumaraswamy` on both inputs, default fit | rel. 1e-4 (warn) | +# | `mlp_fit_mean` | `MLPKriging`, hidden layer of 8 (`tanh`), 2 features, `BFGS10` | rel. 1e-4 (warn) | +# | `nested_fit_mean` | `NestedKriging`, 2 groups (`kmeans`, seed 1), `NK` | rel. 1e-4 (warn) | + +# %% +import csv + +import numpy as np +import pylibkriging as lk + +import lkbench +from lkbench import compat +from lkbench.checks import DATA_DIR as DATA, RESULTS_DIR + +suite = lkbench.Suite("cross-binding-reference_all-classes_python") +print("pylibkriging", compat.version()) +DATASETS = ["branin", "hartmann3", "borehole"] +rows = [] + + +def put(dataset, quantity, values): + for i, v in enumerate(np.ravel(values), start=1): + rows.append(("python", dataset, quantity, i, f"{float(v):.17g}")) + + +def load(name, stem): + return np.loadtxt(DATA / f"{name}_{stem}.csv", delimiter=",", ndmin=2) + +# %% +for name in DATASETS: + X, y, Xt = load(name, "X"), load(name, "y").ravel(), load(name, "Xt")[:50] + d = X.shape[1] + theta0 = np.full((1, d), 0.3) + kf = lk.Kriging(y, X, "matern5_2", regmodel="constant", optim="none", + parameters={"theta": theta0, "is_theta_estim": False}) + put(name, "fixed_sigma2", kf.sigma2()) + put(name, "fixed_beta", kf.beta()) + put(name, "fixed_ll", kf.logLikelihood()) + put(name, "fixed_loo", kf.leaveOneOut()) + m, s, *_ = kf.predict(Xt, return_stdev=True) + put(name, "fixed_mean", m) + put(name, "fixed_stdev", s) + ll, grad, _ = kf.logLikelihoodFun(theta0.ravel(), True) + put(name, "llfun", ll) + put(name, "llfun_grad", grad) + suite.close(f"{name}: logLikelihood() == logLikelihoodFun(theta0)", kf.logLikelihood(), ll, rtol=1e-10) + k = lk.Kriging(y, X, "matern5_2") + put(name, "fit_theta", k.theta()) + put(name, "fit_ll", k.logLikelihood()) + put(name, "fit_mean", k.predict(Xt, return_stdev=False)[0]) + put(name, "sim", np.ravel(np.asarray(kf.simulate(3, 123, Xt[:5])), order="F")) # column-major, as R/Octave/Julia + +# %% [markdown] +# ## Other classes and fit methods (Branin) + +# %% +name = "branin" +X, y, Xt = load(name, "X"), load(name, "y").ravel(), load(name, "Xt")[:50] +theta0 = np.full((1, 2), 0.3) +s2 = float(np.var(y)); nug = 0.01 * s2 +fixed = {"theta": theta0, "is_theta_estim": False, "sigma2": s2, "is_sigma2_estim": False} +kn = lk.Kriging(y, X, "matern5_2", noise="nugget", optim="none", + parameters={**fixed, "nugget": nug, "is_nugget_estim": False}) +m, s, *_ = kn.predict(Xt, return_stdev=True) +put(name, "nugget_fixed_mean", m); put(name, "nugget_fixed_stdev", s) +kh = lk.Kriging(y, X, "matern5_2", noise=np.full(len(y), nug), optim="none", parameters=fixed) +m, s, *_ = kh.predict(Xt, return_stdev=True) +put(name, "noise_fixed_mean", m); put(name, "noise_fixed_stdev", s) +kf = lk.Kriging(y, X, "matern5_2", noise="nugget") +put(name, "nugget_fit_ll", kf.logLikelihood()); put(name, "nugget_fit_mean", kf.predict(Xt, return_stdev=False)[0]) +for meth in ["vecchia", "nystrom"] if compat.has("method") else []: + ka = lk.Kriging(y, X, "matern5_2", optim="none", parameters={"theta": theta0, "is_theta_estim": False}, + method=f"{meth}(10)") + put(name, f"{meth}_fixed_ll", ka.logLikelihood(method=meth)) + put(name, f"{meth}_fixed_mean", ka.predict(Xt, return_stdev=False, method=meth)[0]) +if not compat.has("method"): + suite.skip("vecchia / nystrom quantities", "pylibkriging without the method= argument") +kw = lk.WarpKriging(y, X, ["kumaraswamy", "kumaraswamy"], "matern5_2") +put(name, "warp_fit_mean", kw.predict(Xt, return_stdev=False)[0]) +km = lk.MLPKriging(y, X, hidden_dims=[8], d_out=2, activation="tanh", kernel="matern5_2", optim="BFGS10") +put(name, "mlp_fit_mean", km.predict(Xt, return_stdev=False)[0]) +kk = lk.NestedKriging(y, X, "matern5_2", nb_groups=2, aggregation="NK", seed=1) +put(name, "nested_fit_mean", kk.predict(Xt, return_stdev=False)[0]) + +# %% +RESULTS_DIR.mkdir(exist_ok=True) +with open(RESULTS_DIR / "xbind_python.csv", "w", newline="") as fh: + w = csv.writer(fh) + w.writerow(["lang", "dataset", "quantity", "index", "value"]) + w.writerows(rows) +print(len(rows), "values written") + +# %% +suite.finalize() diff --git a/tools/src/cross-binding_Kriging_R.R b/tools/src/cross-binding_Kriging_R.R deleted file mode 100644 index 4496ac8..0000000 --- a/tools/src/cross-binding_Kriging_R.R +++ /dev/null @@ -1,49 +0,0 @@ -# %% [markdown] -# # R · Kriging — cross-binding consistency: the R side -# -# Same protocol as `cross-binding-reference_Kriging_python.ipynb` (see there), on the -# shared datasets of `data/`; writes `results/xbind_R.csv`, compared with the -# Python values by `tools/report.py`. - -# %% -suppressPackageStartupMessages(library(rlibkriging)) -source("tools/r/checks.R") -s <- new_suite("cross-binding_Kriging_R") -cat("rlibkriging", as.character(packageVersion("rlibkriging")), "\n") -rows <- list() -put <- function(dataset, quantity, values) { - v <- as.numeric(values) - rows[[length(rows) + 1]] <<- data.frame(lang = "R", dataset = dataset, quantity = quantity, - index = seq_along(v), value = sprintf("%.17g", v)) -} - -# %% -for (name in c("branin", "hartmann3", "borehole")) { - X <- read_data(name, "X"); y <- as.numeric(read_data(name, "y")); Xt <- read_data(name, "Xt")[1:50, , drop = FALSE] - d <- ncol(X) - theta0 <- matrix(0.3, nrow = 1, ncol = d) - kf <- Kriging(y, X, "matern5_2", regmodel = "constant", optim = "none", - parameters = list(theta = theta0, is_theta_estim = FALSE)) - put(name, "fixed_sigma2", kf$sigma2()) - put(name, "fixed_beta", kf$beta()) - put(name, "fixed_ll", logLikelihood(kf)) - put(name, "fixed_loo", leaveOneOut(kf)) - p <- predict(kf, Xt, return_stdev = TRUE) - put(name, "fixed_mean", p$mean) - put(name, "fixed_stdev", p$stdev) - llf <- logLikelihoodFun(kf, theta0, return_grad = TRUE) - put(name, "llfun", llf$logLikelihood) - put(name, "llfun_grad", llf$logLikelihoodGrad) - check_close(s, paste0(name, ": logLikelihood() == logLikelihoodFun(theta0)"), logLikelihood(kf), llf$logLikelihood, rtol = 1e-10) - k <- Kriging(y, X, "matern5_2") - put(name, "fit_theta", k$theta()) - put(name, "fit_ll", logLikelihood(k)) - put(name, "fit_mean", predict(k, Xt, return_stdev = FALSE)$mean) - put(name, "sim", simulate(kf, nsim = 3, seed = 123, x = Xt[1:5, , drop = FALSE])) -} -out <- do.call(rbind, rows) -write.csv(out, file.path(results_dir, "xbind_R.csv"), row.names = FALSE, quote = FALSE) -cat(nrow(out), "values written\n") - -# %% -finalize(s) diff --git a/tools/src/cross-binding_Kriging_julia.jl b/tools/src/cross-binding_Kriging_julia.jl deleted file mode 100644 index 4e41d5a..0000000 --- a/tools/src/cross-binding_Kriging_julia.jl +++ /dev/null @@ -1,52 +0,0 @@ -# %% [markdown] -# # Julia · Kriging — cross-binding consistency: the Julia side -# -# Same protocol as `cross-binding-reference_Kriging_python.ipynb` (see there), on -# the shared datasets of `data/`; writes `results/xbind_julia.csv`, compared with -# the Python values by `tools/report.py`. -# -# The module is `JLibKriging` (the packaged binding, [JLibKriging.jl](https://github.com/libKriging/JLibKriging.jl)), -# imported under a short name so that its exported accessors (`X`, `y`, `theta`, ...) -# do not clash with the variables of the notebook. - -# %% -import JLibKriging as LK -include("tools/julia/checks.jl") -s = Suite("cross-binding_Kriging_julia") -rows = Vector{Vector{String}}() -record!(dataset, quantity, values) = - for (i, v) in enumerate(vec(Float64.(collect(values)))) - push!(rows, ["julia", dataset, quantity, string(i), @sprintf("%.17g", v)]) - end - -# %% -for name in ("branin", "hartmann3", "borehole") - Xd = read_data(name, "X"); yd = vec(read_data(name, "y")); Xt = read_data(name, "Xt")[1:50, :] - d = size(Xd, 2) - theta0 = fill(0.3, d) - kf = LK.Kriging(yd, Xd, "matern5_2"; regmodel="constant", optim="none", theta=theta0, is_theta_estim=false) - record!(name, "fixed_sigma2", LK.sigma2(kf)) - record!(name, "fixed_beta", LK.beta(kf)) - record!(name, "fixed_ll", LK.log_likelihood(kf)) - record!(name, "fixed_loo", LK.leave_one_out(kf)) - p = LK.predict(kf, Xt; return_stdev=true) - record!(name, "fixed_mean", p.mean) - record!(name, "fixed_stdev", p.stdev) - llf = LK.log_likelihood_fun(kf, theta0; return_grad=true) - record!(name, "llfun", llf.ll) - record!(name, "llfun_grad", llf.grad) - check_close!(s, "$name: log_likelihood() == log_likelihood_fun(theta0)", LK.log_likelihood(kf), llf.ll; rtol=1e-10) - k = LK.Kriging(yd, Xd, "matern5_2") - record!(name, "fit_theta", LK.theta(k)) - record!(name, "fit_ll", LK.log_likelihood(k)) - record!(name, "fit_mean", LK.predict(k, Xt; return_stdev=false).mean) - record!(name, "sim", LK.simulate(kf, 3, 123, Xt[1:5, :])) -end -open(joinpath(RESULTS_DIR, "xbind_julia.csv"), "w") do io - println(io, "lang,dataset,quantity,index,value") - for r in rows; println(io, join(r, ",")); end -end -println(length(rows), " values written") - -# %% -finalize!(s) diff --git a/tools/src/cross-binding_Kriging_octave.m b/tools/src/cross-binding_Kriging_octave.m deleted file mode 100644 index 5c2d89e..0000000 --- a/tools/src/cross-binding_Kriging_octave.m +++ /dev/null @@ -1,60 +0,0 @@ -% %% [markdown] -% # Octave · Kriging — cross-binding consistency: the Octave side -% -% Same protocol as `cross-binding-reference_Kriging_python.ipynb` (see there), on the -% shared datasets of `data/`; writes `results/xbind_octave.csv`, compared with -% the Python values by `tools/report.py`. mLibKriging must be on the path -% (`OCTAVE_PATH` or `addpath`). - -% %% -addpath(fullfile(pwd, "tools", "octave")); -s = lkb_suite("cross-binding_Kriging_octave"); -out = {}; -function out = put(out, dataset, quantity, values) - v = double(values(:)); - for i = 1:numel(v) - out(end + 1, :) = {"octave", dataset, quantity, i, sprintf("%.17g", v(i))}; - end -end - -% %% -for name = {"branin", "hartmann3", "borehole"} - name = name{1}; - X = read_data(name, "X"); y = read_data(name, "y"); Xt = read_data(name, "Xt")(1:50, :); - d = columns(X); - theta0 = 0.3 * ones(1, d); - kf = Kriging(y, X, "matern5_2", "constant", false, "none", "LL", Params("theta", theta0, "is_theta_estim", false)); - v = kf.sigma2(); out = put(out, name, "fixed_sigma2", v); - v = kf.beta(); out = put(out, name, "fixed_beta", v); - ll = kf.logLikelihood(); - out = put(out, name, "fixed_ll", ll); - loo = kf.leaveOneOut(); - out = put(out, name, "fixed_loo", loo); - [m, sd] = kf.predict(Xt, true, false, false); - out = put(out, name, "fixed_mean", m); - out = put(out, name, "fixed_stdev", sd); - [llf, grad] = kf.logLikelihoodFun(theta0', true, false); - out = put(out, name, "llfun", llf); - out = put(out, name, "llfun_grad", grad); - s = lkb_close(s, [name ": logLikelihood() == logLikelihoodFun(theta0)"], ll, llf, 1e-10); - k = Kriging(y, X, "matern5_2"); - v = k.theta(); out = put(out, name, "fit_theta", v); - llk = k.logLikelihood(); - out = put(out, name, "fit_ll", llk); - [mk, ~] = k.predict(Xt, false, false, false); - out = put(out, name, "fit_mean", mk); - S = kf.simulate(int32(3), int32(123), Xt(1:5, :), false); - out = put(out, name, "sim", S); -end -d = getenv("LKBENCH_RESULTS"); if isempty(d), d = "results"; end -if ~exist(d, "dir"), mkdir(d); end -fid = fopen(fullfile(d, "xbind_octave.csv"), "w"); -fprintf(fid, "lang,dataset,quantity,index,value\n"); -for i = 1:size(out, 1) - fprintf(fid, "%s,%s,%s,%d,%s\n", out{i, :}); -end -fclose(fid); -printf("%d values written\n", size(out, 1)); - -% %% -lkb_finalize(s); diff --git a/tools/src/cross-binding_all-classes_R.R b/tools/src/cross-binding_all-classes_R.R new file mode 100644 index 0000000..0bdb655 --- /dev/null +++ b/tools/src/cross-binding_all-classes_R.R @@ -0,0 +1,82 @@ +# %% [markdown] +# # R · all classes — cross-binding consistency: the R side +# +# Same protocol as `cross-binding-reference_all-classes_python.ipynb` (see there), on the +# shared datasets of `data/`; writes `results/xbind_R.csv`, compared with the +# Python values by `tools/report.py`. + +# %% +suppressPackageStartupMessages(library(rlibkriging)) +source("tools/r/checks.R") +s <- new_suite("cross-binding_all-classes_R") +cat("rlibkriging", as.character(packageVersion("rlibkriging")), "\n") +rows <- list() +put <- function(dataset, quantity, values) { + v <- as.numeric(values) + rows[[length(rows) + 1]] <<- data.frame(lang = "R", dataset = dataset, quantity = quantity, + index = seq_along(v), value = sprintf("%.17g", v)) +} + +# %% +for (name in c("branin", "hartmann3", "borehole")) { + X <- read_data(name, "X"); y <- as.numeric(read_data(name, "y")); Xt <- read_data(name, "Xt")[1:50, , drop = FALSE] + d <- ncol(X) + theta0 <- matrix(0.3, nrow = 1, ncol = d) + kf <- Kriging(y, X, "matern5_2", regmodel = "constant", optim = "none", + parameters = list(theta = theta0, is_theta_estim = FALSE)) + put(name, "fixed_sigma2", kf$sigma2()) + put(name, "fixed_beta", kf$beta()) + put(name, "fixed_ll", logLikelihood(kf)) + put(name, "fixed_loo", leaveOneOut(kf)) + p <- predict(kf, Xt, return_stdev = TRUE) + put(name, "fixed_mean", p$mean) + put(name, "fixed_stdev", p$stdev) + llf <- logLikelihoodFun(kf, theta0, return_grad = TRUE) + put(name, "llfun", llf$logLikelihood) + put(name, "llfun_grad", llf$logLikelihoodGrad) + check_close(s, paste0(name, ": logLikelihood() == logLikelihoodFun(theta0)"), logLikelihood(kf), llf$logLikelihood, rtol = 1e-10) + k <- Kriging(y, X, "matern5_2") + put(name, "fit_theta", k$theta()) + put(name, "fit_ll", logLikelihood(k)) + put(name, "fit_mean", predict(k, Xt, return_stdev = FALSE)$mean) + put(name, "sim", simulate(kf, nsim = 3, seed = 123, x = Xt[1:5, , drop = FALSE])) +} + +# %% [markdown] +# ## Other classes and fit methods (Branin) + +# %% +name <- "branin" +X <- read_data(name, "X"); y <- as.numeric(read_data(name, "y")); Xt <- read_data(name, "Xt")[1:50, , drop = FALSE] +theta0 <- matrix(0.3, nrow = 1, ncol = 2) +s2 <- mean((y - mean(y))^2); nug <- 0.01 * s2 # population variance, as numpy.var +fixed <- list(theta = theta0, is_theta_estim = FALSE, sigma2 = s2, is_sigma2_estim = FALSE) +kn <- Kriging(y, X, "matern5_2", noise = "nugget", optim = "none", + parameters = c(fixed, list(nugget = nug, is_nugget_estim = FALSE))) +p <- predict(kn, Xt, return_stdev = TRUE) +put(name, "nugget_fixed_mean", p$mean); put(name, "nugget_fixed_stdev", p$stdev) +kh <- Kriging(y, X, "matern5_2", noise = rep(nug, length(y)), optim = "none", parameters = fixed) +p <- predict(kh, Xt, return_stdev = TRUE) +put(name, "noise_fixed_mean", p$mean); put(name, "noise_fixed_stdev", p$stdev) +kf <- Kriging(y, X, "matern5_2", noise = "nugget") +put(name, "nugget_fit_ll", logLikelihood(kf)); put(name, "nugget_fit_mean", predict(kf, Xt, return_stdev = FALSE)$mean) +for (meth in c("vecchia", "nystrom")) tryCatch({ # method= is absent from older releases + ka <- Kriging(y, X, "matern5_2", optim = "none", parameters = list(theta = theta0, is_theta_estim = FALSE), + method = paste0(meth, "(10)")) + put(name, paste0(meth, "_fixed_ll"), ka$logLikelihood(method = meth)) + put(name, paste0(meth, "_fixed_mean"), predict(ka, Xt, return_stdev = FALSE, method = meth)$mean) +}, error = function(e) message(meth, " skipped (reported as missing): ", conditionMessage(e))) +kw <- WarpKriging(y, X, warping = c("kumaraswamy", "kumaraswamy"), kernel = "matern5_2") +put(name, "warp_fit_mean", predict(kw, Xt, return_stdev = FALSE)$mean) +km <- MLPKriging(y, X, hidden_dims = c(8), d_out = 2, activation = "tanh", kernel = "matern5_2", optim = "BFGS10") +put(name, "mlp_fit_mean", predict(km, Xt, return_stdev = FALSE)$mean) +kk <- NestedKriging(y, X, kernel = "matern5_2", nb_groups = 2, aggregation = "NK", seed = 1) +put(name, "nested_fit_mean", predict(kk, Xt, return_stdev = FALSE)$mean) + +# %% +out <- do.call(rbind, rows) +write.csv(out, file.path(results_dir, "xbind_R.csv"), row.names = FALSE, quote = FALSE) +cat(nrow(out), "values written\n") + +# %% +finalize(s) diff --git a/tools/src/cross-binding_all-classes_julia.jl b/tools/src/cross-binding_all-classes_julia.jl new file mode 100644 index 0000000..821cdbf --- /dev/null +++ b/tools/src/cross-binding_all-classes_julia.jl @@ -0,0 +1,85 @@ +# %% [markdown] +# # Julia · all classes — cross-binding consistency: the Julia side +# +# Same protocol as `cross-binding-reference_all-classes_python.ipynb` (see there), on +# the shared datasets of `data/`; writes `results/xbind_julia.csv`, compared with +# the Python values by `tools/report.py`. +# +# The module is `JLibKriging` (the packaged binding, [JLibKriging.jl](https://github.com/libKriging/JLibKriging.jl)), +# imported under a short name so that its exported accessors (`X`, `y`, `theta`, ...) +# do not clash with the variables of the notebook. + +# %% +import JLibKriging as LK +include("tools/julia/checks.jl") +s = Suite("cross-binding_all-classes_julia") +rows = Vector{Vector{String}}() +record!(dataset, quantity, values) = + for (i, v) in enumerate(vec(Float64.(collect(values)))) + push!(rows, ["julia", dataset, quantity, string(i), @sprintf("%.17g", v)]) + end + +# %% +for name in ("branin", "hartmann3", "borehole") + Xd = read_data(name, "X"); yd = vec(read_data(name, "y")); Xt = read_data(name, "Xt")[1:50, :] + d = size(Xd, 2) + theta0 = fill(0.3, d) + kf = LK.Kriging(yd, Xd, "matern5_2"; regmodel="constant", optim="none", theta=theta0, is_theta_estim=false) + record!(name, "fixed_sigma2", LK.sigma2(kf)) + record!(name, "fixed_beta", LK.beta(kf)) + record!(name, "fixed_ll", LK.log_likelihood(kf)) + record!(name, "fixed_loo", LK.leave_one_out(kf)) + p = LK.predict(kf, Xt; return_stdev=true) + record!(name, "fixed_mean", p.mean) + record!(name, "fixed_stdev", p.stdev) + llf = LK.log_likelihood_fun(kf, theta0; return_grad=true) + record!(name, "llfun", llf.ll) + record!(name, "llfun_grad", llf.grad) + check_close!(s, "$name: log_likelihood() == log_likelihood_fun(theta0)", LK.log_likelihood(kf), llf.ll; rtol=1e-10) + k = LK.Kriging(yd, Xd, "matern5_2") + record!(name, "fit_theta", LK.theta(k)) + record!(name, "fit_ll", LK.log_likelihood(k)) + record!(name, "fit_mean", LK.predict(k, Xt; return_stdev=false).mean) + record!(name, "sim", LK.simulate(kf, 3, 123, Xt[1:5, :])) +end + +# %% [markdown] +# ## Other classes and fit methods (Branin) + +# %% +name = "branin" +Xd = read_data(name, "X"); yd = vec(read_data(name, "y")); Xt = read_data(name, "Xt")[1:50, :] +theta0 = [0.3, 0.3] +s2 = sum((yd .- sum(yd) / length(yd)) .^ 2) / length(yd); nug = 0.01 * s2 # population variance, as numpy.var +kn = LK.Kriging(yd, Xd, "matern5_2"; noise="nugget", optim="none", theta=theta0, is_theta_estim=false, + sigma2=s2, is_sigma2_estim=false, nugget=nug, is_nugget_estim=false) +p = LK.predict(kn, Xt; return_stdev=true) +record!(name, "nugget_fixed_mean", p.mean); record!(name, "nugget_fixed_stdev", p.stdev) +kh = LK.Kriging(yd, Xd, "matern5_2"; noise=fill(nug, length(yd)), optim="none", theta=theta0, is_theta_estim=false, + sigma2=s2, is_sigma2_estim=false) +p = LK.predict(kh, Xt; return_stdev=true) +record!(name, "noise_fixed_mean", p.mean); record!(name, "noise_fixed_stdev", p.stdev) +kf = LK.Kriging(yd, Xd, "matern5_2"; noise="nugget") +record!(name, "nugget_fit_ll", LK.log_likelihood(kf)) +record!(name, "nugget_fit_mean", LK.predict(kf, Xt; return_stdev=false).mean) +for meth in ("vecchia", "nystrom") + ka = LK.Kriging(yd, Xd, "matern5_2"; optim="none", theta=theta0, is_theta_estim=false, method="$meth(10)") + record!(name, "$(meth)_fixed_ll", LK.log_likelihood(ka; method=meth)) + record!(name, "$(meth)_fixed_mean", LK.predict(ka, Xt; return_stdev=false, method=meth).mean) +end +kw = LK.WarpKriging(yd, Xd, ["kumaraswamy", "kumaraswamy"], "matern5_2") +record!(name, "warp_fit_mean", LK.predict(kw, Xt; return_stdev=false).mean) +km = LK.MLPKriging(yd, Xd, [8], 2; activation="tanh", kernel="matern5_2", optim="BFGS10") +record!(name, "mlp_fit_mean", LK.predict(km, Xt; return_stdev=false).mean) +kk = LK.NestedKriging(yd, Xd, "matern5_2", 2; aggregation="NK", seed=1) +record!(name, "nested_fit_mean", LK.predict(kk, Xt; return_stdev=false).mean) + +# %% +open(joinpath(RESULTS_DIR, "xbind_julia.csv"), "w") do io + println(io, "lang,dataset,quantity,index,value") + for r in rows; println(io, join(r, ",")); end +end +println(length(rows), " values written") + +# %% +finalize!(s) diff --git a/tools/src/cross-binding_all-classes_octave.m b/tools/src/cross-binding_all-classes_octave.m new file mode 100644 index 0000000..a562aed --- /dev/null +++ b/tools/src/cross-binding_all-classes_octave.m @@ -0,0 +1,101 @@ +% %% [markdown] +% # Octave · all classes — cross-binding consistency: the Octave side +% +% Same protocol as `cross-binding-reference_all-classes_python.ipynb` (see there), on the +% shared datasets of `data/`; writes `results/xbind_octave.csv`, compared with +% the Python values by `tools/report.py`. mLibKriging must be on the path +% (`OCTAVE_PATH` or `addpath`). + +% %% +addpath(fullfile(pwd, "tools", "octave")); +s = lkb_suite("cross-binding_all-classes_octave"); +out = {}; +function out = put(out, dataset, quantity, values) + v = double(values(:)); + for i = 1:numel(v) + out(end + 1, :) = {"octave", dataset, quantity, i, sprintf("%.17g", v(i))}; + end +end + +% %% +for name = {"branin", "hartmann3", "borehole"} + name = name{1}; + X = read_data(name, "X"); y = read_data(name, "y"); Xt = read_data(name, "Xt")(1:50, :); + d = columns(X); + theta0 = 0.3 * ones(1, d); + kf = Kriging(y, X, "matern5_2", "constant", false, "none", "LL", Params("theta", theta0, "is_theta_estim", false)); + v = kf.sigma2(); out = put(out, name, "fixed_sigma2", v); + v = kf.beta(); out = put(out, name, "fixed_beta", v); + ll = kf.logLikelihood(); + out = put(out, name, "fixed_ll", ll); + loo = kf.leaveOneOut(); + out = put(out, name, "fixed_loo", loo); + [m, sd] = kf.predict(Xt, true, false, false); + out = put(out, name, "fixed_mean", m); + out = put(out, name, "fixed_stdev", sd); + [llf, grad] = kf.logLikelihoodFun(theta0', true, false); + out = put(out, name, "llfun", llf); + out = put(out, name, "llfun_grad", grad); + s = lkb_close(s, [name ": logLikelihood() == logLikelihoodFun(theta0)"], ll, llf, 1e-10); + k = Kriging(y, X, "matern5_2"); + v = k.theta(); out = put(out, name, "fit_theta", v); + llk = k.logLikelihood(); + out = put(out, name, "fit_ll", llk); + [mk, ~] = k.predict(Xt, false, false, false); + out = put(out, name, "fit_mean", mk); + S = kf.simulate(int32(3), int32(123), Xt(1:5, :), false); + out = put(out, name, "sim", S); +end + +% %% [markdown] +% ## Other classes and fit methods (Branin) + +% %% +name = "branin"; +X = read_data(name, "X"); y = read_data(name, "y"); Xt = read_data(name, "Xt")(1:50, :); +theta0 = [0.3, 0.3]; +s2 = mean((y - mean(y)).^2); nug = 0.01 * s2; % population variance, as numpy.var +kn = Kriging(y, X, "matern5_2", "constant", false, "none", "LL", ... + Params("theta", theta0, "is_theta_estim", false, "sigma2", s2, "is_sigma2_estim", false, ... + "nugget", nug, "is_nugget_estim", false), "nugget"); +[m, sd] = kn.predict(Xt, true, false, false); +out = put(out, name, "nugget_fixed_mean", m); out = put(out, name, "nugget_fixed_stdev", sd); +kh = Kriging(y, X, "matern5_2", "constant", false, "none", "LL", ... + Params("theta", theta0, "is_theta_estim", false, "sigma2", s2, "is_sigma2_estim", false), ... + "heterogeneous", nug * ones(size(y))); +[m, sd] = kh.predict(Xt, true, false, false); +out = put(out, name, "noise_fixed_mean", m); out = put(out, name, "noise_fixed_stdev", sd); +kf = Kriging(y, X, "matern5_2", "constant", false, "BFGS", "LL", [], "nugget"); +ll = kf.logLikelihood(); out = put(out, name, "nugget_fit_ll", ll); +m = kf.predict(Xt, false, false, false); out = put(out, name, "nugget_fit_mean", m); +for meth = {"vecchia", "nystrom"} + meth = meth{1}; + try % method is absent from older releases + ka = Kriging(y, X, "matern5_2", "constant", false, "none", "LL", Params("theta", theta0, "is_theta_estim", false), ... + "none", [], [meth "(10)"]); + ll = ka.logLikelihood(meth); out = put(out, name, [meth "_fixed_ll"], ll); + m = ka.predict(Xt, false, false, false, meth); out = put(out, name, [meth "_fixed_mean"], m); + catch err + printf("%s skipped (reported as missing): %s\n", meth, err.message); + end +end +kw = WarpKriging(y, X, {"kumaraswamy", "kumaraswamy"}, "matern5_2"); +m = kw.predict(Xt, false, false, false); out = put(out, name, "warp_fit_mean", m); +km = MLPKriging(y, X, [8], 2, "tanh", "matern5_2", "constant", false, "BFGS10"); +m = km.predict(Xt, false, false, false); out = put(out, name, "mlp_fit_mean", m); +kk = NestedKriging(y, X, "matern5_2", 2, "NK", "kmeans", 1); +m = kk.predict(Xt); out = put(out, name, "nested_fit_mean", m); + +% %% +d = getenv("LKBENCH_RESULTS"); if isempty(d), d = "results"; end +if ~exist(d, "dir"), mkdir(d); end +fid = fopen(fullfile(d, "xbind_octave.csv"), "w"); +fprintf(fid, "lang,dataset,quantity,index,value\n"); +for i = 1:size(out, 1) + fprintf(fid, "%s,%s,%s,%d,%s\n", out{i, :}); +end +fclose(fid); +printf("%d values written\n", size(out, 1)); + +% %% +lkb_finalize(s); diff --git a/tools/src/deep-kernel-projected-branin_MLPKriging_python.py b/tools/src/deep-kernel-projected-branin_MLPKriging_python.py index 966a59e..037ae12 100644 --- a/tools/src/deep-kernel-projected-branin_MLPKriging_python.py +++ b/tools/src/deep-kernel-projected-branin_MLPKriging_python.py @@ -14,6 +14,7 @@ # %% import time +import matplotlib.pyplot as plt import numpy as np import pylibkriging as lk @@ -62,7 +63,10 @@ def projected(X): suite.greater(f"{case}: Q2", res["q2"], 0.8) else: suite.greater(f"{case}: Q2 - Q2(Kriging)", res["q2"] - res_k["q2"], -0.1, severity="warn") - display(suite.table().loc["projected_branin_10d"].round(4)) + tab = suite.table().loc["projected_branin_10d"] + tab["q2"].plot.barh(figsize=(6, 3)); plt.xlabel("Q² (test set)"); plt.xlim(min(0, tab["q2"].min()), 1) + plt.title("Branin(Wx), x in [0,1]^10"); plt.show() + display(tab.round(4)) else: suite.skip("MLPKriging", "not available in this pylibkriging") diff --git a/tools/src/deep-kernel_MLPKriging_R.R b/tools/src/deep-kernel_MLPKriging_R.R deleted file mode 100644 index b0184da..0000000 --- a/tools/src/deep-kernel_MLPKriging_R.R +++ /dev/null @@ -1,69 +0,0 @@ -# %% [markdown] -# # R · `MLPKriging` — a joint feature map ℝ² → ℝ² before the kernel -# -# Branin on [0,1]² (n = 30 LHS points unless stated, Q² on 500 uniform test -# points), through rlibkriging. Checks (error-level unless stated): -# -# * 10 restarts (`BFGS10`): finite predictions, Q² > 0.9 (warn-level) -# * `save` then `load`: the same predicted mean -# * the default optimizer as good as `BFGS10` (warn-level; see libKriging/libKriging#382) -# -# A class missing from the installed rlibkriging (release channel) is recorded as SKIP. - -# %% -suppressPackageStartupMessages(library(rlibkriging)) -source("tools/r/checks.R") -s <- new_suite("deep-kernel_MLPKriging_R") -cat("rlibkriging", as.character(packageVersion("rlibkriging")), "\n") - -branin <- function(U) { # on [0,1]^2 - x1 <- 15 * U[, 1] - 5; x2 <- 15 * U[, 2] - (x2 - 5.1 / (4 * pi^2) * x1^2 + 5 / pi * x1 - 6)^2 + 10 * (1 - 1 / (8 * pi)) * cos(x1) + 10 -} -set.seed(1) -lhs <- function(n, d) sapply(1:d, function(j) (sample(n) - runif(n)) / n) -X <- lhs(30, 2); y <- branin(X) -Xt <- matrix(runif(1000), ncol = 2); yt <- branin(Xt) -tmp <- tempdir() -has_class <- function(name) exists(name, mode = "function") -skip <- function(s, check, reason) { - s$checks[[length(s$checks) + 1]] <- data.frame(suite = s$name, lang = "R", check = check, severity = "info", - status = "SKIP", value = "", expected = "", detail = reason, stringsAsFactors = FALSE) - message("[SKIP] ", check, ": ", reason) -} -roundtrip_same_mean <- function(s, label, k, Xn) { - f <- file.path(tmp, paste0(gsub("[^A-Za-z0-9]", "_", label), ".json")) - check_runs(s, paste0(label, ": save + load"), { - save(k, f) - k2 <- rlibkriging::load(f) - check_close(s, paste0(label, ": load(save(k)) predicts the same mean"), - predict(k2, Xn, return_stdev = FALSE)$mean, predict(k, Xn, return_stdev = FALSE)$mean, rtol = 1e-10) - }) -} - -# %% [markdown] -# ## `MLPKriging`: a joint feature map ℝ² → ℝ² before the kernel - -# %% -if (has_class("MLPKriging")) { - check_runs(s, "MLPKriging: fit + predict", { - km <- MLPKriging(y, X, hidden_dims = c(8), d_out = 2, activation = "tanh", kernel = "matern5_2", optim = "BFGS10") - pm <- predict(km, Xt, return_stdev = TRUE) - check_less(s, "MLPKriging: non-finite predictions", sum(!is.finite(c(pm$mean, pm$stdev))), 1) - add_metric(s, "branin", "MLPKriging tanh [8] -> 2, BFGS10", q2 = q2(yt, pm$mean)) - check_greater(s, "MLPKriging: Q2", q2(yt, pm$mean), 0.9, severity = "warn") - roundtrip_same_mean(s, "MLPKriging", km, Xt) - # the default optimizer ("BFGS+Adam") - kd <- MLPKriging(y, X, hidden_dims = c(8), d_out = 2, activation = "tanh", kernel = "matern5_2") - qd <- q2(yt, predict(kd, Xt, return_stdev = FALSE)$mean) - add_metric(s, "branin", "MLPKriging tanh [8] -> 2, default optim", q2 = qd) - check_greater(s, "MLPKriging: Q2(default optim) - Q2(BFGS10)", qd - q2(yt, pm$mean), -0.05, severity = "warn") - }) -} else skip(s, "MLPKriging", "not in this rlibkriging") - -# %% -me <- do.call(rbind, s$metrics) -print(me[me$metric == "q2", c("case", "package", "value")], row.names = FALSE) - -# %% -finalize(s) diff --git a/tools/src/deep-kernel_MLPKriging_julia.jl b/tools/src/deep-kernel_MLPKriging_julia.jl deleted file mode 100644 index 57b1d87..0000000 --- a/tools/src/deep-kernel_MLPKriging_julia.jl +++ /dev/null @@ -1,71 +0,0 @@ -# %% [markdown] -# # Julia · `MLPKriging` — a joint feature map ℝ² → ℝ² before the kernel -# -# Branin on [0,1]² (n = 30 LHS points unless stated, Q² on 500 uniform test -# points), through JLibKriging. Checks (error-level unless stated): -# -# * 10 restarts (`BFGS10`): finite predictions, Q² > 0.9 (warn-level) -# * `save` then `load`: the same predicted mean -# * the default optimizer as good as `BFGS10` (warn-level; see libKriging/libKriging#382) -# -# The same checks as `deep-kernel_MLPKriging_R.ipynb`, with the Julia binding. - -# %% -import JLibKriging as LK -using Random -include("tools/julia/checks.jl") -s = Suite("deep-kernel_MLPKriging_julia") - -branin(U) = begin - x1 = 15 .* U[:, 1] .- 5; x2 = 15 .* U[:, 2] - (x2 .- 5.1 / (4π^2) .* x1 .^ 2 .+ 5 / π .* x1 .- 6) .^ 2 .+ 10 * (1 - 1 / (8π)) .* cos.(x1) .+ 10 -end -rng = MersenneTwister(1) -lhs(n, d) = hcat([(randperm(rng, n) .- rand(rng, n)) ./ n for _ in 1:d]...) -X = lhs(30, 2); y = branin(X) -Xt = rand(rng, 500, 2); yt = branin(Xt) -sd(v) = sqrt(sum((v .- sum(v) / length(v)) .^ 2) / (length(v) - 1)) -finite_count(v...) = count(!isfinite, vcat(v...)) - -function check_runs!(f, s, check) - try - f() - catch e - add_check!(s, check, false; value=string(typeof(e)), expected="no error", detail=first(sprint(showerror, e), 200)) - end -end - -function roundtrip!(s, label, k, Xn) - check_runs!(s, "$label: save + load") do - f = joinpath(mktempdir(), "k.json") - LK.save(k, f) - k2 = LK.load(f) - check_close!(s, "$label: load(save(k)) predicts the same mean", - LK.predict(k2, Xn; return_stdev=false).mean, LK.predict(k, Xn; return_stdev=false).mean; rtol=1e-10) - end -end - -# %% [markdown] -# ## `MLPKriging`: a joint feature map ℝ² → ℝ² before the kernel - -# %% -check_runs!(s, "MLPKriging: fit + predict") do - km = LK.MLPKriging(y, X, [8], 2; activation="tanh", kernel="matern5_2", optim="BFGS10") - pm = LK.predict(km, Xt; return_stdev=true) - add_check!(s, "MLPKriging: finite predictions", finite_count(pm.mean, pm.stdev) == 0; - value=finite_count(pm.mean, pm.stdev), expected="0 non-finite") - add_metric!(s, "branin", "MLPKriging tanh [8] -> 2, BFGS10"; q2=q2(yt, pm.mean)) - add_check!(s, "MLPKriging: Q2", q2(yt, pm.mean) > 0.9; value=q2(yt, pm.mean), expected="> 0.9", severity="warn") - roundtrip!(s, "MLPKriging", km, Xt) - kd = LK.MLPKriging(y, X, [8], 2; activation="tanh", kernel="matern5_2") # default optimizer - qd = q2(yt, LK.predict(kd, Xt; return_stdev=false).mean) - add_metric!(s, "branin", "MLPKriging tanh [8] -> 2, default optim"; q2=qd) - add_check!(s, "MLPKriging: Q2(default optim) - Q2(BFGS10)", qd - q2(yt, pm.mean) > -0.05; - value=qd - q2(yt, pm.mean), expected="> -0.05", severity="warn") -end - -# %% -foreach(r -> r[5] == "q2" && println(rpad(r[3], 24), rpad(r[4], 28), r[6]), s.metrics) - -# %% -finalize!(s) diff --git a/tools/src/deep-kernel_MLPKriging_octave.m b/tools/src/deep-kernel_MLPKriging_octave.m deleted file mode 100644 index 46476ac..0000000 --- a/tools/src/deep-kernel_MLPKriging_octave.m +++ /dev/null @@ -1,77 +0,0 @@ -% %% [markdown] -% # Octave · `MLPKriging` — a joint feature map ℝ² → ℝ² before the kernel -% -% Branin on [0,1]² (n = 30 LHS points unless stated, Q² on 500 uniform test -% points), through mLibKriging. Checks (error-level unless stated): -% -% * 10 restarts (`BFGS10`): finite predictions, Q² > 0.9 (warn-level) -% * `save` then `load`: the same predicted mean -% * the default optimizer as good as `BFGS10` (warn-level; see libKriging/libKriging#382) -% -% The same checks as `deep-kernel_MLPKriging_R.ipynb`, with the Octave binding. -% -% A class missing from the installed mLibKriging (release channel) is recorded as SKIP. - -% %% -addpath(fullfile(pwd, "tools", "octave")); -s = lkb_suite("deep-kernel_MLPKriging_octave"); -q2 = @(y, m) 1 - sum((y - m).^2) / sum((y - mean(y)).^2); -branin = @(U) ((15 * U(:, 2)) - 5.1 / (4 * pi^2) * (15 * U(:, 1) - 5).^2 + 5 / pi * (15 * U(:, 1) - 5) - 6).^2 ... - + 10 * (1 - 1 / (8 * pi)) * cos(15 * U(:, 1) - 5) + 10; -rand("seed", 1); randn("seed", 1); -function U = lhs(n, d) - U = zeros(n, d); - for j = 1:d, U(:, j) = (randperm(n)' - rand(n, 1)) / n; end -end -X = lhs(30, 2); y = branin(X); -Xt = rand(500, 2); yt = branin(Xt); -tmp = tempname(); mkdir(tmp); -has_class = @(name) exist(name) == 2; % a class file on the path - -function s = roundtrip(s, label, k, Xn, loader) - f = fullfile(tempdir(), [regexprep(label, "[^A-Za-z0-9]", "_") ".json"]); - try - k.save(f); - k2 = loader(f); - m1 = k.predict(Xn, false, false, false); - m2 = k2.predict(Xn, false, false, false); - s = lkb_close(s, [label ": load(save(k)) predicts the same mean"], m2, m1, 1e-10); - catch err - s = lkb_check(s, [label ": save + load"], false, "error", "no error", err.message); - end -end - -function s = skip(s, check, reason) - s.checks(end + 1, :) = {s.name, "octave", check, "info", "SKIP", "", "", reason}; - printf("[SKIP] %s: %s\n", check, reason); -end - -% %% [markdown] -% ## `MLPKriging`: a joint feature map ℝ² → ℝ² before the kernel - -% %% -if has_class("MLPKriging") - try - km = MLPKriging(y, X, [8], 2, "tanh", "matern5_2", "constant", false, "BFGS10"); - [mm, sm] = km.predict(Xt, true, false, false); - s = lkb_check(s, "MLPKriging: finite predictions", all(isfinite([mm; sm])), sum(~isfinite([mm; sm])), "0 non-finite"); - s = lkb_metric(s, "branin", "MLPKriging tanh [8] -> 2, BFGS10", "q2", q2(yt, mm)); - s = lkb_check(s, "MLPKriging: Q2", q2(yt, mm) > 0.9, q2(yt, mm), "> 0.9", "", "warn"); - s = roundtrip(s, "MLPKriging", km, Xt, @load_kriging); - kd = MLPKriging(y, X, [8], 2, "tanh", "matern5_2"); % default optimizer - qd = q2(yt, kd.predict(Xt, false, false, false)); - s = lkb_metric(s, "branin", "MLPKriging tanh [8] -> 2, default optim", "q2", qd); - s = lkb_check(s, "MLPKriging: Q2(default optim) - Q2(BFGS10)", qd - q2(yt, mm) > -0.05, qd - q2(yt, mm), "> -0.05", "", "warn"); - catch err - s = lkb_check(s, "MLPKriging: fit + predict", false, "error", "no error", err.message); - end -else - s = skip(s, "MLPKriging", "not in this mLibKriging"); -end - -% %% -q = strcmp(s.metrics(:, 5), "q2"); -disp(s.metrics(q, [3, 4, 6])) - -% %% -lkb_finalize(s); diff --git a/tools/src/high-dimension-normalize-screening_Kriging_python.py b/tools/src/high-dimension-normalize-screening_Kriging_python.py index 0282456..e332dd0 100644 --- a/tools/src/high-dimension-normalize-screening_Kriging_python.py +++ b/tools/src/high-dimension-normalize-screening_Kriging_python.py @@ -25,6 +25,7 @@ # %% import time +import matplotlib.pyplot as plt import numpy as np import pandas as pd import pylibkriging as lk @@ -88,6 +89,11 @@ def pred(k, X): active = [0, 3, 4, 11, 18, 19] # x1, x4, x5, x12, x19, x20 suite.greater("welch: min theta(inactive) / max theta(active)", theta[inactive].min() / theta[active].max(), 1.0, severity="warn") +colors = ["tab:red" if i in active else "tab:gray" if i in inactive else "tab:blue" for i in range(20)] +plt.figure(figsize=(9, 3)) +plt.bar([f"x{i + 1}" for i in range(20)], theta, color=colors) +plt.yscale("log"); plt.ylabel("fitted θ (log)") +plt.title("Welch (20D): red = most influential inputs, gray = inactive (x8, x16)"); plt.show() pd.Series(theta, index=[f"x{i + 1}" for i in range(20)]).round(3).to_frame("theta").T # %% diff --git a/tools/src/interpolation-simulate-update-save_Kriging_R.R b/tools/src/interpolation-simulate-update-save_Kriging_R.R deleted file mode 100644 index 8a3726b..0000000 --- a/tools/src/interpolation-simulate-update-save_Kriging_R.R +++ /dev/null @@ -1,76 +0,0 @@ -# %% [markdown] -# # R · `Kriging`, noise-free — interpolation, conditional simulation, update, save/load -# -# Branin on [0,1]² (n = 30 LHS points unless stated, Q² on 500 uniform test -# points), through rlibkriging. Checks (error-level unless stated): -# -# * interpolation (max |m(X) − y| / sd(y) < 1e-6) and Q² > 0.95 on 500 test points -# * conditional simulations through the data (relative to σ) -# * stdev > 0 off the design -# * `update` without refit = a fit on all the points at the same θ, σ² -# * `save` then `load`: the same predicted mean - -# %% -suppressPackageStartupMessages(library(rlibkriging)) -source("tools/r/checks.R") -s <- new_suite("interpolation-simulate-update-save_Kriging_R") -cat("rlibkriging", as.character(packageVersion("rlibkriging")), "\n") - -branin <- function(U) { # on [0,1]^2 - x1 <- 15 * U[, 1] - 5; x2 <- 15 * U[, 2] - (x2 - 5.1 / (4 * pi^2) * x1^2 + 5 / pi * x1 - 6)^2 + 10 * (1 - 1 / (8 * pi)) * cos(x1) + 10 -} -set.seed(1) -lhs <- function(n, d) sapply(1:d, function(j) (sample(n) - runif(n)) / n) -X <- lhs(30, 2); y <- branin(X) -Xt <- matrix(runif(1000), ncol = 2); yt <- branin(Xt) -tmp <- tempdir() -has_class <- function(name) exists(name, mode = "function") -skip <- function(s, check, reason) { - s$checks[[length(s$checks) + 1]] <- data.frame(suite = s$name, lang = "R", check = check, severity = "info", - status = "SKIP", value = "", expected = "", detail = reason, stringsAsFactors = FALSE) - message("[SKIP] ", check, ": ", reason) -} -roundtrip_same_mean <- function(s, label, k, Xn) { - f <- file.path(tmp, paste0(gsub("[^A-Za-z0-9]", "_", label), ".json")) - check_runs(s, paste0(label, ": save + load"), { - save(k, f) - k2 <- rlibkriging::load(f) - check_close(s, paste0(label, ": load(save(k)) predicts the same mean"), - predict(k2, Xn, return_stdev = FALSE)$mean, predict(k, Xn, return_stdev = FALSE)$mean, rtol = 1e-10) - }) -} - -# %% [markdown] -# ## `Kriging`, noise-free - -# %% -k <- Kriging(y, X, "matern5_2") -print(k) -p <- predict(k, Xt, return_stdev = TRUE) -add_metric(s, "branin", "Kriging", q2 = q2(yt, p$mean), rmse = rmse(yt, p$mean)) -check_greater(s, "Kriging: Q2", q2(yt, p$mean), 0.95) -check_less(s, "Kriging: interpolates (max |m(X) - y| / sd(y))", - max(abs(predict(k, X, return_stdev = FALSE)$mean - y)) / sd(y), 1e-6) -sims <- simulate(k, nsim = 5, seed = 2, x = X) -check_less(s, "Kriging: conditional simulations go through the data (max |sim(X) - y| / sigma)", - max(abs(sims - y)) / sqrt(k$sigma2()), 1e-4) -check_greater(s, "Kriging: stdev > 0 off the design", min(p$stdev), 0) -# update with 5 new points (no refit) = a fit on the 35 points at the same theta -Xu <- lhs(5, 2); yu <- branin(Xu) -ku <- copy(k) -update(ku, yu, Xu, refit = FALSE) -k35 <- Kriging(c(y, yu), rbind(X, Xu), "matern5_2", optim = "none", - parameters = list(theta = matrix(k$theta(), nrow = 1), is_theta_estim = FALSE, - sigma2 = k$sigma2(), is_sigma2_estim = FALSE)) -check_close(s, "Kriging: update(refit = FALSE) == fit on all points (same theta, sigma2)", - predict(ku, Xt, return_stdev = FALSE)$mean, predict(k35, Xt, return_stdev = FALSE)$mean, - rtol = 1e-6, atol = 1e-8 * sd(y)) -roundtrip_same_mean(s, "Kriging", k, Xt) - -# %% -me <- do.call(rbind, s$metrics) -print(me[me$metric == "q2", c("case", "package", "value")], row.names = FALSE) - -# %% -finalize(s) diff --git a/tools/src/interpolation-simulate-update-save_Kriging_julia.jl b/tools/src/interpolation-simulate-update-save_Kriging_julia.jl deleted file mode 100644 index 6ea1743..0000000 --- a/tools/src/interpolation-simulate-update-save_Kriging_julia.jl +++ /dev/null @@ -1,79 +0,0 @@ -# %% [markdown] -# # Julia · `Kriging`, noise-free — interpolation, conditional simulation, update, save/load -# -# Branin on [0,1]² (n = 30 LHS points unless stated, Q² on 500 uniform test -# points), through JLibKriging. Checks (error-level unless stated): -# -# * interpolation (max |m(X) − y| / sd(y) < 1e-6) and Q² > 0.95 on 500 test points -# * conditional simulations through the data (relative to σ) -# * stdev > 0 off the design -# * `update` without refit = a fit on all the points at the same θ, σ² -# * `save` then `load`: the same predicted mean -# -# The same checks as `interpolation-simulate-update-save_Kriging_R.ipynb`, with the Julia binding. - -# %% -import JLibKriging as LK -using Random -include("tools/julia/checks.jl") -s = Suite("interpolation-simulate-update-save_Kriging_julia") - -branin(U) = begin - x1 = 15 .* U[:, 1] .- 5; x2 = 15 .* U[:, 2] - (x2 .- 5.1 / (4π^2) .* x1 .^ 2 .+ 5 / π .* x1 .- 6) .^ 2 .+ 10 * (1 - 1 / (8π)) .* cos.(x1) .+ 10 -end -rng = MersenneTwister(1) -lhs(n, d) = hcat([(randperm(rng, n) .- rand(rng, n)) ./ n for _ in 1:d]...) -X = lhs(30, 2); y = branin(X) -Xt = rand(rng, 500, 2); yt = branin(Xt) -sd(v) = sqrt(sum((v .- sum(v) / length(v)) .^ 2) / (length(v) - 1)) -finite_count(v...) = count(!isfinite, vcat(v...)) - -function check_runs!(f, s, check) - try - f() - catch e - add_check!(s, check, false; value=string(typeof(e)), expected="no error", detail=first(sprint(showerror, e), 200)) - end -end - -function roundtrip!(s, label, k, Xn) - check_runs!(s, "$label: save + load") do - f = joinpath(mktempdir(), "k.json") - LK.save(k, f) - k2 = LK.load(f) - check_close!(s, "$label: load(save(k)) predicts the same mean", - LK.predict(k2, Xn; return_stdev=false).mean, LK.predict(k, Xn; return_stdev=false).mean; rtol=1e-10) - end -end - -# %% [markdown] -# ## `Kriging`, noise-free - -# %% -k = LK.Kriging(y, X, "matern5_2") -p = LK.predict(k, Xt; return_stdev=true) -add_metric!(s, "branin", "Kriging"; q2=q2(yt, p.mean)) -add_check!(s, "Kriging: Q2", q2(yt, p.mean) > 0.95; value=q2(yt, p.mean), expected="> 0.95") -e = maximum(abs.(LK.predict(k, X; return_stdev=false).mean .- y)) / sd(y) -add_check!(s, "Kriging: interpolates (max |m(X) - y| / sd(y))", e < 1e-6; value=e, expected="< 1e-6") -S = LK.simulate(k, 5, 2, X) -e = maximum(abs.(S .- y)) / sqrt(LK.sigma2(k)) -add_check!(s, "Kriging: conditional simulations go through the data (max |sim(X) - y| / sigma)", e < 1e-4; - value=e, expected="< 1e-4") -add_check!(s, "Kriging: stdev > 0 off the design", minimum(p.stdev) > 0; value=minimum(p.stdev), expected="> 0") -Xu = lhs(5, 2); yu = branin(Xu) -ku = copy(k) -LK.update!(ku, yu, Xu; refit=false) -k35 = LK.Kriging(vcat(y, yu), vcat(X, Xu), "matern5_2"; optim="none", - theta=vec(LK.theta(k)), is_theta_estim=false, sigma2=LK.sigma2(k), is_sigma2_estim=false) -check_close!(s, "Kriging: update!(refit=false) == fit on all points (same theta, sigma2)", - LK.predict(ku, Xt; return_stdev=false).mean, LK.predict(k35, Xt; return_stdev=false).mean; - rtol=1e-6, atol=1e-8 * sd(y)) -roundtrip!(s, "Kriging", k, Xt) - -# %% -foreach(r -> r[5] == "q2" && println(rpad(r[3], 24), rpad(r[4], 28), r[6]), s.metrics) - -# %% -finalize!(s) diff --git a/tools/src/interpolation-simulate-update-save_Kriging_octave.m b/tools/src/interpolation-simulate-update-save_Kriging_octave.m deleted file mode 100644 index bad7144..0000000 --- a/tools/src/interpolation-simulate-update-save_Kriging_octave.m +++ /dev/null @@ -1,79 +0,0 @@ -% %% [markdown] -% # Octave · `Kriging`, noise-free — interpolation, conditional simulation, update, save/load -% -% Branin on [0,1]² (n = 30 LHS points unless stated, Q² on 500 uniform test -% points), through mLibKriging. Checks (error-level unless stated): -% -% * interpolation (max |m(X) − y| / sd(y) < 1e-6) and Q² > 0.95 on 500 test points -% * conditional simulations through the data (relative to σ) -% * stdev > 0 off the design -% * `update` without refit = a fit on all the points at the same θ, σ² -% * `save` then `load`: the same predicted mean -% -% The same checks as `interpolation-simulate-update-save_Kriging_R.ipynb`, with the Octave binding. - -% %% -addpath(fullfile(pwd, "tools", "octave")); -s = lkb_suite("interpolation-simulate-update-save_Kriging_octave"); -q2 = @(y, m) 1 - sum((y - m).^2) / sum((y - mean(y)).^2); -branin = @(U) ((15 * U(:, 2)) - 5.1 / (4 * pi^2) * (15 * U(:, 1) - 5).^2 + 5 / pi * (15 * U(:, 1) - 5) - 6).^2 ... - + 10 * (1 - 1 / (8 * pi)) * cos(15 * U(:, 1) - 5) + 10; -rand("seed", 1); randn("seed", 1); -function U = lhs(n, d) - U = zeros(n, d); - for j = 1:d, U(:, j) = (randperm(n)' - rand(n, 1)) / n; end -end -X = lhs(30, 2); y = branin(X); -Xt = rand(500, 2); yt = branin(Xt); -tmp = tempname(); mkdir(tmp); -has_class = @(name) exist(name) == 2; % a class file on the path - -function s = roundtrip(s, label, k, Xn, loader) - f = fullfile(tempdir(), [regexprep(label, "[^A-Za-z0-9]", "_") ".json"]); - try - k.save(f); - k2 = loader(f); - m1 = k.predict(Xn, false, false, false); - m2 = k2.predict(Xn, false, false, false); - s = lkb_close(s, [label ": load(save(k)) predicts the same mean"], m2, m1, 1e-10); - catch err - s = lkb_check(s, [label ": save + load"], false, "error", "no error", err.message); - end -end - -function s = skip(s, check, reason) - s.checks(end + 1, :) = {s.name, "octave", check, "info", "SKIP", "", "", reason}; - printf("[SKIP] %s: %s\n", check, reason); -end - -% %% [markdown] -% ## `Kriging`, noise-free - -% %% -k = Kriging(y, X, "matern5_2"); -[m, sd] = k.predict(Xt, true, false, false); -s = lkb_metric(s, "branin", "Kriging", "q2", q2(yt, m)); -s = lkb_check(s, "Kriging: Q2", q2(yt, m) > 0.95, q2(yt, m), "> 0.95"); -mX = k.predict(X, false, false, false); -e = max(abs(mX - y)) / std(y); -s = lkb_check(s, "Kriging: interpolates (max |m(X) - y| / sd(y))", e < 1e-6, e, "< 1e-6"); -S = k.simulate(int32(5), int32(2), X, false); -e = max(max(abs(S - y))) / sqrt(k.sigma2()); -s = lkb_check(s, "Kriging: conditional simulations go through the data (max |sim(X) - y| / sigma)", e < 1e-4, e, "< 1e-4"); -s = lkb_check(s, "Kriging: stdev > 0 off the design", min(sd) > 0, min(sd), "> 0"); -Xu = lhs(5, 2); yu = branin(Xu); -ku = k.copy(); -ku.update(yu, Xu, false); -th = k.theta(); -k35 = Kriging([y; yu], [X; Xu], "matern5_2", "constant", false, "none", "LL", ... - Params("theta", th(:)', "is_theta_estim", false, "sigma2", k.sigma2(), "is_sigma2_estim", false)); -s = lkb_close(s, "Kriging: update(refit = false) == fit on all points (same theta, sigma2)", ... - ku.predict(Xt, false, false, false), k35.predict(Xt, false, false, false), 1e-6, 1e-8 * std(y)); -s = roundtrip(s, "Kriging", k, Xt, @load_kriging); - -% %% -q = strcmp(s.metrics(:, 5), "q2"); -disp(s.metrics(q, [3, 4, 6])) - -% %% -lkb_finalize(s); diff --git a/tools/src/large-n_Kriging-vecchia-nystrom_R.R b/tools/src/large-n_Kriging-vecchia-nystrom_R.R deleted file mode 100644 index bd288c7..0000000 --- a/tools/src/large-n_Kriging-vecchia-nystrom_R.R +++ /dev/null @@ -1,65 +0,0 @@ -# %% [markdown] -# # R · `Kriging` with an approximate fit — Vecchia, Nyström (n = 400) -# -# Branin on [0,1]² (n = 30 LHS points unless stated, Q² on 500 uniform test -# points), through rlibkriging. Checks (error-level unless stated): -# -# * the fit method is reported by `method()` -# * `predict` with the fit method: finite mean and stdev -# * Q² > 0.95 (warn-level) - -# %% -suppressPackageStartupMessages(library(rlibkriging)) -source("tools/r/checks.R") -s <- new_suite("large-n_Kriging-vecchia-nystrom_R") -cat("rlibkriging", as.character(packageVersion("rlibkriging")), "\n") - -branin <- function(U) { # on [0,1]^2 - x1 <- 15 * U[, 1] - 5; x2 <- 15 * U[, 2] - (x2 - 5.1 / (4 * pi^2) * x1^2 + 5 / pi * x1 - 6)^2 + 10 * (1 - 1 / (8 * pi)) * cos(x1) + 10 -} -set.seed(1) -lhs <- function(n, d) sapply(1:d, function(j) (sample(n) - runif(n)) / n) -X <- lhs(30, 2); y <- branin(X) -Xt <- matrix(runif(1000), ncol = 2); yt <- branin(Xt) -tmp <- tempdir() -has_class <- function(name) exists(name, mode = "function") -skip <- function(s, check, reason) { - s$checks[[length(s$checks) + 1]] <- data.frame(suite = s$name, lang = "R", check = check, severity = "info", - status = "SKIP", value = "", expected = "", detail = reason, stringsAsFactors = FALSE) - message("[SKIP] ", check, ": ", reason) -} -roundtrip_same_mean <- function(s, label, k, Xn) { - f <- file.path(tmp, paste0(gsub("[^A-Za-z0-9]", "_", label), ".json")) - check_runs(s, paste0(label, ": save + load"), { - save(k, f) - k2 <- rlibkriging::load(f) - check_close(s, paste0(label, ": load(save(k)) predicts the same mean"), - predict(k2, Xn, return_stdev = FALSE)$mean, predict(k, Xn, return_stdev = FALSE)$mean, rtol = 1e-10) - }) -} - -# %% [markdown] -# ## `Kriging` with an approximate fit: Vecchia, Nyström (n = 400) - -# %% -Xl <- lhs(400, 2); yl <- branin(Xl) -for (m in c("vecchia(20)", "nystrom(60)")) { - short <- sub("\\(.*", "", m) - check_runs(s, paste0("Kriging(method = '", m, "'): fit + predict"), { - ka <- Kriging(yl, Xl, "matern5_2", method = m) - add_check(s, paste0("Kriging(method = '", m, "'): method()"), identical(ka$method(), m), ka$method(), m) - pa <- predict(ka, Xt, return_stdev = TRUE, method = short) - check_less(s, paste0("Kriging(method = '", m, "'): non-finite predictions"), - sum(!is.finite(c(pa$mean, pa$stdev))), 1) - add_metric(s, "branin n=400", paste("Kriging", m), q2 = q2(yt, pa$mean)) - check_greater(s, paste0("Kriging(method = '", m, "'): Q2"), q2(yt, pa$mean), 0.95, severity = "warn") - }) -} - -# %% -me <- do.call(rbind, s$metrics) -print(me[me$metric == "q2", c("case", "package", "value")], row.names = FALSE) - -# %% -finalize(s) diff --git a/tools/src/large-n_Kriging-vecchia-nystrom_julia.jl b/tools/src/large-n_Kriging-vecchia-nystrom_julia.jl deleted file mode 100644 index 71cfd59..0000000 --- a/tools/src/large-n_Kriging-vecchia-nystrom_julia.jl +++ /dev/null @@ -1,71 +0,0 @@ -# %% [markdown] -# # Julia · `Kriging` with an approximate fit — Vecchia, Nyström (n = 400) -# -# Branin on [0,1]² (n = 30 LHS points unless stated, Q² on 500 uniform test -# points), through JLibKriging. Checks (error-level unless stated): -# -# * the fit method is reported by `method()` -# * `predict` with the fit method: finite mean and stdev -# * Q² > 0.95 (warn-level) -# -# The same checks as `large-n_Kriging-vecchia-nystrom_R.ipynb`, with the Julia binding. - -# %% -import JLibKriging as LK -using Random -include("tools/julia/checks.jl") -s = Suite("large-n_Kriging-vecchia-nystrom_julia") - -branin(U) = begin - x1 = 15 .* U[:, 1] .- 5; x2 = 15 .* U[:, 2] - (x2 .- 5.1 / (4π^2) .* x1 .^ 2 .+ 5 / π .* x1 .- 6) .^ 2 .+ 10 * (1 - 1 / (8π)) .* cos.(x1) .+ 10 -end -rng = MersenneTwister(1) -lhs(n, d) = hcat([(randperm(rng, n) .- rand(rng, n)) ./ n for _ in 1:d]...) -X = lhs(30, 2); y = branin(X) -Xt = rand(rng, 500, 2); yt = branin(Xt) -sd(v) = sqrt(sum((v .- sum(v) / length(v)) .^ 2) / (length(v) - 1)) -finite_count(v...) = count(!isfinite, vcat(v...)) - -function check_runs!(f, s, check) - try - f() - catch e - add_check!(s, check, false; value=string(typeof(e)), expected="no error", detail=first(sprint(showerror, e), 200)) - end -end - -function roundtrip!(s, label, k, Xn) - check_runs!(s, "$label: save + load") do - f = joinpath(mktempdir(), "k.json") - LK.save(k, f) - k2 = LK.load(f) - check_close!(s, "$label: load(save(k)) predicts the same mean", - LK.predict(k2, Xn; return_stdev=false).mean, LK.predict(k, Xn; return_stdev=false).mean; rtol=1e-10) - end -end - -# %% [markdown] -# ## `Kriging` with an approximate fit: Vecchia, Nyström (n = 400) - -# %% -Xl = lhs(400, 2); yl = branin(Xl) -for m in ("vecchia(20)", "nystrom(60)") - short = split(m, "(")[1] - label = "Kriging(method = '$m')" - check_runs!(s, "$label: fit + predict") do - ka = LK.Kriging(yl, Xl, "matern5_2"; method=m) - add_check!(s, "$label: method()", LK.method(ka) == m; value=LK.method(ka), expected=m) - pa = LK.predict(ka, Xt; return_stdev=true, method=String(short)) - add_check!(s, "$label: finite predictions", finite_count(pa.mean, pa.stdev) == 0; - value=finite_count(pa.mean, pa.stdev), expected="0 non-finite") - add_metric!(s, "branin n=400", "Kriging $m"; q2=q2(yt, pa.mean)) - add_check!(s, "$label: Q2", q2(yt, pa.mean) > 0.95; value=q2(yt, pa.mean), expected="> 0.95", severity="warn") - end -end - -# %% -foreach(r -> r[5] == "q2" && println(rpad(r[3], 24), rpad(r[4], 28), r[6]), s.metrics) - -# %% -finalize!(s) diff --git a/tools/src/large-n_Kriging-vecchia-nystrom_octave.m b/tools/src/large-n_Kriging-vecchia-nystrom_octave.m deleted file mode 100644 index b3c4521..0000000 --- a/tools/src/large-n_Kriging-vecchia-nystrom_octave.m +++ /dev/null @@ -1,73 +0,0 @@ -% %% [markdown] -% # Octave · `Kriging` with an approximate fit — Vecchia, Nyström (n = 400) -% -% Branin on [0,1]² (n = 30 LHS points unless stated, Q² on 500 uniform test -% points), through mLibKriging. Checks (error-level unless stated): -% -% * the fit method is reported by `method()` -% * `predict` with the fit method: finite mean and stdev -% * Q² > 0.95 (warn-level) -% -% The same checks as `large-n_Kriging-vecchia-nystrom_R.ipynb`, with the Octave binding. - -% %% -addpath(fullfile(pwd, "tools", "octave")); -s = lkb_suite("large-n_Kriging-vecchia-nystrom_octave"); -q2 = @(y, m) 1 - sum((y - m).^2) / sum((y - mean(y)).^2); -branin = @(U) ((15 * U(:, 2)) - 5.1 / (4 * pi^2) * (15 * U(:, 1) - 5).^2 + 5 / pi * (15 * U(:, 1) - 5) - 6).^2 ... - + 10 * (1 - 1 / (8 * pi)) * cos(15 * U(:, 1) - 5) + 10; -rand("seed", 1); randn("seed", 1); -function U = lhs(n, d) - U = zeros(n, d); - for j = 1:d, U(:, j) = (randperm(n)' - rand(n, 1)) / n; end -end -X = lhs(30, 2); y = branin(X); -Xt = rand(500, 2); yt = branin(Xt); -tmp = tempname(); mkdir(tmp); -has_class = @(name) exist(name) == 2; % a class file on the path - -function s = roundtrip(s, label, k, Xn, loader) - f = fullfile(tempdir(), [regexprep(label, "[^A-Za-z0-9]", "_") ".json"]); - try - k.save(f); - k2 = loader(f); - m1 = k.predict(Xn, false, false, false); - m2 = k2.predict(Xn, false, false, false); - s = lkb_close(s, [label ": load(save(k)) predicts the same mean"], m2, m1, 1e-10); - catch err - s = lkb_check(s, [label ": save + load"], false, "error", "no error", err.message); - end -end - -function s = skip(s, check, reason) - s.checks(end + 1, :) = {s.name, "octave", check, "info", "SKIP", "", "", reason}; - printf("[SKIP] %s: %s\n", check, reason); -end - -% %% [markdown] -% ## `Kriging` with an approximate fit: Vecchia, Nyström (n = 400) - -% %% -Xl = lhs(400, 2); yl = branin(Xl); -for mth = {"vecchia(20)", "nystrom(60)"} - mth = mth{1}; short = regexprep(mth, "\\(.*", ""); - label = ["Kriging(method = '" mth "')"]; - try - ka = Kriging(yl, Xl, "matern5_2", "constant", false, "BFGS", "LL", [], "none", [], mth); - got = ka.method(); - s = lkb_check(s, [label ": method()"], strcmp(got, mth), got, mth); - [ma, sa] = ka.predict(Xt, true, false, false, short); - s = lkb_check(s, [label ": finite predictions"], all(isfinite([ma; sa])), sum(~isfinite([ma; sa])), "0 non-finite"); - s = lkb_metric(s, "branin n=400", ["Kriging " mth], "q2", q2(yt, ma)); - s = lkb_check(s, [label ": Q2"], q2(yt, ma) > 0.95, q2(yt, ma), "> 0.95", "", "warn"); - catch err - s = lkb_check(s, [label ": fit + predict"], false, "error", "no error", err.message); - end -end - -% %% -q = strcmp(s.metrics(:, 5), "q2"); -disp(s.metrics(q, [3, 4, 6])) - -% %% -lkb_finalize(s); diff --git a/tools/src/large-n_Kriging-vecchia-nystrom_python.py b/tools/src/large-n_Kriging-vecchia-nystrom_python.py index f5e0b17..71fde68 100644 --- a/tools/src/large-n_Kriging-vecchia-nystrom_python.py +++ b/tools/src/large-n_Kriging-vecchia-nystrom_python.py @@ -17,6 +17,10 @@ # for reference) and n = 5000 (approximations only). # # Checks: +# * **convergence to the exact model** at fixed θ (Hartmann 3, n = 300): the gap +# between the approximate and the exact log-likelihood and predicted mean, +# as m (Vecchia) and k (Nyström) grow, down to the limit cases m = n − 1 and +# k = n where the approximation *is* the exact model; # * **limit cases**: Vecchia with m = n − 1 neighbours *is* the exact likelihood; # * the method is explicit at prediction time: `predict(method="exact")` on a # Nyström/Vecchia fit raises, `method=` gives finite predictions; @@ -26,6 +30,7 @@ # %% import time +import matplotlib.pyplot as plt import numpy as np import pandas as pd import pylibkriging as lk @@ -73,6 +78,47 @@ def timed(f): suite.true("method() reports the fit method", kv.method().startswith("vecchia") and kn.method().startswith("nystrom"), detail=f"{kv.method()}, {kn.method()}") +# %% [markdown] +# ## Convergence to the exact model, at fixed θ +# +# Hartmann 3, n = 300, θ of the exact fit kept fixed: the reference is the exact +# model with the same hyperparameters, so the gap measures the approximation +# alone (no optimizer effect). Relative gap of the log-likelihood, and RMS gap +# of the predicted mean on 500 test points relative to its standard deviation. + +# %% +if HAS_METHOD: + tf = TESTS["hartmann3"] + X, y, Xt, yt = design.dataset(tf, 300, n_test=500, seed=54) + n = len(y) + theta = np.asarray(lk.Kriging(y, X, "matern5_2").theta()).reshape(1, -1) + fixed = {"theta": theta, "is_theta_estim": False} + ke = lk.Kriging(y, X, "matern5_2", optim="none", parameters=fixed) + ll_e, m_e = ke.logLikelihood(), pred(ke, Xt)[0] + rows = [] + for meth, sizes in [("vecchia", [2, 5, 10, 20, 40, 80, 160, n - 1]), ("nystrom", [10, 25, 50, 100, 200, n])]: + for size in sizes: + with suite.runs(f"{meth}({size}) at fixed theta: fit + predict"): + ka = lk.Kriging(y, X, "matern5_2", optim="none", parameters=fixed, method=f"{meth}({size})") + rows.append(dict(method=meth, size=size, + ll_gap=abs(ka.logLikelihood(method=meth) - ll_e) / abs(ll_e), + mean_gap=np.sqrt(np.mean((pred(ka, Xt, method=meth)[0] - m_e) ** 2)) / np.std(m_e))) + conv = pd.DataFrame(rows) + for meth, g in conv.groupby("method"): + suite.metric(f"hartmann3/n={n}/fixed theta", meth, **{f"ll_gap_{s}": v for s, v in zip(g["size"], g["ll_gap"])}) + for col, lim in [("ll_gap", 1e-6), ("mean_gap", 1e-2)]: + suite.less(f"{meth}: {col} at the limit size ({g['size'].iloc[-1]})", g[col].iloc[-1], lim) + suite.true(f"{meth}: {col} decreases with the size", bool(np.all(np.diff(g[col].to_numpy()) < 0)), + detail=", ".join(f"{v:.2g}" for v in g[col]), severity="warn") + fig, axes = plt.subplots(1, 2, figsize=(10, 3.8)) + for ax, col, label in [(axes[0], "ll_gap", "|LL − LL(exact)| / |LL(exact)|"), + (axes[1], "mean_gap", "RMS(mean − mean(exact)) / sd")]: + for meth, g in conv.groupby("method"): + ax.loglog(g["size"], g[col], "o-", label=f"{meth}(m)" if meth == "vecchia" else f"{meth}(k)") + ax.set_xlabel("m (neighbours) or k (landmarks)"); ax.set_ylabel(label); ax.grid(True, which="both", alpha=.3) + axes[0].legend(); fig.suptitle(f"Hartmann 3, n = {n}, θ fixed: approximation → exact model"); plt.show() + display(conv.pivot(index="size", columns="method", values=["ll_gap", "mean_gap"]).round(6)) + # %% [markdown] # ## Accuracy and cost diff --git a/tools/src/noisy-data_Kriging-nugget-noise_R.R b/tools/src/noisy-data_Kriging-nugget-noise_R.R deleted file mode 100644 index be4a553..0000000 --- a/tools/src/noisy-data_Kriging-nugget-noise_R.R +++ /dev/null @@ -1,82 +0,0 @@ -# %% [markdown] -# # R · `Kriging` with noise — estimated nugget, known variances -# -# Branin on [0,1]² (n = 30 LHS points unless stated, Q² on 500 uniform test -# points), through rlibkriging. Checks (error-level unless stated): -# -# * `noise = "nugget"` (10 restarts): nugget > 0, Q² > 0.95 on the noise-free function, smoothing next to the design points -# * the default single start reaches the multistart likelihood (warn-level) -# * `save` then `load`: the same predicted mean -# * `noise = `: the stored variances, finite predictions - -# %% -suppressPackageStartupMessages(library(rlibkriging)) -source("tools/r/checks.R") -s <- new_suite("noisy-data_Kriging-nugget-noise_R") -cat("rlibkriging", as.character(packageVersion("rlibkriging")), "\n") - -branin <- function(U) { # on [0,1]^2 - x1 <- 15 * U[, 1] - 5; x2 <- 15 * U[, 2] - (x2 - 5.1 / (4 * pi^2) * x1^2 + 5 / pi * x1 - 6)^2 + 10 * (1 - 1 / (8 * pi)) * cos(x1) + 10 -} -set.seed(1) -lhs <- function(n, d) sapply(1:d, function(j) (sample(n) - runif(n)) / n) -X <- lhs(30, 2); y <- branin(X) -Xt <- matrix(runif(1000), ncol = 2); yt <- branin(Xt) -tmp <- tempdir() -has_class <- function(name) exists(name, mode = "function") -skip <- function(s, check, reason) { - s$checks[[length(s$checks) + 1]] <- data.frame(suite = s$name, lang = "R", check = check, severity = "info", - status = "SKIP", value = "", expected = "", detail = reason, stringsAsFactors = FALSE) - message("[SKIP] ", check, ": ", reason) -} -roundtrip_same_mean <- function(s, label, k, Xn) { - f <- file.path(tmp, paste0(gsub("[^A-Za-z0-9]", "_", label), ".json")) - check_runs(s, paste0(label, ": save + load"), { - save(k, f) - k2 <- rlibkriging::load(f) - check_close(s, paste0(label, ": load(save(k)) predicts the same mean"), - predict(k2, Xn, return_stdev = FALSE)$mean, predict(k, Xn, return_stdev = FALSE)$mean, rtol = 1e-10) - }) -} - -# %% [markdown] -# ## `Kriging` with noise: estimated nugget, known variances -# -# Noise of standard deviation 5% of sd(y). The nugget model is fitted with 10 -# restarts (`optim = "BFGS10"`); the default single start is compared with it -# (warn-level: a lower likelihood means a local optimum). As DiceKriging's -# nugget, the predictor returns y exactly *at* the design points (the nugget is -# a micro-scale variation): smoothing is checked next to them. - -# %% -sd_noise <- 0.05 * sd(y) -yn <- y + rnorm(length(y), sd = sd_noise) -kn <- Kriging(yn, X, "matern5_2", noise = "nugget", optim = "BFGS10") -pn <- predict(kn, Xt, return_stdev = TRUE) -add_metric(s, "branin noisy", "Kriging nugget BFGS10", q2 = q2(yt, pn$mean), nugget = kn$nugget(), - ll = logLikelihood(kn)) -check_greater(s, "Kriging(nugget): estimated nugget > 0", kn$nugget(), 0) -check_greater(s, "Kriging(nugget): Q2", q2(yt, pn$mean), 0.95) -near <- pmin(X + 1e-7, 1) -check_greater(s, "Kriging(nugget): smooths the noisy data next to the design points (rms residual / noise sd)", - rmse(yn, predict(kn, near, return_stdev = FALSE)$mean) / sd_noise, 0.3) -kn1 <- Kriging(yn, X, "matern5_2", noise = "nugget") -add_metric(s, "branin noisy", "Kriging nugget BFGS", q2 = q2(yt, predict(kn1, Xt, return_stdev = FALSE)$mean), - nugget = kn1$nugget(), ll = logLikelihood(kn1)) -check_greater(s, "Kriging(nugget): LL(single start) - LL(10 restarts)", logLikelihood(kn1) - logLikelihood(kn), -1, - severity = "warn") -roundtrip_same_mean(s, "Kriging(nugget)", kn, Xt) -kh <- Kriging(yn, X, "matern5_2", noise = rep(sd_noise^2, length(yn))) -check_close(s, "Kriging(noise = variances): noise() returns them", kh$noise(), rep(sd_noise^2, length(yn)), - rtol = 1e-12) -ph <- predict(kh, Xt, return_stdev = TRUE) -check_less(s, "Kriging(noise = variances): non-finite predictions", sum(!is.finite(c(ph$mean, ph$stdev))), 1) -add_metric(s, "branin noisy", "Kriging known noise", q2 = q2(yt, ph$mean)) - -# %% -me <- do.call(rbind, s$metrics) -print(me[me$metric == "q2", c("case", "package", "value")], row.names = FALSE) - -# %% -finalize(s) diff --git a/tools/src/noisy-data_Kriging-nugget-noise_julia.jl b/tools/src/noisy-data_Kriging-nugget-noise_julia.jl deleted file mode 100644 index ad25148..0000000 --- a/tools/src/noisy-data_Kriging-nugget-noise_julia.jl +++ /dev/null @@ -1,86 +0,0 @@ -# %% [markdown] -# # Julia · `Kriging` with noise — estimated nugget, known variances -# -# Branin on [0,1]² (n = 30 LHS points unless stated, Q² on 500 uniform test -# points), through JLibKriging. Checks (error-level unless stated): -# -# * `noise = "nugget"` (10 restarts): nugget > 0, Q² > 0.95 on the noise-free function, smoothing next to the design points -# * the default single start reaches the multistart likelihood (warn-level) -# * `save` then `load`: the same predicted mean -# * `noise = `: the stored variances, finite predictions -# -# The same checks as `noisy-data_Kriging-nugget-noise_R.ipynb`, with the Julia binding. - -# %% -import JLibKriging as LK -using Random -include("tools/julia/checks.jl") -s = Suite("noisy-data_Kriging-nugget-noise_julia") - -branin(U) = begin - x1 = 15 .* U[:, 1] .- 5; x2 = 15 .* U[:, 2] - (x2 .- 5.1 / (4π^2) .* x1 .^ 2 .+ 5 / π .* x1 .- 6) .^ 2 .+ 10 * (1 - 1 / (8π)) .* cos.(x1) .+ 10 -end -rng = MersenneTwister(1) -lhs(n, d) = hcat([(randperm(rng, n) .- rand(rng, n)) ./ n for _ in 1:d]...) -X = lhs(30, 2); y = branin(X) -Xt = rand(rng, 500, 2); yt = branin(Xt) -sd(v) = sqrt(sum((v .- sum(v) / length(v)) .^ 2) / (length(v) - 1)) -finite_count(v...) = count(!isfinite, vcat(v...)) - -function check_runs!(f, s, check) - try - f() - catch e - add_check!(s, check, false; value=string(typeof(e)), expected="no error", detail=first(sprint(showerror, e), 200)) - end -end - -function roundtrip!(s, label, k, Xn) - check_runs!(s, "$label: save + load") do - f = joinpath(mktempdir(), "k.json") - LK.save(k, f) - k2 = LK.load(f) - check_close!(s, "$label: load(save(k)) predicts the same mean", - LK.predict(k2, Xn; return_stdev=false).mean, LK.predict(k, Xn; return_stdev=false).mean; rtol=1e-10) - end -end - -# %% [markdown] -# ## `Kriging` with noise: estimated nugget, known variances -# -# Noise of standard deviation 5% of sd(y). The nugget model is fitted with 10 -# restarts (`optim="BFGS10"`); the default single start is compared with it -# (warn-level: a lower likelihood means a local optimum). As DiceKriging's -# nugget, the predictor returns y exactly *at* the design points (the nugget is -# a micro-scale variation): smoothing is checked next to them. - -# %% -sd_noise = 0.05 * sd(y) -yn = y .+ sd_noise .* randn(rng, length(y)) -kn = LK.Kriging(yn, X, "matern5_2"; noise="nugget", optim="BFGS10") -mn = LK.predict(kn, Xt; return_stdev=false).mean -add_metric!(s, "branin noisy", "Kriging nugget BFGS10"; q2=q2(yt, mn)) -add_check!(s, "Kriging(nugget): estimated nugget > 0", LK.nugget(kn) > 0; value=LK.nugget(kn), expected="> 0") -add_check!(s, "Kriging(nugget): Q2", q2(yt, mn) > 0.95; value=q2(yt, mn), expected="> 0.95") -r = sqrt(sum((LK.predict(kn, min.(X .+ 1e-7, 1.0); return_stdev=false).mean .- yn) .^ 2) / length(yn)) / sd_noise -add_check!(s, "Kriging(nugget): smooths the noisy data next to the design points (rms residual / noise sd)", r > 0.3; - value=r, expected="> 0.3") -kn1 = LK.Kriging(yn, X, "matern5_2"; noise="nugget") -add_metric!(s, "branin noisy", "Kriging nugget BFGS"; q2=q2(yt, LK.predict(kn1, Xt; return_stdev=false).mean)) -dll = LK.log_likelihood(kn1) - LK.log_likelihood(kn) -add_check!(s, "Kriging(nugget): LL(single start) - LL(10 restarts)", dll > -1; value=dll, expected="> -1", severity="warn") -roundtrip!(s, "Kriging(nugget)", kn, Xt) -v = fill(sd_noise^2, length(yn)) -kh = LK.Kriging(yn, X, "matern5_2"; noise=v) -check_close!(s, "Kriging(noise = variances): noise() returns them", LK.noise(kh), v; rtol=1e-12) -ph = LK.predict(kh, Xt; return_stdev=true) -add_check!(s, "Kriging(noise = variances): finite predictions", finite_count(ph.mean, ph.stdev) == 0; - value=finite_count(ph.mean, ph.stdev), expected="0 non-finite") -add_metric!(s, "branin noisy", "Kriging known noise"; q2=q2(yt, ph.mean)) - -# %% -foreach(r -> r[5] == "q2" && println(rpad(r[3], 24), rpad(r[4], 28), r[6]), s.metrics) - -# %% -finalize!(s) diff --git a/tools/src/noisy-data_Kriging-nugget-noise_octave.m b/tools/src/noisy-data_Kriging-nugget-noise_octave.m deleted file mode 100644 index f9bd732..0000000 --- a/tools/src/noisy-data_Kriging-nugget-noise_octave.m +++ /dev/null @@ -1,85 +0,0 @@ -% %% [markdown] -% # Octave · `Kriging` with noise — estimated nugget, known variances -% -% Branin on [0,1]² (n = 30 LHS points unless stated, Q² on 500 uniform test -% points), through mLibKriging. Checks (error-level unless stated): -% -% * `noise = "nugget"` (10 restarts): nugget > 0, Q² > 0.95 on the noise-free function, smoothing next to the design points -% * the default single start reaches the multistart likelihood (warn-level) -% * `save` then `load`: the same predicted mean -% * `noise = `: the stored variances, finite predictions -% -% The same checks as `noisy-data_Kriging-nugget-noise_R.ipynb`, with the Octave binding. - -% %% -addpath(fullfile(pwd, "tools", "octave")); -s = lkb_suite("noisy-data_Kriging-nugget-noise_octave"); -q2 = @(y, m) 1 - sum((y - m).^2) / sum((y - mean(y)).^2); -branin = @(U) ((15 * U(:, 2)) - 5.1 / (4 * pi^2) * (15 * U(:, 1) - 5).^2 + 5 / pi * (15 * U(:, 1) - 5) - 6).^2 ... - + 10 * (1 - 1 / (8 * pi)) * cos(15 * U(:, 1) - 5) + 10; -rand("seed", 1); randn("seed", 1); -function U = lhs(n, d) - U = zeros(n, d); - for j = 1:d, U(:, j) = (randperm(n)' - rand(n, 1)) / n; end -end -X = lhs(30, 2); y = branin(X); -Xt = rand(500, 2); yt = branin(Xt); -tmp = tempname(); mkdir(tmp); -has_class = @(name) exist(name) == 2; % a class file on the path - -function s = roundtrip(s, label, k, Xn, loader) - f = fullfile(tempdir(), [regexprep(label, "[^A-Za-z0-9]", "_") ".json"]); - try - k.save(f); - k2 = loader(f); - m1 = k.predict(Xn, false, false, false); - m2 = k2.predict(Xn, false, false, false); - s = lkb_close(s, [label ": load(save(k)) predicts the same mean"], m2, m1, 1e-10); - catch err - s = lkb_check(s, [label ": save + load"], false, "error", "no error", err.message); - end -end - -function s = skip(s, check, reason) - s.checks(end + 1, :) = {s.name, "octave", check, "info", "SKIP", "", "", reason}; - printf("[SKIP] %s: %s\n", check, reason); -end - -% %% [markdown] -% ## `Kriging` with noise: estimated nugget, known variances -% -% Noise of standard deviation 5% of sd(y). The nugget model is fitted with 10 -% restarts (`"BFGS10"`); the default single start is compared with it -% (warn-level: a lower likelihood means a local optimum). As DiceKriging's -% nugget, the predictor returns y exactly *at* the design points (the nugget is -% a micro-scale variation): smoothing is checked next to them. - -% %% -sd_noise = 0.05 * std(y); -yn = y + sd_noise * randn(size(y)); -kn = Kriging(yn, X, "matern5_2", "constant", false, "BFGS10", "LL", [], "nugget"); -mn = kn.predict(Xt, false, false, false); -nug = kn.nugget(); ll10 = kn.logLikelihood(); -s = lkb_metric(s, "branin noisy", "Kriging nugget BFGS10", "q2", q2(yt, mn)); -s = lkb_check(s, "Kriging(nugget): estimated nugget > 0", nug > 0, nug, "> 0"); -s = lkb_check(s, "Kriging(nugget): Q2", q2(yt, mn) > 0.95, q2(yt, mn), "> 0.95"); -r = sqrt(mean((kn.predict(min(X + 1e-7, 1), false, false, false) - yn).^2)) / sd_noise; -s = lkb_check(s, "Kriging(nugget): smooths the noisy data next to the design points (rms residual / noise sd)", r > 0.3, r, "> 0.3"); -kn1 = Kriging(yn, X, "matern5_2", "constant", false, "BFGS", "LL", [], "nugget"); -ll1 = kn1.logLikelihood(); -s = lkb_metric(s, "branin noisy", "Kriging nugget BFGS", "q2", q2(yt, kn1.predict(Xt, false, false, false))); -s = lkb_check(s, "Kriging(nugget): LL(single start) - LL(10 restarts)", ll1 - ll10 > -1, ll1 - ll10, "> -1", "", "warn"); -s = roundtrip(s, "Kriging(nugget)", kn, Xt, @load_kriging); -v = sd_noise^2 * ones(size(yn)); -kh = Kriging(yn, X, "matern5_2", "constant", false, "BFGS", "LL", [], "heterogeneous", v); -s = lkb_close(s, "Kriging(noise = variances): noise() returns them", kh.noise(), v, 1e-12); -[mh, sh] = kh.predict(Xt, true, false, false); -s = lkb_check(s, "Kriging(noise = variances): finite predictions", all(isfinite([mh; sh])), sum(~isfinite([mh; sh])), "0 non-finite"); -s = lkb_metric(s, "branin noisy", "Kriging known noise", "q2", q2(yt, mh)); - -% %% -q = strcmp(s.metrics(:, 5), "q2"); -disp(s.metrics(q, [3, 4, 6])) - -% %% -lkb_finalize(s); diff --git a/tools/src/noisy-data_Kriging-nugget-noise_python.py b/tools/src/noisy-data_Kriging-nugget-noise_python.py index 48aaff1..bf9ab12 100644 --- a/tools/src/noisy-data_Kriging-nugget-noise_python.py +++ b/tools/src/noisy-data_Kriging-nugget-noise_python.py @@ -56,6 +56,11 @@ def pred(k, X): res = metrics.all_metrics(yt, m, s) suite.metric(fname, "nugget", nugget=nugget, true_var=sd ** 2, **res) suite.between(f"{fname}: nugget / true noise variance", nugget / sd ** 2, 1 / 3, 3) + k1 = lk.Kriging(y, X, "matern5_2", noise="nugget") # default single start + suite.metric(fname, "nugget, single start", nugget=float(np.ravel(k1.nugget())[0]), ll=float(k1.logLikelihood()), + **metrics.all_metrics(yt, *pred(k1, Xt))) + suite.greater(f"{fname}: LL(single start) - LL(10 restarts)", k1.logLikelihood() - k.logLikelihood(), -1, + severity="warn") suite.greater(f"{fname}: Q2 vs noise-free truth", res["q2"], q2_floor) mX, _ = pred(k, X) mX_near, _ = pred(k, np.clip(X + 1e-7, 0, 1)) diff --git a/tools/src/robustness-invalid-inputs_Kriging_python.py b/tools/src/robustness-invalid-inputs_Kriging_python.py index b948abf..75490d7 100644 --- a/tools/src/robustness-invalid-inputs_Kriging_python.py +++ b/tools/src/robustness-invalid-inputs_Kriging_python.py @@ -21,6 +21,7 @@ # %% import time +import matplotlib.pyplot as plt import numpy as np import pylibkriging as lk @@ -159,6 +160,13 @@ def finite_model(check, make, Xn=Xt, severity="error"): expo = np.log2(times[8000] / times[4000]) suite.less("predict(stdev) time growth exponent in q (8000 vs 4000)", expo, 1.5, detail=", ".join(f"q={q}: {t:.3f}s" for q, t in times.items())) +qs = np.array(list(times)); ts = np.array(list(times.values())) +plt.figure(figsize=(5, 3.6)) +plt.loglog(qs, ts, "o-", label="predict(return_stdev=True)") +plt.loglog(qs, ts[0] * qs / qs[0], "k--", lw=.8, label="O(q) reference") +plt.loglog(qs, ts[0] * (qs / qs[0]) ** 2, "k:", lw=.8, label="O(q²) reference") +plt.xlabel("q (prediction points)"); plt.ylabel("time (s)"); plt.legend(); plt.grid(True, which="both", alpha=.3) +plt.show() # %% suite.finalize() diff --git a/tools/src/update-simulate_Kriging_python.py b/tools/src/update-simulate_Kriging_python.py index 3c3ed9c..fdd5eca 100644 --- a/tools/src/update-simulate_Kriging_python.py +++ b/tools/src/update-simulate_Kriging_python.py @@ -18,6 +18,7 @@ # Functions: Branin (2D), Hartmann 3 (3D), Hartmann 6 (6D). # %% +import matplotlib.pyplot as plt import numpy as np import pylibkriging as lk @@ -132,5 +133,30 @@ def fixed_params(k): rel = np.linalg.norm(np.cov(U) - c) / np.linalg.norm(c) suite.less("update_simulate: covariance vs updated model (rel. Frobenius)", rel, 0.1) +# %% [markdown] +# ## In pictures: Forrester, 6 points, then 2 more +# +# Conditional paths of the 6-point model (left), the same paths corrected by +# `update_simulate` with 2 new observations (right): they now go through the new +# points, without redrawing them. + +# %% +tf = TESTS["forrester"] +X = np.array([[0.0], [0.2], [0.4], [0.6], [0.8], [1.0]]); y = tf.unit(X) +Xu = np.array([[0.7], [0.9]]); yu = tf.unit(Xu) +G = np.linspace(0, 1, 201).reshape(-1, 1) +k = lk.Kriging(y, X, "matern5_2") +P = np.asarray(k.simulate(20, 3, G, will_update=True)) +PU = np.asarray(k.update_simulate(yu, Xu)) +suite.less("update_simulate: corrected paths go through the new points (max |path - y_u| / sd(y))", + np.max(np.abs(PU[[140, 180], :] - yu[:, None])) / np.std(y), 1e-3) +fig, axes = plt.subplots(1, 2, figsize=(11, 3.8), sharey=True) +for ax, paths, title in [(axes[0], P, "6 points: conditional paths"), (axes[1], PU, "+ 2 points: update_simulate")]: + ax.plot(G, paths, color="tab:blue", lw=.6, alpha=.6) + ax.plot(G, tf.unit(G), "k--", lw=1, label="Forrester") + ax.plot(X, y, "ko", label="design") + ax.set_title(title); ax.set_xlabel("x") +axes[1].plot(Xu, yu, "rs", label="new points"); axes[1].legend(); plt.show() + # %% suite.finalize() diff --git a/tools/src/vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python.py b/tools/src/vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python.py index 7a251d9..f71be9c 100644 --- a/tools/src/vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python.py +++ b/tools/src/vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python.py @@ -43,6 +43,7 @@ import time import warnings +import matplotlib.pyplot as plt import numpy as np import pandas as pd import pylibkriging as lk @@ -218,6 +219,12 @@ def p(Z): suite.metric(fname, pkg, **r.to_dict()) summary.unstack("package")["q2"].round(4) +# %% +q2tab = summary.unstack("package")["q2"].clip(lower=-0.1) +q2tab.plot.bar(figsize=(11, 3.8), width=.8) +plt.ylabel("median Q² (test set)"); plt.ylim(-0.1, 1.02); plt.legend(ncol=3, fontsize=8); plt.grid(axis="y", alpha=.3) +plt.title("Default fits: accuracy per function and package"); plt.show() + # %% summary.unstack("package")[["fit_s"]].round(3) diff --git a/tools/src/warpings-categorical_WarpKriging_R.R b/tools/src/warpings-categorical_WarpKriging_R.R deleted file mode 100644 index 0d2106b..0000000 --- a/tools/src/warpings-categorical_WarpKriging_R.R +++ /dev/null @@ -1,78 +0,0 @@ -# %% [markdown] -# # R · `WarpKriging` — continuous warpings, a categorical input -# -# Branin on [0,1]² (n = 30 LHS points unless stated, Q² on 500 uniform test -# points), through rlibkriging. Checks (error-level unless stated): -# -# * `kumaraswamy` on both inputs, then x₂ restricted to 5 levels with `categorical(5,2)`: finite predictions -# * Q² floors (warn-level) -# * `save` then `load`: the same predicted mean -# -# A class missing from the installed rlibkriging (release channel) is recorded as SKIP. - -# %% -suppressPackageStartupMessages(library(rlibkriging)) -source("tools/r/checks.R") -s <- new_suite("warpings-categorical_WarpKriging_R") -cat("rlibkriging", as.character(packageVersion("rlibkriging")), "\n") - -branin <- function(U) { # on [0,1]^2 - x1 <- 15 * U[, 1] - 5; x2 <- 15 * U[, 2] - (x2 - 5.1 / (4 * pi^2) * x1^2 + 5 / pi * x1 - 6)^2 + 10 * (1 - 1 / (8 * pi)) * cos(x1) + 10 -} -set.seed(1) -lhs <- function(n, d) sapply(1:d, function(j) (sample(n) - runif(n)) / n) -X <- lhs(30, 2); y <- branin(X) -Xt <- matrix(runif(1000), ncol = 2); yt <- branin(Xt) -tmp <- tempdir() -has_class <- function(name) exists(name, mode = "function") -skip <- function(s, check, reason) { - s$checks[[length(s$checks) + 1]] <- data.frame(suite = s$name, lang = "R", check = check, severity = "info", - status = "SKIP", value = "", expected = "", detail = reason, stringsAsFactors = FALSE) - message("[SKIP] ", check, ": ", reason) -} -roundtrip_same_mean <- function(s, label, k, Xn) { - f <- file.path(tmp, paste0(gsub("[^A-Za-z0-9]", "_", label), ".json")) - check_runs(s, paste0(label, ": save + load"), { - save(k, f) - k2 <- rlibkriging::load(f) - check_close(s, paste0(label, ": load(save(k)) predicts the same mean"), - predict(k2, Xn, return_stdev = FALSE)$mean, predict(k, Xn, return_stdev = FALSE)$mean, rtol = 1e-10) - }) -} - -# %% [markdown] -# ## `WarpKriging`: continuous warpings, then a categorical input -# -# The categorical case: x₂ restricted to 5 levels (coded 0, …, 4), embedded in ℝ². - -# %% -if (has_class("WarpKriging")) { - check_runs(s, "WarpKriging(kumaraswamy, kumaraswamy): fit + predict", { - kw <- WarpKriging(y, X, warping = c("kumaraswamy", "kumaraswamy"), kernel = "matern5_2") - pw <- predict(kw, Xt, return_stdev = TRUE) - check_less(s, "WarpKriging(kumaraswamy): non-finite predictions", sum(!is.finite(c(pw$mean, pw$stdev))), 1) - add_metric(s, "branin", "WarpKriging kumaraswamy", q2 = q2(yt, pw$mean)) - check_greater(s, "WarpKriging(kumaraswamy): Q2", q2(yt, pw$mean), 0.9, severity = "warn") - roundtrip_same_mean(s, "WarpKriging(kumaraswamy)", kw, Xt) - }) - lev <- function(U) cbind(U[, 1], round(U[, 2] * 4)) # level codes 0..4 - unlev <- function(L) cbind(L[, 1], L[, 2] / 4) - Xc <- lev(lhs(40, 2)); yc <- branin(unlev(Xc)) - Xtc <- lev(Xt); ytc <- branin(unlev(Xtc)) - check_runs(s, "WarpKriging(none, categorical(5,2)): fit + predict", { - kc <- WarpKriging(yc, Xc, warping = c("none", "categorical(5,2)"), kernel = "matern5_2") - pc <- predict(kc, Xtc, return_stdev = TRUE) - check_less(s, "WarpKriging(categorical): non-finite predictions", sum(!is.finite(c(pc$mean, pc$stdev))), 1) - add_metric(s, "branin, x2 categorical", "WarpKriging categorical", q2 = q2(ytc, pc$mean)) - check_greater(s, "WarpKriging(categorical): Q2", q2(ytc, pc$mean), 0.8, severity = "warn") - roundtrip_same_mean(s, "WarpKriging(categorical)", kc, Xtc) - }) -} else skip(s, "WarpKriging", "not in this rlibkriging") - -# %% -me <- do.call(rbind, s$metrics) -print(me[me$metric == "q2", c("case", "package", "value")], row.names = FALSE) - -# %% -finalize(s) diff --git a/tools/src/warpings-categorical_WarpKriging_julia.jl b/tools/src/warpings-categorical_WarpKriging_julia.jl deleted file mode 100644 index 2a30927..0000000 --- a/tools/src/warpings-categorical_WarpKriging_julia.jl +++ /dev/null @@ -1,83 +0,0 @@ -# %% [markdown] -# # Julia · `WarpKriging` — continuous warpings, a categorical input -# -# Branin on [0,1]² (n = 30 LHS points unless stated, Q² on 500 uniform test -# points), through JLibKriging. Checks (error-level unless stated): -# -# * `kumaraswamy` on both inputs, then x₂ restricted to 5 levels with `categorical(5,2)`: finite predictions -# * Q² floors (warn-level) -# * `save` then `load`: the same predicted mean -# -# The same checks as `warpings-categorical_WarpKriging_R.ipynb`, with the Julia binding. - -# %% -import JLibKriging as LK -using Random -include("tools/julia/checks.jl") -s = Suite("warpings-categorical_WarpKriging_julia") - -branin(U) = begin - x1 = 15 .* U[:, 1] .- 5; x2 = 15 .* U[:, 2] - (x2 .- 5.1 / (4π^2) .* x1 .^ 2 .+ 5 / π .* x1 .- 6) .^ 2 .+ 10 * (1 - 1 / (8π)) .* cos.(x1) .+ 10 -end -rng = MersenneTwister(1) -lhs(n, d) = hcat([(randperm(rng, n) .- rand(rng, n)) ./ n for _ in 1:d]...) -X = lhs(30, 2); y = branin(X) -Xt = rand(rng, 500, 2); yt = branin(Xt) -sd(v) = sqrt(sum((v .- sum(v) / length(v)) .^ 2) / (length(v) - 1)) -finite_count(v...) = count(!isfinite, vcat(v...)) - -function check_runs!(f, s, check) - try - f() - catch e - add_check!(s, check, false; value=string(typeof(e)), expected="no error", detail=first(sprint(showerror, e), 200)) - end -end - -function roundtrip!(s, label, k, Xn) - check_runs!(s, "$label: save + load") do - f = joinpath(mktempdir(), "k.json") - LK.save(k, f) - k2 = LK.load(f) - check_close!(s, "$label: load(save(k)) predicts the same mean", - LK.predict(k2, Xn; return_stdev=false).mean, LK.predict(k, Xn; return_stdev=false).mean; rtol=1e-10) - end -end - -# %% [markdown] -# ## `WarpKriging`: continuous warpings, then a categorical input -# -# The categorical case: x₂ restricted to 5 levels (coded 0, …, 4), embedded in ℝ². - -# %% -check_runs!(s, "WarpKriging(kumaraswamy, kumaraswamy): fit + predict") do - kw = LK.WarpKriging(y, X, ["kumaraswamy", "kumaraswamy"], "matern5_2") - pw = LK.predict(kw, Xt; return_stdev=true) - add_check!(s, "WarpKriging(kumaraswamy): finite predictions", finite_count(pw.mean, pw.stdev) == 0; - value=finite_count(pw.mean, pw.stdev), expected="0 non-finite") - add_metric!(s, "branin", "WarpKriging kumaraswamy"; q2=q2(yt, pw.mean)) - add_check!(s, "WarpKriging(kumaraswamy): Q2", q2(yt, pw.mean) > 0.9; value=q2(yt, pw.mean), expected="> 0.9", - severity="warn") - roundtrip!(s, "WarpKriging(kumaraswamy)", kw, Xt) -end -lev(U) = hcat(U[:, 1], round.(U[:, 2] .* 4)) -unlev(L) = hcat(L[:, 1], L[:, 2] ./ 4) -Xc = lev(lhs(40, 2)); yc = branin(unlev(Xc)) -Xtc = lev(Xt); ytc = branin(unlev(Xtc)) -check_runs!(s, "WarpKriging(none, categorical(5,2)): fit + predict") do - kc = LK.WarpKriging(yc, Xc, ["none", "categorical(5,2)"], "matern5_2") - pc = LK.predict(kc, Xtc; return_stdev=true) - add_check!(s, "WarpKriging(categorical): finite predictions", finite_count(pc.mean, pc.stdev) == 0; - value=finite_count(pc.mean, pc.stdev), expected="0 non-finite") - add_metric!(s, "branin, x2 categorical", "WarpKriging categorical"; q2=q2(ytc, pc.mean)) - add_check!(s, "WarpKriging(categorical): Q2", q2(ytc, pc.mean) > 0.8; value=q2(ytc, pc.mean), expected="> 0.8", - severity="warn") - roundtrip!(s, "WarpKriging(categorical)", kc, Xtc) -end - -# %% -foreach(r -> r[5] == "q2" && println(rpad(r[3], 24), rpad(r[4], 28), r[6]), s.metrics) - -# %% -finalize!(s) diff --git a/tools/src/warpings-categorical_WarpKriging_octave.m b/tools/src/warpings-categorical_WarpKriging_octave.m deleted file mode 100644 index adc32f6..0000000 --- a/tools/src/warpings-categorical_WarpKriging_octave.m +++ /dev/null @@ -1,89 +0,0 @@ -% %% [markdown] -% # Octave · `WarpKriging` — continuous warpings, a categorical input -% -% Branin on [0,1]² (n = 30 LHS points unless stated, Q² on 500 uniform test -% points), through mLibKriging. Checks (error-level unless stated): -% -% * `kumaraswamy` on both inputs, then x₂ restricted to 5 levels with `categorical(5,2)`: finite predictions -% * Q² floors (warn-level) -% * `save` then `load`: the same predicted mean -% -% The same checks as `warpings-categorical_WarpKriging_R.ipynb`, with the Octave binding. -% -% A class missing from the installed mLibKriging (release channel) is recorded as SKIP. - -% %% -addpath(fullfile(pwd, "tools", "octave")); -s = lkb_suite("warpings-categorical_WarpKriging_octave"); -q2 = @(y, m) 1 - sum((y - m).^2) / sum((y - mean(y)).^2); -branin = @(U) ((15 * U(:, 2)) - 5.1 / (4 * pi^2) * (15 * U(:, 1) - 5).^2 + 5 / pi * (15 * U(:, 1) - 5) - 6).^2 ... - + 10 * (1 - 1 / (8 * pi)) * cos(15 * U(:, 1) - 5) + 10; -rand("seed", 1); randn("seed", 1); -function U = lhs(n, d) - U = zeros(n, d); - for j = 1:d, U(:, j) = (randperm(n)' - rand(n, 1)) / n; end -end -X = lhs(30, 2); y = branin(X); -Xt = rand(500, 2); yt = branin(Xt); -tmp = tempname(); mkdir(tmp); -has_class = @(name) exist(name) == 2; % a class file on the path - -function s = roundtrip(s, label, k, Xn, loader) - f = fullfile(tempdir(), [regexprep(label, "[^A-Za-z0-9]", "_") ".json"]); - try - k.save(f); - k2 = loader(f); - m1 = k.predict(Xn, false, false, false); - m2 = k2.predict(Xn, false, false, false); - s = lkb_close(s, [label ": load(save(k)) predicts the same mean"], m2, m1, 1e-10); - catch err - s = lkb_check(s, [label ": save + load"], false, "error", "no error", err.message); - end -end - -function s = skip(s, check, reason) - s.checks(end + 1, :) = {s.name, "octave", check, "info", "SKIP", "", "", reason}; - printf("[SKIP] %s: %s\n", check, reason); -end - -% %% [markdown] -% ## `WarpKriging`: continuous warpings, then a categorical input -% -% The categorical case: x₂ restricted to 5 levels (coded 0, …, 4), embedded in ℝ². - -% %% -if has_class("WarpKriging") - try - kw = WarpKriging(y, X, {"kumaraswamy", "kumaraswamy"}, "matern5_2"); - [mw, sw] = kw.predict(Xt, true, false, false); - s = lkb_check(s, "WarpKriging(kumaraswamy): finite predictions", all(isfinite([mw; sw])), sum(~isfinite([mw; sw])), "0 non-finite"); - s = lkb_metric(s, "branin", "WarpKriging kumaraswamy", "q2", q2(yt, mw)); - s = lkb_check(s, "WarpKriging(kumaraswamy): Q2", q2(yt, mw) > 0.9, q2(yt, mw), "> 0.9", "", "warn"); - s = roundtrip(s, "WarpKriging(kumaraswamy)", kw, Xt, @load_kriging); - catch err - s = lkb_check(s, "WarpKriging(kumaraswamy): fit + predict", false, "error", "no error", err.message); - end - lev = @(U) [U(:, 1), round(U(:, 2) * 4)]; - unlev = @(L) [L(:, 1), L(:, 2) / 4]; - Xc = lev(lhs(40, 2)); yc = branin(unlev(Xc)); - Xtc = lev(Xt); ytc = branin(unlev(Xtc)); - try - kc = WarpKriging(yc, Xc, {"none", "categorical(5,2)"}, "matern5_2"); - [mc, sc] = kc.predict(Xtc, true, false, false); - s = lkb_check(s, "WarpKriging(categorical): finite predictions", all(isfinite([mc; sc])), sum(~isfinite([mc; sc])), "0 non-finite"); - s = lkb_metric(s, "branin, x2 categorical", "WarpKriging categorical", "q2", q2(ytc, mc)); - s = lkb_check(s, "WarpKriging(categorical): Q2", q2(ytc, mc) > 0.8, q2(ytc, mc), "> 0.8", "", "warn"); - s = roundtrip(s, "WarpKriging(categorical)", kc, Xtc, @load_kriging); - catch err - s = lkb_check(s, "WarpKriging(categorical): fit + predict", false, "error", "no error", err.message); - end -else - s = skip(s, "WarpKriging", "not in this mLibKriging"); -end - -% %% -q = strcmp(s.metrics(:, 5), "q2"); -disp(s.metrics(q, [3, 4, 6])) - -% %% -lkb_finalize(s); diff --git a/update-simulate_Kriging_python.ipynb b/update-simulate_Kriging_python.ipynb index f1a3496..86b2ccb 100644 --- a/update-simulate_Kriging_python.ipynb +++ b/update-simulate_Kriging_python.ipynb @@ -31,6 +31,42 @@ "metadata": {}, "outputs": [], "source": [ + "# Setup for running this notebook outside the repository (Colab, Binder, a lone copy):\n", + "# nothing to do in the repository itself (CI, local runs). Otherwise the repository\n", + "# (datasets of data/, helpers of tools/) is cloned and the released pylibkriging installed.\n", + "import importlib\n", + "import os\n", + "import subprocess\n", + "import sys\n", + "from pathlib import Path\n", + "\n", + "if not Path(\"tools/known_issues.csv\").is_file():\n", + " if not Path(\"benchmarks/tools/known_issues.csv\").is_file():\n", + " subprocess.run([\"git\", \"clone\", \"--depth\", \"1\", \"https://github.com/libKriging/benchmarks\"], check=True)\n", + " os.chdir(\"benchmarks\")\n", + " try:\n", + " import pylibkriging # noqa: F401\n", + " except ImportError:\n", + " r = subprocess.run([sys.executable, \"-m\", \"pip\", \"install\", \"--quiet\", \"pylibkriging\"],\n", + " capture_output=True, text=True)\n", + " if r.returncode != 0:\n", + " raise RuntimeError(\n", + " f\"pylibkriging could not be installed for Python {sys.version_info.major}.{sys.version_info.minor} \"\n", + " \"(PyPI has wheels for CPython 3.10 to 3.12 only). On Colab: Runtime > Change runtime version > \"\n", + " \"2026.07 (Python 3.12), then run again.\\n\" + r.stderr[-500:])\n", + " importlib.invalidate_caches()\n", + "sys.path.insert(0, str(Path(\"tools\").resolve())) # lkbench, when not installed (Colab, Binder)\n", + "print(\"working directory:\", os.getcwd())" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c002", + "metadata": {}, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", "import numpy as np\n", "import pylibkriging as lk\n", "\n", @@ -53,7 +89,7 @@ }, { "cell_type": "markdown", - "id": "c002", + "id": "c003", "metadata": {}, "source": [ "## `update` vs a fit from scratch" @@ -62,7 +98,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c003", + "id": "c004", "metadata": {}, "outputs": [], "source": [ @@ -97,7 +133,7 @@ }, { "cell_type": "markdown", - "id": "c004", + "id": "c005", "metadata": {}, "source": [ "Several small updates in a row equal one update with all the points." @@ -106,7 +142,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c005", + "id": "c006", "metadata": {}, "outputs": [], "source": [ @@ -123,7 +159,7 @@ }, { "cell_type": "markdown", - "id": "c006", + "id": "c007", "metadata": {}, "source": [ "## `simulate`: moments, seeds, conditioning" @@ -132,7 +168,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c007", + "id": "c008", "metadata": {}, "outputs": [], "source": [ @@ -160,7 +196,7 @@ }, { "cell_type": "markdown", - "id": "c008", + "id": "c009", "metadata": {}, "source": [ "## `update_simulate`: paths conditioned on new data\n", @@ -172,7 +208,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c009", + "id": "c010", "metadata": {}, "outputs": [], "source": [ @@ -194,10 +230,47 @@ " suite.less(\"update_simulate: covariance vs updated model (rel. Frobenius)\", rel, 0.1)" ] }, + { + "cell_type": "markdown", + "id": "c011", + "metadata": {}, + "source": [ + "## In pictures: Forrester, 6 points, then 2 more\n", + "\n", + "Conditional paths of the 6-point model (left), the same paths corrected by\n", + "`update_simulate` with 2 new observations (right): they now go through the new\n", + "points, without redrawing them." + ] + }, { "cell_type": "code", "execution_count": null, - "id": "c010", + "id": "c012", + "metadata": {}, + "outputs": [], + "source": [ + "tf = TESTS[\"forrester\"]\n", + "X = np.array([[0.0], [0.2], [0.4], [0.6], [0.8], [1.0]]); y = tf.unit(X)\n", + "Xu = np.array([[0.7], [0.9]]); yu = tf.unit(Xu)\n", + "G = np.linspace(0, 1, 201).reshape(-1, 1)\n", + "k = lk.Kriging(y, X, \"matern5_2\")\n", + "P = np.asarray(k.simulate(20, 3, G, will_update=True))\n", + "PU = np.asarray(k.update_simulate(yu, Xu))\n", + "suite.less(\"update_simulate: corrected paths go through the new points (max |path - y_u| / sd(y))\",\n", + " np.max(np.abs(PU[[140, 180], :] - yu[:, None])) / np.std(y), 1e-3)\n", + "fig, axes = plt.subplots(1, 2, figsize=(11, 3.8), sharey=True)\n", + "for ax, paths, title in [(axes[0], P, \"6 points: conditional paths\"), (axes[1], PU, \"+ 2 points: update_simulate\")]:\n", + " ax.plot(G, paths, color=\"tab:blue\", lw=.6, alpha=.6)\n", + " ax.plot(G, tf.unit(G), \"k--\", lw=1, label=\"Forrester\")\n", + " ax.plot(X, y, \"ko\", label=\"design\")\n", + " ax.set_title(title); ax.set_xlabel(\"x\")\n", + "axes[1].plot(Xu, yu, \"rs\", label=\"new points\"); axes[1].legend(); plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c013", "metadata": {}, "outputs": [], "source": [ diff --git a/vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python.ipynb b/vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python.ipynb index bbdaf49..022e337 100644 --- a/vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python.ipynb +++ b/vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python.ipynb @@ -26,9 +26,44 @@ ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": null, "id": "c001", "metadata": {}, + "outputs": [], + "source": [ + "# Setup for running this notebook outside the repository (Colab, Binder, a lone copy):\n", + "# nothing to do in the repository itself (CI, local runs). Otherwise the repository\n", + "# (datasets of data/, helpers of tools/) is cloned and the released pylibkriging installed.\n", + "import importlib\n", + "import os\n", + "import subprocess\n", + "import sys\n", + "from pathlib import Path\n", + "\n", + "if not Path(\"tools/known_issues.csv\").is_file():\n", + " if not Path(\"benchmarks/tools/known_issues.csv\").is_file():\n", + " subprocess.run([\"git\", \"clone\", \"--depth\", \"1\", \"https://github.com/libKriging/benchmarks\"], check=True)\n", + " os.chdir(\"benchmarks\")\n", + " try:\n", + " import pylibkriging # noqa: F401\n", + " except ImportError:\n", + " r = subprocess.run([sys.executable, \"-m\", \"pip\", \"install\", \"--quiet\", \"pylibkriging\"],\n", + " capture_output=True, text=True)\n", + " if r.returncode != 0:\n", + " raise RuntimeError(\n", + " f\"pylibkriging could not be installed for Python {sys.version_info.major}.{sys.version_info.minor} \"\n", + " \"(PyPI has wheels for CPython 3.10 to 3.12 only). On Colab: Runtime > Change runtime version > \"\n", + " \"2026.07 (Python 3.12), then run again.\\n\" + r.stderr[-500:])\n", + " importlib.invalidate_caches()\n", + "sys.path.insert(0, str(Path(\"tools\").resolve())) # lkbench, when not installed (Colab, Binder)\n", + "print(\"working directory:\", os.getcwd())" + ] + }, + { + "cell_type": "markdown", + "id": "c002", + "metadata": {}, "source": [ "## Installing scikit-learn, SMT, OpenTURNS, GPy\n", "\n", @@ -41,7 +76,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c002", + "id": "c003", "metadata": {}, "outputs": [], "source": [ @@ -61,13 +96,14 @@ { "cell_type": "code", "execution_count": null, - "id": "c003", + "id": "c004", "metadata": {}, "outputs": [], "source": [ "import time\n", "import warnings\n", "\n", + "import matplotlib.pyplot as plt\n", "import numpy as np\n", "import pandas as pd\n", "import pylibkriging as lk\n", @@ -94,7 +130,7 @@ }, { "cell_type": "markdown", - "id": "c004", + "id": "c005", "metadata": {}, "source": [ "## 1. Same hyperparameters, same numbers" @@ -103,7 +139,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c005", + "id": "c006", "metadata": {}, "outputs": [], "source": [ @@ -145,7 +181,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c006", + "id": "c007", "metadata": {}, "outputs": [], "source": [ @@ -177,7 +213,7 @@ }, { "cell_type": "markdown", - "id": "c007", + "id": "c008", "metadata": {}, "source": [ "## 2. Default fits on a set of test functions\n", @@ -189,7 +225,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c008", + "id": "c009", "metadata": {}, "outputs": [], "source": [ @@ -278,7 +314,20 @@ { "cell_type": "code", "execution_count": null, - "id": "c009", + "id": "c010", + "metadata": {}, + "outputs": [], + "source": [ + "q2tab = summary.unstack(\"package\")[\"q2\"].clip(lower=-0.1)\n", + "q2tab.plot.bar(figsize=(11, 3.8), width=.8)\n", + "plt.ylabel(\"median Q² (test set)\"); plt.ylim(-0.1, 1.02); plt.legend(ncol=3, fontsize=8); plt.grid(axis=\"y\", alpha=.3)\n", + "plt.title(\"Default fits: accuracy per function and package\"); plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c011", "metadata": {}, "outputs": [], "source": [ @@ -288,7 +337,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c010", + "id": "c012", "metadata": {}, "outputs": [], "source": [ @@ -306,7 +355,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c011", + "id": "c013", "metadata": {}, "outputs": [], "source": [ diff --git a/warpings-categorical-ordinal_WarpKriging_python.ipynb b/warpings-categorical-ordinal_WarpKriging_python.ipynb index bd201a6..7760dc4 100644 --- a/warpings-categorical-ordinal_WarpKriging_python.ipynb +++ b/warpings-categorical-ordinal_WarpKriging_python.ipynb @@ -28,6 +28,41 @@ "id": "c001", "metadata": {}, "outputs": [], + "source": [ + "# Setup for running this notebook outside the repository (Colab, Binder, a lone copy):\n", + "# nothing to do in the repository itself (CI, local runs). Otherwise the repository\n", + "# (datasets of data/, helpers of tools/) is cloned and the released pylibkriging installed.\n", + "import importlib\n", + "import os\n", + "import subprocess\n", + "import sys\n", + "from pathlib import Path\n", + "\n", + "if not Path(\"tools/known_issues.csv\").is_file():\n", + " if not Path(\"benchmarks/tools/known_issues.csv\").is_file():\n", + " subprocess.run([\"git\", \"clone\", \"--depth\", \"1\", \"https://github.com/libKriging/benchmarks\"], check=True)\n", + " os.chdir(\"benchmarks\")\n", + " try:\n", + " import pylibkriging # noqa: F401\n", + " except ImportError:\n", + " r = subprocess.run([sys.executable, \"-m\", \"pip\", \"install\", \"--quiet\", \"pylibkriging\"],\n", + " capture_output=True, text=True)\n", + " if r.returncode != 0:\n", + " raise RuntimeError(\n", + " f\"pylibkriging could not be installed for Python {sys.version_info.major}.{sys.version_info.minor} \"\n", + " \"(PyPI has wheels for CPython 3.10 to 3.12 only). On Colab: Runtime > Change runtime version > \"\n", + " \"2026.07 (Python 3.12), then run again.\\n\" + r.stderr[-500:])\n", + " importlib.invalidate_caches()\n", + "sys.path.insert(0, str(Path(\"tools\").resolve())) # lkbench, when not installed (Colab, Binder)\n", + "print(\"working directory:\", os.getcwd())" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c002", + "metadata": {}, + "outputs": [], "source": [ "import numpy as np\n", "import pandas as pd\n", @@ -71,7 +106,7 @@ }, { "cell_type": "markdown", - "id": "c002", + "id": "c003", "metadata": {}, "source": [ "## `none` warping == plain Kriging" @@ -80,7 +115,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c003", + "id": "c004", "metadata": {}, "outputs": [], "source": [ @@ -98,7 +133,7 @@ }, { "cell_type": "markdown", - "id": "c004", + "id": "c005", "metadata": {}, "source": [ "## 1D non-stationary: Gramacy & Lee (2012)" @@ -107,7 +142,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c005", + "id": "c006", "metadata": {}, "outputs": [], "source": [ @@ -139,7 +174,7 @@ }, { "cell_type": "markdown", - "id": "c006", + "id": "c007", "metadata": {}, "source": [ "## 2D non-stationary: Gramacy & Lee (2008)" @@ -148,7 +183,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c007", + "id": "c008", "metadata": {}, "outputs": [], "source": [ @@ -166,7 +201,7 @@ }, { "cell_type": "markdown", - "id": "c008", + "id": "c009", "metadata": {}, "source": [ "## Mixed continuous / categorical\n", @@ -179,7 +214,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c009", + "id": "c010", "metadata": {}, "outputs": [], "source": [ @@ -211,7 +246,7 @@ }, { "cell_type": "markdown", - "id": "c010", + "id": "c011", "metadata": {}, "source": [ "## Mixed continuous / ordinal\n", @@ -223,7 +258,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c011", + "id": "c012", "metadata": {}, "outputs": [], "source": [ @@ -253,7 +288,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c012", + "id": "c013", "metadata": {}, "outputs": [], "source": [ @@ -262,7 +297,7 @@ }, { "cell_type": "markdown", - "id": "c013", + "id": "c014", "metadata": {}, "source": [ "## save / load, pickle, copy\n", @@ -275,7 +310,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c014", + "id": "c015", "metadata": {}, "outputs": [], "source": [ @@ -288,7 +323,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c015", + "id": "c016", "metadata": {}, "outputs": [], "source": [ diff --git a/warpings-categorical_WarpKriging_R.ipynb b/warpings-categorical_WarpKriging_R.ipynb deleted file mode 100644 index 869874e..0000000 --- a/warpings-categorical_WarpKriging_R.ipynb +++ /dev/null @@ -1,133 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "c000", - "metadata": {}, - "source": [ - "# R · `WarpKriging` — continuous warpings, a categorical input\n", - "\n", - "Branin on [0,1]² (n = 30 LHS points unless stated, Q² on 500 uniform test\n", - "points), through rlibkriging. Checks (error-level unless stated):\n", - "\n", - "* `kumaraswamy` on both inputs, then x₂ restricted to 5 levels with `categorical(5,2)`: finite predictions\n", - "* Q² floors (warn-level)\n", - "* `save` then `load`: the same predicted mean\n", - "\n", - "A class missing from the installed rlibkriging (release channel) is recorded as SKIP." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c001", - "metadata": {}, - "outputs": [], - "source": [ - "suppressPackageStartupMessages(library(rlibkriging))\n", - "source(\"tools/r/checks.R\")\n", - "s <- new_suite(\"warpings-categorical_WarpKriging_R\")\n", - "cat(\"rlibkriging\", as.character(packageVersion(\"rlibkriging\")), \"\\n\")\n", - "\n", - "branin <- function(U) { # on [0,1]^2\n", - " x1 <- 15 * U[, 1] - 5; x2 <- 15 * U[, 2]\n", - " (x2 - 5.1 / (4 * pi^2) * x1^2 + 5 / pi * x1 - 6)^2 + 10 * (1 - 1 / (8 * pi)) * cos(x1) + 10\n", - "}\n", - "set.seed(1)\n", - "lhs <- function(n, d) sapply(1:d, function(j) (sample(n) - runif(n)) / n)\n", - "X <- lhs(30, 2); y <- branin(X)\n", - "Xt <- matrix(runif(1000), ncol = 2); yt <- branin(Xt)\n", - "tmp <- tempdir()\n", - "has_class <- function(name) exists(name, mode = \"function\")\n", - "skip <- function(s, check, reason) {\n", - " s$checks[[length(s$checks) + 1]] <- data.frame(suite = s$name, lang = \"R\", check = check, severity = \"info\",\n", - " status = \"SKIP\", value = \"\", expected = \"\", detail = reason, stringsAsFactors = FALSE)\n", - " message(\"[SKIP] \", check, \": \", reason)\n", - "}\n", - "roundtrip_same_mean <- function(s, label, k, Xn) {\n", - " f <- file.path(tmp, paste0(gsub(\"[^A-Za-z0-9]\", \"_\", label), \".json\"))\n", - " check_runs(s, paste0(label, \": save + load\"), {\n", - " save(k, f)\n", - " k2 <- rlibkriging::load(f)\n", - " check_close(s, paste0(label, \": load(save(k)) predicts the same mean\"),\n", - " predict(k2, Xn, return_stdev = FALSE)$mean, predict(k, Xn, return_stdev = FALSE)$mean, rtol = 1e-10)\n", - " })\n", - "}" - ] - }, - { - "cell_type": "markdown", - "id": "c002", - "metadata": {}, - "source": [ - "## `WarpKriging`: continuous warpings, then a categorical input\n", - "\n", - "The categorical case: x₂ restricted to 5 levels (coded 0, …, 4), embedded in ℝ²." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c003", - "metadata": {}, - "outputs": [], - "source": [ - "if (has_class(\"WarpKriging\")) {\n", - " check_runs(s, \"WarpKriging(kumaraswamy, kumaraswamy): fit + predict\", {\n", - " kw <- WarpKriging(y, X, warping = c(\"kumaraswamy\", \"kumaraswamy\"), kernel = \"matern5_2\")\n", - " pw <- predict(kw, Xt, return_stdev = TRUE)\n", - " check_less(s, \"WarpKriging(kumaraswamy): non-finite predictions\", sum(!is.finite(c(pw$mean, pw$stdev))), 1)\n", - " add_metric(s, \"branin\", \"WarpKriging kumaraswamy\", q2 = q2(yt, pw$mean))\n", - " check_greater(s, \"WarpKriging(kumaraswamy): Q2\", q2(yt, pw$mean), 0.9, severity = \"warn\")\n", - " roundtrip_same_mean(s, \"WarpKriging(kumaraswamy)\", kw, Xt)\n", - " })\n", - " lev <- function(U) cbind(U[, 1], round(U[, 2] * 4)) # level codes 0..4\n", - " unlev <- function(L) cbind(L[, 1], L[, 2] / 4)\n", - " Xc <- lev(lhs(40, 2)); yc <- branin(unlev(Xc))\n", - " Xtc <- lev(Xt); ytc <- branin(unlev(Xtc))\n", - " check_runs(s, \"WarpKriging(none, categorical(5,2)): fit + predict\", {\n", - " kc <- WarpKriging(yc, Xc, warping = c(\"none\", \"categorical(5,2)\"), kernel = \"matern5_2\")\n", - " pc <- predict(kc, Xtc, return_stdev = TRUE)\n", - " check_less(s, \"WarpKriging(categorical): non-finite predictions\", sum(!is.finite(c(pc$mean, pc$stdev))), 1)\n", - " add_metric(s, \"branin, x2 categorical\", \"WarpKriging categorical\", q2 = q2(ytc, pc$mean))\n", - " check_greater(s, \"WarpKriging(categorical): Q2\", q2(ytc, pc$mean), 0.8, severity = \"warn\")\n", - " roundtrip_same_mean(s, \"WarpKriging(categorical)\", kc, Xtc)\n", - " })\n", - "} else skip(s, \"WarpKriging\", \"not in this rlibkriging\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c004", - "metadata": {}, - "outputs": [], - "source": [ - "me <- do.call(rbind, s$metrics)\n", - "print(me[me$metric == \"q2\", c(\"case\", \"package\", \"value\")], row.names = FALSE)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c005", - "metadata": {}, - "outputs": [], - "source": [ - "finalize(s)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "R", - "language": "R", - "name": "ir" - }, - "language_info": { - "name": "R" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/warpings-categorical_WarpKriging_julia.ipynb b/warpings-categorical_WarpKriging_julia.ipynb deleted file mode 100644 index 61daf60..0000000 --- a/warpings-categorical_WarpKriging_julia.ipynb +++ /dev/null @@ -1,138 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "c000", - "metadata": {}, - "source": [ - "# Julia · `WarpKriging` — continuous warpings, a categorical input\n", - "\n", - "Branin on [0,1]² (n = 30 LHS points unless stated, Q² on 500 uniform test\n", - "points), through JLibKriging. Checks (error-level unless stated):\n", - "\n", - "* `kumaraswamy` on both inputs, then x₂ restricted to 5 levels with `categorical(5,2)`: finite predictions\n", - "* Q² floors (warn-level)\n", - "* `save` then `load`: the same predicted mean\n", - "\n", - "The same checks as `warpings-categorical_WarpKriging_R.ipynb`, with the Julia binding." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c001", - "metadata": {}, - "outputs": [], - "source": [ - "import JLibKriging as LK\n", - "using Random\n", - "include(\"tools/julia/checks.jl\")\n", - "s = Suite(\"warpings-categorical_WarpKriging_julia\")\n", - "\n", - "branin(U) = begin\n", - " x1 = 15 .* U[:, 1] .- 5; x2 = 15 .* U[:, 2]\n", - " (x2 .- 5.1 / (4π^2) .* x1 .^ 2 .+ 5 / π .* x1 .- 6) .^ 2 .+ 10 * (1 - 1 / (8π)) .* cos.(x1) .+ 10\n", - "end\n", - "rng = MersenneTwister(1)\n", - "lhs(n, d) = hcat([(randperm(rng, n) .- rand(rng, n)) ./ n for _ in 1:d]...)\n", - "X = lhs(30, 2); y = branin(X)\n", - "Xt = rand(rng, 500, 2); yt = branin(Xt)\n", - "sd(v) = sqrt(sum((v .- sum(v) / length(v)) .^ 2) / (length(v) - 1))\n", - "finite_count(v...) = count(!isfinite, vcat(v...))\n", - "\n", - "function check_runs!(f, s, check)\n", - " try\n", - " f()\n", - " catch e\n", - " add_check!(s, check, false; value=string(typeof(e)), expected=\"no error\", detail=first(sprint(showerror, e), 200))\n", - " end\n", - "end\n", - "\n", - "function roundtrip!(s, label, k, Xn)\n", - " check_runs!(s, \"$label: save + load\") do\n", - " f = joinpath(mktempdir(), \"k.json\")\n", - " LK.save(k, f)\n", - " k2 = LK.load(f)\n", - " check_close!(s, \"$label: load(save(k)) predicts the same mean\",\n", - " LK.predict(k2, Xn; return_stdev=false).mean, LK.predict(k, Xn; return_stdev=false).mean; rtol=1e-10)\n", - " end\n", - "end" - ] - }, - { - "cell_type": "markdown", - "id": "c002", - "metadata": {}, - "source": [ - "## `WarpKriging`: continuous warpings, then a categorical input\n", - "\n", - "The categorical case: x₂ restricted to 5 levels (coded 0, …, 4), embedded in ℝ²." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c003", - "metadata": {}, - "outputs": [], - "source": [ - "check_runs!(s, \"WarpKriging(kumaraswamy, kumaraswamy): fit + predict\") do\n", - " kw = LK.WarpKriging(y, X, [\"kumaraswamy\", \"kumaraswamy\"], \"matern5_2\")\n", - " pw = LK.predict(kw, Xt; return_stdev=true)\n", - " add_check!(s, \"WarpKriging(kumaraswamy): finite predictions\", finite_count(pw.mean, pw.stdev) == 0;\n", - " value=finite_count(pw.mean, pw.stdev), expected=\"0 non-finite\")\n", - " add_metric!(s, \"branin\", \"WarpKriging kumaraswamy\"; q2=q2(yt, pw.mean))\n", - " add_check!(s, \"WarpKriging(kumaraswamy): Q2\", q2(yt, pw.mean) > 0.9; value=q2(yt, pw.mean), expected=\"> 0.9\",\n", - " severity=\"warn\")\n", - " roundtrip!(s, \"WarpKriging(kumaraswamy)\", kw, Xt)\n", - "end\n", - "lev(U) = hcat(U[:, 1], round.(U[:, 2] .* 4))\n", - "unlev(L) = hcat(L[:, 1], L[:, 2] ./ 4)\n", - "Xc = lev(lhs(40, 2)); yc = branin(unlev(Xc))\n", - "Xtc = lev(Xt); ytc = branin(unlev(Xtc))\n", - "check_runs!(s, \"WarpKriging(none, categorical(5,2)): fit + predict\") do\n", - " kc = LK.WarpKriging(yc, Xc, [\"none\", \"categorical(5,2)\"], \"matern5_2\")\n", - " pc = LK.predict(kc, Xtc; return_stdev=true)\n", - " add_check!(s, \"WarpKriging(categorical): finite predictions\", finite_count(pc.mean, pc.stdev) == 0;\n", - " value=finite_count(pc.mean, pc.stdev), expected=\"0 non-finite\")\n", - " add_metric!(s, \"branin, x2 categorical\", \"WarpKriging categorical\"; q2=q2(ytc, pc.mean))\n", - " add_check!(s, \"WarpKriging(categorical): Q2\", q2(ytc, pc.mean) > 0.8; value=q2(ytc, pc.mean), expected=\"> 0.8\",\n", - " severity=\"warn\")\n", - " roundtrip!(s, \"WarpKriging(categorical)\", kc, Xtc)\n", - "end" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c004", - "metadata": {}, - "outputs": [], - "source": [ - "foreach(r -> r[5] == \"q2\" && println(rpad(r[3], 24), rpad(r[4], 28), r[6]), s.metrics)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c005", - "metadata": {}, - "outputs": [], - "source": [ - "finalize!(s)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Julia 1.11", - "language": "julia", - "name": "julia-1.11" - }, - "language_info": { - "name": "julia" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/warpings-categorical_WarpKriging_octave.ipynb b/warpings-categorical_WarpKriging_octave.ipynb deleted file mode 100644 index 877690f..0000000 --- a/warpings-categorical_WarpKriging_octave.ipynb +++ /dev/null @@ -1,144 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "c000", - "metadata": {}, - "source": [ - "# Octave · `WarpKriging` — continuous warpings, a categorical input\n", - "\n", - "Branin on [0,1]² (n = 30 LHS points unless stated, Q² on 500 uniform test\n", - "points), through mLibKriging. Checks (error-level unless stated):\n", - "\n", - "* `kumaraswamy` on both inputs, then x₂ restricted to 5 levels with `categorical(5,2)`: finite predictions\n", - "* Q² floors (warn-level)\n", - "* `save` then `load`: the same predicted mean\n", - "\n", - "The same checks as `warpings-categorical_WarpKriging_R.ipynb`, with the Octave binding.\n", - "\n", - "A class missing from the installed mLibKriging (release channel) is recorded as SKIP." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c001", - "metadata": {}, - "outputs": [], - "source": [ - "addpath(fullfile(pwd, \"tools\", \"octave\"));\n", - "s = lkb_suite(\"warpings-categorical_WarpKriging_octave\");\n", - "q2 = @(y, m) 1 - sum((y - m).^2) / sum((y - mean(y)).^2);\n", - "branin = @(U) ((15 * U(:, 2)) - 5.1 / (4 * pi^2) * (15 * U(:, 1) - 5).^2 + 5 / pi * (15 * U(:, 1) - 5) - 6).^2 ...\n", - " + 10 * (1 - 1 / (8 * pi)) * cos(15 * U(:, 1) - 5) + 10;\n", - "rand(\"seed\", 1); randn(\"seed\", 1);\n", - "function U = lhs(n, d)\n", - " U = zeros(n, d);\n", - " for j = 1:d, U(:, j) = (randperm(n)' - rand(n, 1)) / n; end\n", - "end\n", - "X = lhs(30, 2); y = branin(X);\n", - "Xt = rand(500, 2); yt = branin(Xt);\n", - "tmp = tempname(); mkdir(tmp);\n", - "has_class = @(name) exist(name) == 2; % a class file on the path\n", - "\n", - "function s = roundtrip(s, label, k, Xn, loader)\n", - " f = fullfile(tempdir(), [regexprep(label, \"[^A-Za-z0-9]\", \"_\") \".json\"]);\n", - " try\n", - " k.save(f);\n", - " k2 = loader(f);\n", - " m1 = k.predict(Xn, false, false, false);\n", - " m2 = k2.predict(Xn, false, false, false);\n", - " s = lkb_close(s, [label \": load(save(k)) predicts the same mean\"], m2, m1, 1e-10);\n", - " catch err\n", - " s = lkb_check(s, [label \": save + load\"], false, \"error\", \"no error\", err.message);\n", - " end\n", - "end\n", - "\n", - "function s = skip(s, check, reason)\n", - " s.checks(end + 1, :) = {s.name, \"octave\", check, \"info\", \"SKIP\", \"\", \"\", reason};\n", - " printf(\"[SKIP] %s: %s\\n\", check, reason);\n", - "end" - ] - }, - { - "cell_type": "markdown", - "id": "c002", - "metadata": {}, - "source": [ - "## `WarpKriging`: continuous warpings, then a categorical input\n", - "\n", - "The categorical case: x₂ restricted to 5 levels (coded 0, …, 4), embedded in ℝ²." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c003", - "metadata": {}, - "outputs": [], - "source": [ - "if has_class(\"WarpKriging\")\n", - " try\n", - " kw = WarpKriging(y, X, {\"kumaraswamy\", \"kumaraswamy\"}, \"matern5_2\");\n", - " [mw, sw] = kw.predict(Xt, true, false, false);\n", - " s = lkb_check(s, \"WarpKriging(kumaraswamy): finite predictions\", all(isfinite([mw; sw])), sum(~isfinite([mw; sw])), \"0 non-finite\");\n", - " s = lkb_metric(s, \"branin\", \"WarpKriging kumaraswamy\", \"q2\", q2(yt, mw));\n", - " s = lkb_check(s, \"WarpKriging(kumaraswamy): Q2\", q2(yt, mw) > 0.9, q2(yt, mw), \"> 0.9\", \"\", \"warn\");\n", - " s = roundtrip(s, \"WarpKriging(kumaraswamy)\", kw, Xt, @load_kriging);\n", - " catch err\n", - " s = lkb_check(s, \"WarpKriging(kumaraswamy): fit + predict\", false, \"error\", \"no error\", err.message);\n", - " end\n", - " lev = @(U) [U(:, 1), round(U(:, 2) * 4)];\n", - " unlev = @(L) [L(:, 1), L(:, 2) / 4];\n", - " Xc = lev(lhs(40, 2)); yc = branin(unlev(Xc));\n", - " Xtc = lev(Xt); ytc = branin(unlev(Xtc));\n", - " try\n", - " kc = WarpKriging(yc, Xc, {\"none\", \"categorical(5,2)\"}, \"matern5_2\");\n", - " [mc, sc] = kc.predict(Xtc, true, false, false);\n", - " s = lkb_check(s, \"WarpKriging(categorical): finite predictions\", all(isfinite([mc; sc])), sum(~isfinite([mc; sc])), \"0 non-finite\");\n", - " s = lkb_metric(s, \"branin, x2 categorical\", \"WarpKriging categorical\", \"q2\", q2(ytc, mc));\n", - " s = lkb_check(s, \"WarpKriging(categorical): Q2\", q2(ytc, mc) > 0.8, q2(ytc, mc), \"> 0.8\", \"\", \"warn\");\n", - " s = roundtrip(s, \"WarpKriging(categorical)\", kc, Xtc, @load_kriging);\n", - " catch err\n", - " s = lkb_check(s, \"WarpKriging(categorical): fit + predict\", false, \"error\", \"no error\", err.message);\n", - " end\n", - "else\n", - " s = skip(s, \"WarpKriging\", \"not in this mLibKriging\");\n", - "end" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c004", - "metadata": {}, - "outputs": [], - "source": [ - "q = strcmp(s.metrics(:, 5), \"q2\");\n", - "disp(s.metrics(q, [3, 4, 6]))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c005", - "metadata": {}, - "outputs": [], - "source": [ - "lkb_finalize(s);" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Octave", - "language": "octave", - "name": "octave" - }, - "language_info": { - "name": "octave" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} From d9956caed3b6980138e9a63f67797bf6d235efae Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 15 Oct 2026 17:14:57 +0000 Subject: [PATCH 06/16] tools/install/r.sh: drop the bench-* build before install.lib libKriging fixed install.lib (libKriging/libKriging#384, 757b1a9): rlibkriging now builds from master 41d97bc with install.lib alone (checked locally). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01QiFEE514dpwS57mcFWDeZG --- tools/install/r.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tools/install/r.sh b/tools/install/r.sh index e2f7fa2..a51aa7e 100755 --- a/tools/install/r.sh +++ b/tools/install/r.sh @@ -18,10 +18,6 @@ else CC="$(R CMD config CC)" CXX="$(R CMD config CXX)" cmake -S "$src" -B "$src/build-r" -G "Unix Makefiles" \ -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_LIBDIR=lib -DBUILD_SHARED_LIBS=off -DSTATIC_LIB=on \ -DENABLE_OCTAVE_BINDING=off -DENABLE_MATLAB_BINDING=off -DENABLE_PYTHON_BINDING=off - # install.lib also installs the bench/bench-* executables, which it does not build: build them - # first (without the tests, which the full install target would also build) - mapfile -t benches < <(cmake --build "$src/build-r" --target help | sed -n 's/^\.\.\. \(bench-[A-Za-z0-9_-]*\).*/\1/p') - cmake --build "$src/build-r" --target "${benches[@]}" -j "$(nproc)" cmake --build "$src/build-r" --target install.lib -j "$(nproc)" export LIBKRIGING_PATH="$src/build-r/installed" cd "$src/bindings/R" From 77cf621f5d8aa3a239229638b093e5c8512b94b4 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 15 Oct 2026 20:11:24 +0000 Subject: [PATCH 07/16] Gallery: every notebook a described demo with plots, tables and a summary of its checks The repository is now libKriging/gallery: worked examples of libKriging that are also its end-to-end validation. - every notebook starts with an **Idea** and a **Reference** paragraph after its title, and ends with a Summary section: lkbench Suite.summary() (metrics table, count per status, every check that did not pass), and its R / Octave / Julia equivalents (summary_suite, lkb_summary, summary!); - plots and tables where they were missing: profiles of LL / LOO / LMP along t * theta_hat (with a check that the optimum is at t ~ 1), GridSearchCV scores per kernel, predicted vs true values and the table of the values of the cross-binding notebooks (Python, R), accuracy and cost bar plots against DiceKriging / RobustGaSP; tables only for Octave and Julia (no plotting dependency there); - README rewritten as a gallery: examples grouped by use case (basics, noisy data, large datasets, warpings, deep kernels, other packages, validation only), each with what it shows and what it is validated against; URLs to libKriging/gallery; - the Colab / Binder setup cell clones libKriging/gallery; report and helper texts say gallery. Run locally against libKriging master 41d97bc: every Python notebook, the R and Octave cross-binding notebooks and vs-STK (0 FAIL); not run: Julia, vs-DiceKriging (CRAN unreachable here). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01QiFEE514dpwS57mcFWDeZG --- README.md | 180 +++++++++++------- ...gations-large-n_NestedKriging_python.ipynb | 38 +++- ...binding-reference_all-classes_python.ipynb | 73 ++++++- cross-binding_all-classes_R.ipynb | 61 +++++- cross-binding_all-classes_julia.ipynb | 55 +++++- cross-binding_all-classes_octave.ipynb | 57 +++++- ...l-projected-branin_MLPKriging_python.ipynb | 37 +++- ...n-normalize-screening_Kriging_python.ipynb | 38 +++- ...-trends-interpolation_Kriging_python.ipynb | 39 +++- large-n_Kriging-vecchia-nystrom_python.ipynb | 38 +++- noisy-data_Kriging-nugget-noise_python.ipynb | 38 +++- ...-LL-LOO-LMP-gradients_Kriging_python.ipynb | 85 ++++++++- ...stness-invalid-inputs_Kriging_python.ipynb | 38 +++- ...pickle-copy-accessors_Kriging_python.ipynb | 43 ++++- sklearn-estimators_Kriging_python.ipynb | 47 ++++- ...ons-reference-values_no-model_python.ipynb | 42 +++- tools/julia/checks.jl | 12 ++ tools/lkbench/__init__.py | 2 +- tools/lkbench/checks.py | 19 +- tools/lkbench/functions.py | 2 +- tools/notebook_setup.py | 6 +- tools/octave/lkb_summary.m | 13 ++ tools/pyproject.toml | 2 +- tools/r/checks.R | 15 ++ tools/report.py | 2 +- ...gregations-large-n_NestedKriging_python.py | 18 ++ ...ss-binding-reference_all-classes_python.py | 41 ++++ tools/src/cross-binding_all-classes_R.R | 35 ++++ tools/src/cross-binding_all-classes_julia.jl | 29 +++ tools/src/cross-binding_all-classes_octave.m | 31 +++ ...rnel-projected-branin_MLPKriging_python.py | 17 ++ ...sion-normalize-screening_Kriging_python.py | 18 ++ ...els-trends-interpolation_Kriging_python.py | 19 ++ .../large-n_Kriging-vecchia-nystrom_python.py | 18 ++ .../noisy-data_Kriging-nugget-noise_python.py | 18 ++ ...ves-LL-LOO-LMP-gradients_Kriging_python.py | 43 +++++ ...obustness-invalid-inputs_Kriging_python.py | 18 ++ ...ad-pickle-copy-accessors_Kriging_python.py | 21 ++ .../src/sklearn-estimators_Kriging_python.py | 27 +++ ...ctions-reference-values_no-model_python.py | 22 ++- tools/src/update-simulate_Kriging_python.py | 17 ++ .../src/vs-DiceKriging-RobustGaSP_Kriging_R.R | 26 ++- .../src/vs-GaussianProcesses_Kriging_julia.jl | 16 ++ tools/src/vs-STK_Kriging_octave.m | 17 ++ ...klearn-SMT-OpenTURNS-GPy_Kriging_python.py | 17 ++ ...-categorical-ordinal_WarpKriging_python.py | 18 ++ update-simulate_Kriging_python.ipynb | 37 +++- vs-DiceKriging-RobustGaSP_Kriging_R.ipynb | 40 +++- vs-GaussianProcesses_Kriging_julia.ipynb | 30 ++- vs-STK_Kriging_octave.ipynb | 31 ++- ...arn-SMT-OpenTURNS-GPy_Kriging_python.ipynb | 37 +++- ...tegorical-ordinal_WarpKriging_python.ipynb | 38 +++- 52 files changed, 1521 insertions(+), 160 deletions(-) create mode 100644 tools/octave/lkb_summary.m diff --git a/README.md b/README.md index 55db8df..df3c74d 100644 --- a/README.md +++ b/README.md @@ -1,74 +1,103 @@ -# libKriging benchmarks - -Test and benchmark notebooks for [libKriging](https://github.com/libKriging/libKriging), -kept outside the library so that they can follow both its development head and -its releases. Every notebook **checks** what it computes (error-level checks fail -the run, warn-level ones are reported), and CI executes each of them against the -**development head of libKriging and of its bindings**, saves it with its outputs, -and aggregates the findings of every language into one report. - -The notebooks are at the root of the repository, named -`__.ipynb`; everything else (notebook sources, -helpers, binding installation, CI scripts, known issues, baselines) is under -[`tools/`](tools), the shared datasets under [`data/`](data), and the Binder -environment under [`.binder/`](.binder). - -## Notebooks - -The **executed** column links to the notebook as executed by the last CI run on -`main` against libKriging master (outputs saved, failing checks included: a -notebook that fails is kept up to its failing cell), a zip of the -`executed-master-` artifact (through [nightly.link](https://nightly.link), -which serves the last artifact of a branch at a stable URL). Every run keeps -these artifacts for 90 days: [all runs on `main`](https://github.com/libKriging/benchmarks/actions/workflows/notebooks.yml?query=branch%3Amain). -All the executed notebooks of a run, in one zip: -[executed-notebooks-master](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-notebooks-master.zip); -the aggregated report with all CSV results: -[report-master](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/report-master.zip). - -One binding per theme and Kriging type: Python, the most complete binding, unless the -reference is specific to another language (DiceKriging and RobustGaSP in R, STK in -Octave, GaussianProcesses.jl in Julia). The other bindings are checked against -Python on every model class and fit method by the cross-binding notebooks. - -### Python (`pylibkriging`) - -| notebook | kriging type | theme: problems, features | reference: compared with | executed | open in | -|---|---|---|---|---|---| -| [test-functions-reference-values_no-model_python](test-functions-reference-values_no-model_python.ipynb) | — | the 21 test functions of [`tools/lkbench/functions.py`](tools/lkbench/functions.py), plotted | published optima / values | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-test-functions-reference-values_no-model_python.zip) | [Colab](https://colab.research.google.com/github/libKriging/benchmarks/blob/main/test-functions-reference-values_no-model_python.ipynb) · [Binder](https://mybinder.org/v2/gh/libKriging/benchmarks/main?labpath=test-functions-reference-values_no-model_python.ipynb) | -| [kernels-trends-interpolation_Kriging_python](kernels-trends-interpolation_Kriging_python.ipynb) | `Kriging` | 4 kernels × 4 trends on 8 functions (1D to 3D): interpolation, `predict` with covariance and derivatives | finite differences, Q² floors | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-kernels-trends-interpolation_Kriging_python.zip) | [Colab](https://colab.research.google.com/github/libKriging/benchmarks/blob/main/kernels-trends-interpolation_Kriging_python.ipynb) · [Binder](https://mybinder.org/v2/gh/libKriging/benchmarks/main?labpath=kernels-trends-interpolation_Kriging_python.ipynb) | -| [objectives-LL-LOO-LMP-gradients_Kriging_python](objectives-LL-LOO-LMP-gradients_Kriging_python.ipynb) | `Kriging` | `LL` / `LOO` / `LMP` objectives and gradients, optimality of θ̂, `leaveOneOutVec`, multistart | finite differences, explicit refits | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-objectives-LL-LOO-LMP-gradients_Kriging_python.zip) | [Colab](https://colab.research.google.com/github/libKriging/benchmarks/blob/main/objectives-LL-LOO-LMP-gradients_Kriging_python.ipynb) · [Binder](https://mybinder.org/v2/gh/libKriging/benchmarks/main?labpath=objectives-LL-LOO-LMP-gradients_Kriging_python.ipynb) | -| [update-simulate_Kriging_python](update-simulate_Kriging_python.ipynb) | `Kriging` | `update`, `simulate`, `update_simulate`, conditional paths in pictures | fit from scratch, Monte Carlo moments | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-update-simulate_Kriging_python.zip) | [Colab](https://colab.research.google.com/github/libKriging/benchmarks/blob/main/update-simulate_Kriging_python.ipynb) · [Binder](https://mybinder.org/v2/gh/libKriging/benchmarks/main?labpath=update-simulate_Kriging_python.ipynb) | -| [high-dimension-normalize-screening_Kriging_python](high-dimension-normalize-screening_Kriging_python.ipynb) | `Kriging` | 6D to 20D engineering functions, `normalize`, screening (Welch) | explicit rescaling, known active inputs | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-high-dimension-normalize-screening_Kriging_python.zip) | [Colab](https://colab.research.google.com/github/libKriging/benchmarks/blob/main/high-dimension-normalize-screening_Kriging_python.ipynb) · [Binder](https://mybinder.org/v2/gh/libKriging/benchmarks/main?labpath=high-dimension-normalize-screening_Kriging_python.ipynb) | -| [robustness-invalid-inputs_Kriging_python](robustness-invalid-inputs_Kriging_python.ipynb) | `Kriging` | invalid and degenerate inputs, ill-conditioning, scale equivariance, extrapolation, cost in q | equivariance identities, O(q) cost | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-robustness-invalid-inputs_Kriging_python.zip) | [Colab](https://colab.research.google.com/github/libKriging/benchmarks/blob/main/robustness-invalid-inputs_Kriging_python.ipynb) · [Binder](https://mybinder.org/v2/gh/libKriging/benchmarks/main?labpath=robustness-invalid-inputs_Kriging_python.ipynb) | -| [save-load-pickle-copy-accessors_Kriging_python](save-load-pickle-copy-accessors_Kriging_python.ipynb) | `Kriging` | `save`/`load`, pickle, copy, accessors, for every noise model, objective and fit method | the original model | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-save-load-pickle-copy-accessors_Kriging_python.zip) | [Colab](https://colab.research.google.com/github/libKriging/benchmarks/blob/main/save-load-pickle-copy-accessors_Kriging_python.ipynb) · [Binder](https://mybinder.org/v2/gh/libKriging/benchmarks/main?labpath=save-load-pickle-copy-accessors_Kriging_python.ipynb) | -| [sklearn-estimators_Kriging_python](sklearn-estimators_Kriging_python.ipynb) | `Kriging` | `pylibkriging.sklearn`: `Pipeline`, `GridSearchCV`, `clone`, `sample_y` | the native class | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-sklearn-estimators_Kriging_python.zip) | [Colab](https://colab.research.google.com/github/libKriging/benchmarks/blob/main/sklearn-estimators_Kriging_python.ipynb) · [Binder](https://mybinder.org/v2/gh/libKriging/benchmarks/main?labpath=sklearn-estimators_Kriging_python.ipynb) | -| [vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python](vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python.ipynb) | `Kriging` | default fits on 10 functions, 2D to 10D | scikit-learn, SMT, OpenTURNS, GPy (equality at fixed hyperparameters, accuracy, cost) | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python.zip) | [Colab](https://colab.research.google.com/github/libKriging/benchmarks/blob/main/vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python.ipynb) · [Binder](https://mybinder.org/v2/gh/libKriging/benchmarks/main?labpath=vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python.ipynb) | -| [noisy-data_Kriging-nugget-noise_python](noisy-data_Kriging-nugget-noise_python.ipynb) | `Kriging` with `noise="nugget"` or variances | estimated nugget, heteroscedastic noise, calibration, single start vs multistart | true noise, replicate averaging, interpolating fit | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-noisy-data_Kriging-nugget-noise_python.zip) | [Colab](https://colab.research.google.com/github/libKriging/benchmarks/blob/main/noisy-data_Kriging-nugget-noise_python.ipynb) · [Binder](https://mybinder.org/v2/gh/libKriging/benchmarks/main?labpath=noisy-data_Kriging-nugget-noise_python.ipynb) | -| [large-n_Kriging-vecchia-nystrom_python](large-n_Kriging-vecchia-nystrom_python.ipynb) | `Kriging` with `method="vecchia(m)"` / `"nystrom(k)"` | large n (up to 5000), convergence to the exact model as m, k grow, `subsetOfData` | exact model (limit m = n − 1, k = n), exact fit | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-large-n_Kriging-vecchia-nystrom_python.zip) | [Colab](https://colab.research.google.com/github/libKriging/benchmarks/blob/main/large-n_Kriging-vecchia-nystrom_python.ipynb) · [Binder](https://mybinder.org/v2/gh/libKriging/benchmarks/main?labpath=large-n_Kriging-vecchia-nystrom_python.ipynb) | -| [aggregations-large-n_NestedKriging_python](aggregations-large-n_NestedKriging_python.ipynb) | `NestedKriging` | `NK`, `PoE`, `gPoE`, `BCM`, `rBCM`, from 1 group to 32, large n; save/load, pickle | exact predictor (limit: 1 group), exact fit | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-aggregations-large-n_NestedKriging_python.zip) | [Colab](https://colab.research.google.com/github/libKriging/benchmarks/blob/main/aggregations-large-n_NestedKriging_python.ipynb) · [Binder](https://mybinder.org/v2/gh/libKriging/benchmarks/main?labpath=aggregations-large-n_NestedKriging_python.ipynb) | -| [warpings-categorical-ordinal_WarpKriging_python](warpings-categorical-ordinal_WarpKriging_python.ipynb) | `WarpKriging` | affine, boxcox, kumaraswamy, knots, neural_mono, categorical, ordinal warpings; save/load, pickle | plain `Kriging` | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-warpings-categorical-ordinal_WarpKriging_python.zip) | [Colab](https://colab.research.google.com/github/libKriging/benchmarks/blob/main/warpings-categorical-ordinal_WarpKriging_python.ipynb) · [Binder](https://mybinder.org/v2/gh/libKriging/benchmarks/main?labpath=warpings-categorical-ordinal_WarpKriging_python.ipynb) | -| [deep-kernel-projected-branin_MLPKriging_python](deep-kernel-projected-branin_MLPKriging_python.ipynb) | `MLPKriging` | deep kernel on a 10D function with a 2D structure, activations, multistart; save/load, pickle | plain `Kriging` | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-deep-kernel-projected-branin_MLPKriging_python.zip) | [Colab](https://colab.research.google.com/github/libKriging/benchmarks/blob/main/deep-kernel-projected-branin_MLPKriging_python.ipynb) · [Binder](https://mybinder.org/v2/gh/libKriging/benchmarks/main?labpath=deep-kernel-projected-branin_MLPKriging_python.ipynb) | -| [cross-binding-reference_all-classes_python](cross-binding-reference_all-classes_python.ipynb) | all classes | the cross-binding protocol (all classes and fit methods): the Python reference | the other bindings (report) | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-cross-binding-reference_all-classes_python.zip) | [Colab](https://colab.research.google.com/github/libKriging/benchmarks/blob/main/cross-binding-reference_all-classes_python.ipynb) · [Binder](https://mybinder.org/v2/gh/libKriging/benchmarks/main?labpath=cross-binding-reference_all-classes_python.ipynb) | - -### R, Octave, Julia (`rlibkriging`, `mLibKriging`, `JLibKriging`) - -| notebook | kriging type | theme: problems, features | reference: compared with | executed | open in | -|---|---|---|---|---|---| -| [vs-DiceKriging-RobustGaSP_Kriging_R](vs-DiceKriging-RobustGaSP_Kriging_R.ipynb) | `Kriging` | rlibkriging: default fits (Branin, Hartmann 3, Borehole) | DiceKriging (equality at fixed hyperparameters), RobustGaSP | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-vs-DiceKriging-RobustGaSP_Kriging_R.zip) | — | -| [vs-STK_Kriging_octave](vs-STK_Kriging_octave.ipynb) | `Kriging` | mLibKriging: default fits (Branin, Hartmann 3, Borehole) | STK (equality at fixed hyperparameters, Gaussian kernel) | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-vs-STK_Kriging_octave.zip) | — | -| [vs-GaussianProcesses_Kriging_julia](vs-GaussianProcesses_Kriging_julia.ipynb) | `Kriging` | JLibKriging: default fits (Branin, Hartmann 3, Borehole) | GaussianProcesses.jl | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-vs-GaussianProcesses_Kriging_julia.zip) | — | -| [cross-binding_all-classes_R](cross-binding_all-classes_R.ipynb) | all classes | the cross-binding protocol, R side | Python (report) | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-cross-binding_all-classes_R.zip) | — | -| [cross-binding_all-classes_octave](cross-binding_all-classes_octave.ipynb) | all classes | the cross-binding protocol, Octave side | Python (report) | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-cross-binding_all-classes_octave.zip) | — | -| [cross-binding_all-classes_julia](cross-binding_all-classes_julia.ipynb) | all classes | the cross-binding protocol, Julia side | Python (report) | [zip](https://nightly.link/libKriging/benchmarks/workflows/notebooks/main/executed-master-cross-binding_all-classes_julia.zip) | — | - -A notebook that compares libKriging with another package installs that package -itself, in its first cells (pip, CRAN, `Pkg.add`, or a clone of STK); a package -that cannot be installed is reported (warn-level) and its comparison skipped. - -Test functions: [`tools/lkbench/functions.py`](tools/lkbench/functions.py), after the +# libKriging gallery + +Worked examples of [libKriging](https://github.com/libKriging/libKriging), one per +use case: every model class (`Kriging` with its noise models and approximate fit +methods, `NestedKriging`, `WarpKriging`, `MLPKriging`) on classical test problems, +explained, plotted and tabulated, next to the packages it is usually compared with. + +Each example is also a **validation**: what it shows is checked against a reference +(finite differences, a fit from scratch, the exact model an approximation converges +to, another kriging package at the same hyperparameters, the other bindings). +CI runs every notebook against the **development head of libKriging and of its +bindings** (and weekly against the releases), saves it with its outputs, and turns +the checks into a report: the gallery is libKriging's end-to-end acceptance test. + +Every notebook starts with its **idea** and its **reference**, and ends with a +**summary** of its checks; error-level checks fail the run, warn-level ones are +reported (see [Checks and statuses](#checks-and-statuses)). + +## The examples + +**executed**: the notebook as run by the last CI run on `main` against libKriging +master, with its outputs (a zip of the `executed-master-` artifact through +[nightly.link](https://nightly.link), which serves the last artifact of a branch at a +stable URL; [all runs on `main`](https://github.com/libKriging/gallery/actions/workflows/notebooks.yml?query=branch%3Amain) +keep them 90 days; all of a run at once: +[executed-notebooks-master](https://nightly.link/libKriging/gallery/workflows/notebooks/main/executed-notebooks-master.zip), +and the aggregated report: [report-master](https://nightly.link/libKriging/gallery/workflows/notebooks/main/report-master.zip)). +**open in**: run it in a free hosted service (see [below](#running-a-notebook-in-colab-or-binder)). + +One binding per use case: Python, the most complete binding, unless the reference +belongs to another language (DiceKriging and RobustGaSP in R, STK in Octave, +GaussianProcesses.jl in Julia); the cross-binding notebooks check that every +binding gives the same numbers as Python, for every class and fit method. + +### Kriging: the basics + +| notebook | binding | kriging type | what it shows | validated against | executed | open in | +|---|---|---|---|---|---|---| +| [update-simulate_Kriging_python](update-simulate_Kriging_python.ipynb) | python | `Kriging` | the workflow on 1D to 6D functions: fit, `predict`, conditional paths, `update` with new data, `update_simulate`, in pictures | fit from scratch, Monte Carlo moments | [zip](https://nightly.link/libKriging/gallery/workflows/notebooks/main/executed-master-update-simulate_Kriging_python.zip) | [Colab](https://colab.research.google.com/github/libKriging/gallery/blob/main/update-simulate_Kriging_python.ipynb) · [Binder](https://mybinder.org/v2/gh/libKriging/gallery/main?labpath=update-simulate_Kriging_python.ipynb) | +| [kernels-trends-interpolation_Kriging_python](kernels-trends-interpolation_Kriging_python.ipynb) | python | `Kriging` | choosing a kernel and a trend: 4 × 4 combinations on 8 functions, interpolation, predicted covariance and derivatives | finite differences, Q² floors | [zip](https://nightly.link/libKriging/gallery/workflows/notebooks/main/executed-master-kernels-trends-interpolation_Kriging_python.zip) | [Colab](https://colab.research.google.com/github/libKriging/gallery/blob/main/kernels-trends-interpolation_Kriging_python.ipynb) · [Binder](https://mybinder.org/v2/gh/libKriging/gallery/main?labpath=kernels-trends-interpolation_Kriging_python.ipynb) | +| [objectives-LL-LOO-LMP-gradients_Kriging_python](objectives-LL-LOO-LMP-gradients_Kriging_python.ipynb) | python | `Kriging` | choosing the estimation criterion (`LL`, `LOO`, `LMP`), its profile around θ̂, multistart | finite differences, explicit leave-one-out refits | [zip](https://nightly.link/libKriging/gallery/workflows/notebooks/main/executed-master-objectives-LL-LOO-LMP-gradients_Kriging_python.zip) | [Colab](https://colab.research.google.com/github/libKriging/gallery/blob/main/objectives-LL-LOO-LMP-gradients_Kriging_python.ipynb) · [Binder](https://mybinder.org/v2/gh/libKriging/gallery/main?labpath=objectives-LL-LOO-LMP-gradients_Kriging_python.ipynb) | +| [high-dimension-normalize-screening_Kriging_python](high-dimension-normalize-screening_Kriging_python.ipynb) | python | `Kriging` | 6 to 20 inputs in physical units: `normalize`, reading the fitted θ to screen inputs (Welch) | explicit rescaling, known active inputs | [zip](https://nightly.link/libKriging/gallery/workflows/notebooks/main/executed-master-high-dimension-normalize-screening_Kriging_python.zip) | [Colab](https://colab.research.google.com/github/libKriging/gallery/blob/main/high-dimension-normalize-screening_Kriging_python.ipynb) · [Binder](https://mybinder.org/v2/gh/libKriging/gallery/main?labpath=high-dimension-normalize-screening_Kriging_python.ipynb) | +| [save-load-pickle-copy-accessors_Kriging_python](save-load-pickle-copy-accessors_Kriging_python.ipynb) | python | `Kriging` | saving, loading, pickling, copying a model; reading its settings back | the original model (to 1e-12) | [zip](https://nightly.link/libKriging/gallery/workflows/notebooks/main/executed-master-save-load-pickle-copy-accessors_Kriging_python.zip) | [Colab](https://colab.research.google.com/github/libKriging/gallery/blob/main/save-load-pickle-copy-accessors_Kriging_python.ipynb) · [Binder](https://mybinder.org/v2/gh/libKriging/gallery/main?labpath=save-load-pickle-copy-accessors_Kriging_python.ipynb) | +| [sklearn-estimators_Kriging_python](sklearn-estimators_Kriging_python.ipynb) | python | `Kriging` | libKriging inside scikit-learn: `Pipeline`, `GridSearchCV` over kernels, `cross_val_score`, `sample_y` | the native class | [zip](https://nightly.link/libKriging/gallery/workflows/notebooks/main/executed-master-sklearn-estimators_Kriging_python.zip) | [Colab](https://colab.research.google.com/github/libKriging/gallery/blob/main/sklearn-estimators_Kriging_python.ipynb) · [Binder](https://mybinder.org/v2/gh/libKriging/gallery/main?labpath=sklearn-estimators_Kriging_python.ipynb) | + +### Noisy observations + +| notebook | binding | kriging type | what it shows | validated against | executed | open in | +|---|---|---|---|---|---|---| +| [noisy-data_Kriging-nugget-noise_python](noisy-data_Kriging-nugget-noise_python.ipynb) | python | `Kriging` with `noise="nugget"` or variances | estimated nugget vs known noise variances, replicates, calibrated intervals on the noise-free function | true noise, replicate averaging, interpolating fit | [zip](https://nightly.link/libKriging/gallery/workflows/notebooks/main/executed-master-noisy-data_Kriging-nugget-noise_python.zip) | [Colab](https://colab.research.google.com/github/libKriging/gallery/blob/main/noisy-data_Kriging-nugget-noise_python.ipynb) · [Binder](https://mybinder.org/v2/gh/libKriging/gallery/main?labpath=noisy-data_Kriging-nugget-noise_python.ipynb) | + +### Large datasets + +| notebook | binding | kriging type | what it shows | validated against | executed | open in | +|---|---|---|---|---|---|---| +| [large-n_Kriging-vecchia-nystrom_python](large-n_Kriging-vecchia-nystrom_python.ipynb) | python | `Kriging` with `method="vecchia(m)"` / `"nystrom(k)"` | n up to 5000: Vecchia and Nyström fits, their convergence to the exact model as m and k grow, `subsetOfData` | exact model (limits m = n − 1, k = n), exact fit | [zip](https://nightly.link/libKriging/gallery/workflows/notebooks/main/executed-master-large-n_Kriging-vecchia-nystrom_python.zip) | [Colab](https://colab.research.google.com/github/libKriging/gallery/blob/main/large-n_Kriging-vecchia-nystrom_python.ipynb) · [Binder](https://mybinder.org/v2/gh/libKriging/gallery/main?labpath=large-n_Kriging-vecchia-nystrom_python.ipynb) | +| [aggregations-large-n_NestedKriging_python](aggregations-large-n_NestedKriging_python.ipynb) | python | `NestedKriging` | splitting a design into groups: `NK`, `PoE`, `gPoE`, `BCM`, `rBCM`, from 1 group to 32 | exact predictor (limit: 1 group), exact fit | [zip](https://nightly.link/libKriging/gallery/workflows/notebooks/main/executed-master-aggregations-large-n_NestedKriging_python.zip) | [Colab](https://colab.research.google.com/github/libKriging/gallery/blob/main/aggregations-large-n_NestedKriging_python.ipynb) · [Binder](https://mybinder.org/v2/gh/libKriging/gallery/main?labpath=aggregations-large-n_NestedKriging_python.ipynb) | + +### Non-stationary, categorical and ordinal inputs + +| notebook | binding | kriging type | what it shows | validated against | executed | open in | +|---|---|---|---|---|---|---| +| [warpings-categorical-ordinal_WarpKriging_python](warpings-categorical-ordinal_WarpKriging_python.ipynb) | python | `WarpKriging` | input warpings (affine, boxcox, kumaraswamy, knots, neural_mono), categorical and ordinal inputs | plain `Kriging` (accuracy, NLPD), `none` warping = `Kriging` | [zip](https://nightly.link/libKriging/gallery/workflows/notebooks/main/executed-master-warpings-categorical-ordinal_WarpKriging_python.zip) | [Colab](https://colab.research.google.com/github/libKriging/gallery/blob/main/warpings-categorical-ordinal_WarpKriging_python.ipynb) · [Binder](https://mybinder.org/v2/gh/libKriging/gallery/main?labpath=warpings-categorical-ordinal_WarpKriging_python.ipynb) | + +### Deep kernels + +| notebook | binding | kriging type | what it shows | validated against | executed | open in | +|---|---|---|---|---|---|---| +| [deep-kernel-projected-branin_MLPKriging_python](deep-kernel-projected-branin_MLPKriging_python.ipynb) | python | `MLPKriging` | an MLP feature map finding the 2D structure of a 10D function; activations, multistart | plain `Kriging` | [zip](https://nightly.link/libKriging/gallery/workflows/notebooks/main/executed-master-deep-kernel-projected-branin_MLPKriging_python.zip) | [Colab](https://colab.research.google.com/github/libKriging/gallery/blob/main/deep-kernel-projected-branin_MLPKriging_python.ipynb) · [Binder](https://mybinder.org/v2/gh/libKriging/gallery/main?labpath=deep-kernel-projected-branin_MLPKriging_python.ipynb) | + +### libKriging and other kriging packages + +| notebook | binding | kriging type | what it shows | validated against | executed | open in | +|---|---|---|---|---|---|---| +| [vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python](vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python.ipynb) | python | `Kriging` | same hyperparameters → same numbers, then default fits on 10 functions (accuracy, calibration, cost) | scikit-learn, SMT, OpenTURNS, GPy | [zip](https://nightly.link/libKriging/gallery/workflows/notebooks/main/executed-master-vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python.zip) | [Colab](https://colab.research.google.com/github/libKriging/gallery/blob/main/vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python.ipynb) · [Binder](https://mybinder.org/v2/gh/libKriging/gallery/main?labpath=vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python.ipynb) | +| [vs-DiceKriging-RobustGaSP_Kriging_R](vs-DiceKriging-RobustGaSP_Kriging_R.ipynb) | R | `Kriging` | rlibkriging next to the R references: equality at fixed hyperparameters, default fits | DiceKriging, RobustGaSP | [zip](https://nightly.link/libKriging/gallery/workflows/notebooks/main/executed-master-vs-DiceKriging-RobustGaSP_Kriging_R.zip) | — | +| [vs-STK_Kriging_octave](vs-STK_Kriging_octave.ipynb) | octave | `Kriging` | mLibKriging next to STK: equality at fixed hyperparameters (Gaussian kernel), default fits | STK | [zip](https://nightly.link/libKriging/gallery/workflows/notebooks/main/executed-master-vs-STK_Kriging_octave.zip) | — | +| [vs-GaussianProcesses_Kriging_julia](vs-GaussianProcesses_Kriging_julia.ipynb) | julia | `Kriging` | JLibKriging next to GaussianProcesses.jl: default fits | GaussianProcesses.jl | [zip](https://nightly.link/libKriging/gallery/workflows/notebooks/main/executed-master-vs-GaussianProcesses_Kriging_julia.zip) | — | + +### Validation only + +| notebook | binding | kriging type | what it shows | validated against | executed | open in | +|---|---|---|---|---|---|---| +| [test-functions-reference-values_no-model_python](test-functions-reference-values_no-model_python.ipynb) | python | — | the 21 test functions used throughout, plotted and checked | published optima / values | [zip](https://nightly.link/libKriging/gallery/workflows/notebooks/main/executed-master-test-functions-reference-values_no-model_python.zip) | [Colab](https://colab.research.google.com/github/libKriging/gallery/blob/main/test-functions-reference-values_no-model_python.ipynb) · [Binder](https://mybinder.org/v2/gh/libKriging/gallery/main?labpath=test-functions-reference-values_no-model_python.ipynb) | +| [robustness-invalid-inputs_Kriging_python](robustness-invalid-inputs_Kriging_python.ipynb) | python | `Kriging` | invalid and degenerate inputs, ill-conditioning, units, extrapolation, prediction cost in q | equivariance identities, O(q) cost | [zip](https://nightly.link/libKriging/gallery/workflows/notebooks/main/executed-master-robustness-invalid-inputs_Kriging_python.zip) | [Colab](https://colab.research.google.com/github/libKriging/gallery/blob/main/robustness-invalid-inputs_Kriging_python.ipynb) · [Binder](https://mybinder.org/v2/gh/libKriging/gallery/main?labpath=robustness-invalid-inputs_Kriging_python.ipynb) | +| [cross-binding-reference_all-classes_python](cross-binding-reference_all-classes_python.ipynb) | python | all classes | the same data and options in every binding, every class and fit method: the Python values | R, Octave, Julia (report) | [zip](https://nightly.link/libKriging/gallery/workflows/notebooks/main/executed-master-cross-binding-reference_all-classes_python.zip) | [Colab](https://colab.research.google.com/github/libKriging/gallery/blob/main/cross-binding-reference_all-classes_python.ipynb) · [Binder](https://mybinder.org/v2/gh/libKriging/gallery/main?labpath=cross-binding-reference_all-classes_python.ipynb) | +| [cross-binding_all-classes_R](cross-binding_all-classes_R.ipynb) | R | all classes | the R side of the cross-binding protocol | Python (report) | [zip](https://nightly.link/libKriging/gallery/workflows/notebooks/main/executed-master-cross-binding_all-classes_R.zip) | — | +| [cross-binding_all-classes_octave](cross-binding_all-classes_octave.ipynb) | octave | all classes | the Octave side | Python (report) | [zip](https://nightly.link/libKriging/gallery/workflows/notebooks/main/executed-master-cross-binding_all-classes_octave.zip) | — | +| [cross-binding_all-classes_julia](cross-binding_all-classes_julia.ipynb) | julia | all classes | the Julia side | Python (report) | [zip](https://nightly.link/libKriging/gallery/workflows/notebooks/main/executed-master-cross-binding_all-classes_julia.zip) | — | + +Notebooks are named `__.ipynb`. Those comparing +with another package install it themselves, in their first cells (pip, CRAN, +`Pkg.add`, or a clone of STK); a package that cannot be installed is reported +(warn-level) and its comparison skipped. Test functions: +[`tools/lkbench/functions.py`](tools/lkbench/functions.py), after the [Virtual Library of Simulation Experiments](https://www.sfu.ca/~ssurjano/) -(Surjanovic & Bingham). +(Surjanovic & Bingham). Everything that is not a notebook is under +[`tools/`](tools) (sources of the notebooks, helpers, installation of the bindings, +CI scripts, known issues, baselines), the shared datasets under [`data/`](data), +the Binder environment under [`.binder/`](.binder). ## Running a notebook in Colab or Binder @@ -132,16 +161,19 @@ what it needs (BLAS/LAPACK headers, Octave, R, Julia). Notebooks are generated from plain-text sources ([`tools/src/`](tools/src), cells separated by `# %%` / `# %% [markdown]`, or `% %%` in Octave), so that reviews are -plain diffs, and get the setup cell of `tools/notebook_setup.py` (Python); outputs are not committed (CI archives the executed notebooks as -artifacts, see [Notebooks](#notebooks)). +plain diffs, and get the setup cell of `tools/notebook_setup.py` (Python); outputs +are not committed (CI archives the executed notebooks as artifacts, see +[The examples](#the-examples)). ```sh python tools/py2nb.py # regenerate every notebook from tools/src/ python tools/py2nb.py --check # CI: sources and notebooks in sync, no notebook without source ``` -A new notebook is a new source in `tools/src/` named `__`, -with its suite named the same, and a line in the tables above (CI checks that +A new example is a new source in `tools/src/` named `__`, +with its suite named the same, an **Idea** and a **Reference** paragraph after its +title, plots and tables of its results, its checks, a final `## Summary` cell, and a +line in the tables above (CI checks that every notebook is linked from this README); CI picks it up automatically. Shared datasets (`data/`, used by the cross-binding notebooks of every language) diff --git a/aggregations-large-n_NestedKriging_python.ipynb b/aggregations-large-n_NestedKriging_python.ipynb index 2b79601..d69597f 100644 --- a/aggregations-large-n_NestedKriging_python.ipynb +++ b/aggregations-large-n_NestedKriging_python.ipynb @@ -7,6 +7,13 @@ "source": [ "# Python · NestedKriging — aggregations, large n\n", "\n", + "**Idea.** NestedKriging splits a large design into groups and aggregates their\n", + "predictions; how far is it from the exact predictor as the number of groups grows,\n", + "and which aggregation is closest?\n", + "\n", + "**Reference.** The exact predictor at the same θ (limit: 1 group), and the exact\n", + "fit.\n", + "\n", "`NestedKriging` partitions the design into groups, fits one `Kriging`\n", "sub-model per group (common hyperparameters), and aggregates the sub-model\n", "predictions: `NK` (nested kriging: kriging of the sub-predictors, a proper\n", @@ -45,9 +52,9 @@ "from pathlib import Path\n", "\n", "if not Path(\"tools/known_issues.csv\").is_file():\n", - " if not Path(\"benchmarks/tools/known_issues.csv\").is_file():\n", - " subprocess.run([\"git\", \"clone\", \"--depth\", \"1\", \"https://github.com/libKriging/benchmarks\"], check=True)\n", - " os.chdir(\"benchmarks\")\n", + " if not Path(\"gallery/tools/known_issues.csv\").is_file():\n", + " subprocess.run([\"git\", \"clone\", \"--depth\", \"1\", \"https://github.com/libKriging/gallery\"], check=True)\n", + " os.chdir(\"gallery\")\n", " try:\n", " import pylibkriging # noqa: F401\n", " except ImportError:\n", @@ -253,10 +260,33 @@ " lkbench.persistence.check(suite, \"NestedKriging\", lk.NestedKriging(y, X, \"matern5_2\", nb_groups=4, seed=1), Xt, pred)" ] }, + { + "cell_type": "markdown", + "id": "c011", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "The metrics of this notebook, the number of checks per status, and every check that\n", + "did not pass (FAIL: error-level, fails the run; WARN: warn-level; KNOWN: listed in\n", + "`tools/known_issues.csv`, with its issue; SKIP: feature absent from the installed version).\n", + "All of them are also written to `results/aggregations-large-n_NestedKriging_python.checks.csv` and `.metrics.csv`." + ] + }, { "cell_type": "code", "execution_count": null, - "id": "c011", + "id": "c012", + "metadata": {}, + "outputs": [], + "source": [ + "suite.summary()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c013", "metadata": {}, "outputs": [], "source": [ diff --git a/cross-binding-reference_all-classes_python.ipynb b/cross-binding-reference_all-classes_python.ipynb index 49d8aea..1d37ad3 100644 --- a/cross-binding-reference_all-classes_python.ipynb +++ b/cross-binding-reference_all-classes_python.ipynb @@ -7,6 +7,13 @@ "source": [ "# Python · all classes — cross-binding consistency: the Python reference\n", "\n", + "**Idea.** All bindings wrap the same C++ library: on the same data with the same\n", + "options, they must give the same numbers, for every model class and fit method. This\n", + "notebook writes the Python values, the reference of the comparison.\n", + "\n", + "**Reference.** The R, Octave and Julia notebooks `cross-binding_all-classes_*`,\n", + "compared by `tools/report.py`.\n", + "\n", "All bindings (Python, R, Octave, Julia) wrap the same C++ library: on the\n", "same data with the same options they must return the same numbers. Each\n", "language runs the protocol below on the shared datasets of `data/` and writes\n", @@ -58,9 +65,9 @@ "from pathlib import Path\n", "\n", "if not Path(\"tools/known_issues.csv\").is_file():\n", - " if not Path(\"benchmarks/tools/known_issues.csv\").is_file():\n", - " subprocess.run([\"git\", \"clone\", \"--depth\", \"1\", \"https://github.com/libKriging/benchmarks\"], check=True)\n", - " os.chdir(\"benchmarks\")\n", + " if not Path(\"gallery/tools/known_issues.csv\").is_file():\n", + " subprocess.run([\"git\", \"clone\", \"--depth\", \"1\", \"https://github.com/libKriging/gallery\"], check=True)\n", + " os.chdir(\"gallery\")\n", " try:\n", " import pylibkriging # noqa: F401\n", " except ImportError:\n", @@ -85,7 +92,9 @@ "source": [ "import csv\n", "\n", + "import matplotlib.pyplot as plt\n", "import numpy as np\n", + "import pandas as pd\n", "import pylibkriging as lk\n", "\n", "import lkbench\n", @@ -197,10 +206,66 @@ "print(len(rows), \"values written\")" ] }, + { + "cell_type": "markdown", + "id": "c007", + "metadata": {}, + "source": [ + "## The reference values\n", + "\n", + "Number of values per quantity and dataset (the report compares each of them\n", + "with the other bindings), and the default fits they come from: predicted vs\n", + "true values on the 50 test points." + ] + }, { "cell_type": "code", "execution_count": null, - "id": "c007", + "id": "c008", + "metadata": {}, + "outputs": [], + "source": [ + "ref = pd.DataFrame(rows, columns=[\"lang\", \"dataset\", \"quantity\", \"index\", \"value\"])\n", + "ref[\"value\"] = ref[\"value\"].astype(float)\n", + "display(ref.groupby([\"quantity\", \"dataset\"])[\"value\"].agg([\"size\", \"first\"]).unstack(\"dataset\"))\n", + "fig, axes = plt.subplots(1, len(DATASETS), figsize=(12, 3.6))\n", + "for ax, name in zip(axes, DATASETS):\n", + " yt = load(name, \"yt\").ravel()[:50]\n", + " m = ref[(ref.dataset == name) & (ref.quantity == \"fit_mean\")][\"value\"].to_numpy()\n", + " ax.plot(yt, m, \"o\", ms=4); ax.plot([yt.min(), yt.max()], [yt.min(), yt.max()], \"k--\", lw=.8)\n", + " q2 = 1 - np.sum((yt - m) ** 2) / np.sum((yt - yt.mean()) ** 2)\n", + " suite.metric(name, \"pylibkriging default fit\", q2=q2)\n", + " ax.set_title(f\"{name}: Q² = {q2:.3f}\"); ax.set_xlabel(\"true\"); ax.set_ylabel(\"predicted\")\n", + "plt.tight_layout(); plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "c009", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "The metrics of this notebook, the number of checks per status, and every check that\n", + "did not pass (FAIL: error-level, fails the run; WARN: warn-level; KNOWN: listed in\n", + "`tools/known_issues.csv`, with its issue; SKIP: feature absent from the installed version).\n", + "All of them are also written to `results/cross-binding-reference_all-classes_python.checks.csv` and `.metrics.csv`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c010", + "metadata": {}, + "outputs": [], + "source": [ + "suite.summary()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c011", "metadata": {}, "outputs": [], "source": [ diff --git a/cross-binding_all-classes_R.ipynb b/cross-binding_all-classes_R.ipynb index b5e59fd..48c3c6e 100644 --- a/cross-binding_all-classes_R.ipynb +++ b/cross-binding_all-classes_R.ipynb @@ -7,6 +7,12 @@ "source": [ "# R · all classes — cross-binding consistency: the R side\n", "\n", + "**Idea.** The R side of the cross-binding protocol: rlibkriging must give the\n", + "numbers of pylibkriging on the same data, for every model class and fit method.\n", + "\n", + "**Reference.** The Python values (`cross-binding-reference_all-\n", + "classes_python.ipynb`), compared by `tools/report.py`.\n", + "\n", "Same protocol as `cross-binding-reference_all-classes_python.ipynb` (see there), on the\n", "shared datasets of `data/`; writes `results/xbind_R.csv`, compared with the\n", "Python values by `tools/report.py`." @@ -118,10 +124,63 @@ "cat(nrow(out), \"values written\\n\")" ] }, + { + "cell_type": "markdown", + "id": "c006", + "metadata": {}, + "source": [ + "## The values written\n", + "\n", + "Number of values per quantity and dataset (each compared with Python by the\n", + "report), and the default fits they come from: predicted vs true on the 50 test points." + ] + }, { "cell_type": "code", "execution_count": null, - "id": "c006", + "id": "c007", + "metadata": {}, + "outputs": [], + "source": [ + "print(with(out, table(quantity, dataset)))\n", + "op <- par(mfrow = c(1, 3), mar = c(4, 4, 2, 1))\n", + "for (name in c(\"branin\", \"hartmann3\", \"borehole\")) {\n", + " yt <- as.numeric(read_data(name, \"yt\"))[1:50]\n", + " m <- as.numeric(out$value[out$dataset == name & out$quantity == \"fit_mean\"])\n", + " q <- q2(yt, m); add_metric(s, name, \"rlibkriging default fit\", q2 = q)\n", + " plot(yt, m, xlab = \"true\", ylab = \"predicted\", main = sprintf(\"%s: Q2 = %.3f\", name, q), pch = 16, cex = .7)\n", + " abline(0, 1, lty = 2)\n", + "}\n", + "par(op)" + ] + }, + { + "cell_type": "markdown", + "id": "c008", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "The metrics of this notebook, the number of checks per status, and every check that\n", + "did not pass (FAIL: error-level, fails the run; WARN: warn-level; KNOWN: listed in\n", + "`tools/known_issues.csv`, with its issue; SKIP: feature absent from the installed version).\n", + "All of them are also written to `results/cross-binding_all-classes_R.checks.csv` and `.metrics.csv`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c009", + "metadata": {}, + "outputs": [], + "source": [ + "summary_suite(s)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c010", "metadata": {}, "outputs": [], "source": [ diff --git a/cross-binding_all-classes_julia.ipynb b/cross-binding_all-classes_julia.ipynb index bb9c03d..dfa1879 100644 --- a/cross-binding_all-classes_julia.ipynb +++ b/cross-binding_all-classes_julia.ipynb @@ -7,6 +7,12 @@ "source": [ "# Julia · all classes — cross-binding consistency: the Julia side\n", "\n", + "**Idea.** The Julia side of the cross-binding protocol: JLibKriging must give the\n", + "numbers of pylibkriging on the same data, for every model class and fit method.\n", + "\n", + "**Reference.** The Python values (`cross-binding-reference_all-\n", + "classes_python.ipynb`), compared by `tools/report.py`.\n", + "\n", "Same protocol as `cross-binding-reference_all-classes_python.ipynb` (see there), on\n", "the shared datasets of `data/`; writes `results/xbind_julia.csv`, compared with\n", "the Python values by `tools/report.py`.\n", @@ -121,10 +127,57 @@ "println(length(rows), \" values written\")" ] }, + { + "cell_type": "markdown", + "id": "c006", + "metadata": {}, + "source": [ + "## The values written\n", + "\n", + "First value of each quantity and dataset (each compared with Python by the\n", + "report; plots are left to the Python reference, to avoid a plotting package here)." + ] + }, { "cell_type": "code", "execution_count": null, - "id": "c006", + "id": "c007", + "metadata": {}, + "outputs": [], + "source": [ + "for key in unique([(r[2], r[3]) for r in rows])\n", + " vals = [r[5] for r in rows if (r[2], r[3]) == key]\n", + " println(rpad(\"$(key[1]) $(key[2])\", 36), vals[1], \" (\", length(vals), \" values)\")\n", + "end" + ] + }, + { + "cell_type": "markdown", + "id": "c008", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "The metrics of this notebook, the number of checks per status, and every check that\n", + "did not pass (FAIL: error-level, fails the run; WARN: warn-level; KNOWN: listed in\n", + "`tools/known_issues.csv`, with its issue; SKIP: feature absent from the installed version).\n", + "All of them are also written to `results/cross-binding_all-classes_julia.checks.csv` and `.metrics.csv`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c009", + "metadata": {}, + "outputs": [], + "source": [ + "summary!(s)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c010", "metadata": {}, "outputs": [], "source": [ diff --git a/cross-binding_all-classes_octave.ipynb b/cross-binding_all-classes_octave.ipynb index b77ce9a..27aa097 100644 --- a/cross-binding_all-classes_octave.ipynb +++ b/cross-binding_all-classes_octave.ipynb @@ -7,6 +7,12 @@ "source": [ "# Octave · all classes — cross-binding consistency: the Octave side\n", "\n", + "**Idea.** The Octave side of the cross-binding protocol: mLibKriging must give the\n", + "numbers of pylibkriging on the same data, for every model class and fit method.\n", + "\n", + "**Reference.** The Python values (`cross-binding-reference_all-\n", + "classes_python.ipynb`), compared by `tools/report.py`.\n", + "\n", "Same protocol as `cross-binding-reference_all-classes_python.ipynb` (see there), on the\n", "shared datasets of `data/`; writes `results/xbind_octave.csv`, compared with\n", "the Python values by `tools/report.py`. mLibKriging must be on the path\n", @@ -137,10 +143,59 @@ "printf(\"%d values written\\n\", size(out, 1));" ] }, + { + "cell_type": "markdown", + "id": "c006", + "metadata": {}, + "source": [ + "## The values written\n", + "\n", + "First value of each quantity and dataset (each compared with Python by the\n", + "report; plots are left to the Python reference, Octave's graphics toolkit not\n", + "being available on every runner)." + ] + }, { "cell_type": "code", "execution_count": null, - "id": "c006", + "id": "c007", + "metadata": {}, + "outputs": [], + "source": [ + "keys = strcat(out(:, 2), \" \", out(:, 3));\n", + "[u, first] = unique(keys, \"first\");\n", + "for i = 1:numel(u)\n", + " printf(\"%-36s %s (%d values)\\n\", u{i}, out{first(i), 5}, sum(strcmp(keys, u{i})));\n", + "end" + ] + }, + { + "cell_type": "markdown", + "id": "c008", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "The metrics of this notebook, the number of checks per status, and every check that\n", + "did not pass (FAIL: error-level, fails the run; WARN: warn-level; KNOWN: listed in\n", + "`tools/known_issues.csv`, with its issue; SKIP: feature absent from the installed version).\n", + "All of them are also written to `results/cross-binding_all-classes_octave.checks.csv` and `.metrics.csv`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c009", + "metadata": {}, + "outputs": [], + "source": [ + "lkb_summary(s);" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c010", "metadata": {}, "outputs": [], "source": [ diff --git a/deep-kernel-projected-branin_MLPKriging_python.ipynb b/deep-kernel-projected-branin_MLPKriging_python.ipynb index 22c966d..11ec6a4 100644 --- a/deep-kernel-projected-branin_MLPKriging_python.ipynb +++ b/deep-kernel-projected-branin_MLPKriging_python.ipynb @@ -7,6 +7,12 @@ "source": [ "# Python · MLPKriging — deep kernel on a 10D function with a 2D structure\n", "\n", + "**Idea.** A deep kernel (an MLP feature map before the kernel) can find a low-\n", + "dimensional structure that a per-axis kernel cannot: Branin of a 2D projection of 10\n", + "inputs.\n", + "\n", + "**Reference.** Plain `Kriging` on the same data.\n", + "\n", "f(x) = Branin(Wx) for x ∈ [0,1]¹⁰ and a fixed 2 × 10 projection W: the\n", "function only varies in a 2D subspace, which a joint feature map (an MLP from\n", "ℝ¹⁰ to ℝ², then a stationary kernel) can find, whereas a per-axis kernel\n", @@ -35,9 +41,9 @@ "from pathlib import Path\n", "\n", "if not Path(\"tools/known_issues.csv\").is_file():\n", - " if not Path(\"benchmarks/tools/known_issues.csv\").is_file():\n", - " subprocess.run([\"git\", \"clone\", \"--depth\", \"1\", \"https://github.com/libKriging/benchmarks\"], check=True)\n", - " os.chdir(\"benchmarks\")\n", + " if not Path(\"gallery/tools/known_issues.csv\").is_file():\n", + " subprocess.run([\"git\", \"clone\", \"--depth\", \"1\", \"https://github.com/libKriging/gallery\"], check=True)\n", + " os.chdir(\"gallery\")\n", " try:\n", " import pylibkriging # noqa: F401\n", " except ImportError:\n", @@ -152,10 +158,33 @@ " kernel=\"matern5_2\"), Xt, pred)" ] }, + { + "cell_type": "markdown", + "id": "c006", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "The metrics of this notebook, the number of checks per status, and every check that\n", + "did not pass (FAIL: error-level, fails the run; WARN: warn-level; KNOWN: listed in\n", + "`tools/known_issues.csv`, with its issue; SKIP: feature absent from the installed version).\n", + "All of them are also written to `results/deep-kernel-projected-branin_MLPKriging_python.checks.csv` and `.metrics.csv`." + ] + }, { "cell_type": "code", "execution_count": null, - "id": "c006", + "id": "c007", + "metadata": {}, + "outputs": [], + "source": [ + "suite.summary()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c008", "metadata": {}, "outputs": [], "source": [ diff --git a/high-dimension-normalize-screening_Kriging_python.ipynb b/high-dimension-normalize-screening_Kriging_python.ipynb index c77f611..b3ce1a6 100644 --- a/high-dimension-normalize-screening_Kriging_python.ipynb +++ b/high-dimension-normalize-screening_Kriging_python.ipynb @@ -7,6 +7,13 @@ "source": [ "# Python · Kriging — moderate and high dimension: engineering functions, `normalize`, screening\n", "\n", + "**Idea.** From 6 to 20 inputs on engineering functions: accuracy, the effect of\n", + "`normalize` on inputs in physical units, and the ability of the fitted ranges θ to\n", + "tell influential inputs from inactive ones.\n", + "\n", + "**Reference.** The same model on inputs rescaled by hand, and the known active\n", + "inputs of the Welch function.\n", + "\n", "| function | d | character |\n", "|---|---|---|\n", "| OTL circuit | 6 | smooth, inputs on very different scales |\n", @@ -46,9 +53,9 @@ "from pathlib import Path\n", "\n", "if not Path(\"tools/known_issues.csv\").is_file():\n", - " if not Path(\"benchmarks/tools/known_issues.csv\").is_file():\n", - " subprocess.run([\"git\", \"clone\", \"--depth\", \"1\", \"https://github.com/libKriging/benchmarks\"], check=True)\n", - " os.chdir(\"benchmarks\")\n", + " if not Path(\"gallery/tools/known_issues.csv\").is_file():\n", + " subprocess.run([\"git\", \"clone\", \"--depth\", \"1\", \"https://github.com/libKriging/gallery\"], check=True)\n", + " os.chdir(\"gallery\")\n", " try:\n", " import pylibkriging # noqa: F401\n", " except ImportError:\n", @@ -169,10 +176,33 @@ "pd.Series(theta, index=[f\"x{i + 1}\" for i in range(20)]).round(3).to_frame(\"theta\").T" ] }, + { + "cell_type": "markdown", + "id": "c007", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "The metrics of this notebook, the number of checks per status, and every check that\n", + "did not pass (FAIL: error-level, fails the run; WARN: warn-level; KNOWN: listed in\n", + "`tools/known_issues.csv`, with its issue; SKIP: feature absent from the installed version).\n", + "All of them are also written to `results/high-dimension-normalize-screening_Kriging_python.checks.csv` and `.metrics.csv`." + ] + }, { "cell_type": "code", "execution_count": null, - "id": "c007", + "id": "c008", + "metadata": {}, + "outputs": [], + "source": [ + "suite.summary()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c009", "metadata": {}, "outputs": [], "source": [ diff --git a/kernels-trends-interpolation_Kriging_python.ipynb b/kernels-trends-interpolation_Kriging_python.ipynb index 1b1f150..efd2b70 100644 --- a/kernels-trends-interpolation_Kriging_python.ipynb +++ b/kernels-trends-interpolation_Kriging_python.ipynb @@ -7,6 +7,14 @@ "source": [ "# Python · Kriging — kernels × trends, interpolation, predict (covariance, derivatives)\n", "\n", + "**Idea.** The basic promise of noise-free Kriging, over the whole grid of kernels\n", + "and trends: it interpolates the data, its predictive variance vanishes at the design\n", + "points, its predicted derivatives are the derivatives of its prediction, and it is\n", + "accurate on smooth 1D to 3D functions.\n", + "\n", + "**Reference.** Finite differences of the prediction, the data themselves, and per-\n", + "function Q² floors.\n", + "\n", "Plain noise-free `Kriging`, every kernel (`gauss`, `exp`, `matern3_2`,\n", "`matern5_2`) crossed with every trend (`constant`, `linear`, `interactive`,\n", "`quadratic`) on functions of different regularity:\n", @@ -57,9 +65,9 @@ "from pathlib import Path\n", "\n", "if not Path(\"tools/known_issues.csv\").is_file():\n", - " if not Path(\"benchmarks/tools/known_issues.csv\").is_file():\n", - " subprocess.run([\"git\", \"clone\", \"--depth\", \"1\", \"https://github.com/libKriging/benchmarks\"], check=True)\n", - " os.chdir(\"benchmarks\")\n", + " if not Path(\"gallery/tools/known_issues.csv\").is_file():\n", + " subprocess.run([\"git\", \"clone\", \"--depth\", \"1\", \"https://github.com/libKriging/gallery\"], check=True)\n", + " os.chdir(\"gallery\")\n", " try:\n", " import pylibkriging # noqa: F401\n", " except ImportError:\n", @@ -306,10 +314,33 @@ " ax.legend()" ] }, + { + "cell_type": "markdown", + "id": "c014", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "The metrics of this notebook, the number of checks per status, and every check that\n", + "did not pass (FAIL: error-level, fails the run; WARN: warn-level; KNOWN: listed in\n", + "`tools/known_issues.csv`, with its issue; SKIP: feature absent from the installed version).\n", + "All of them are also written to `results/kernels-trends-interpolation_Kriging_python.checks.csv` and `.metrics.csv`." + ] + }, { "cell_type": "code", "execution_count": null, - "id": "c014", + "id": "c015", + "metadata": {}, + "outputs": [], + "source": [ + "suite.summary()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c016", "metadata": {}, "outputs": [], "source": [ diff --git a/large-n_Kriging-vecchia-nystrom_python.ipynb b/large-n_Kriging-vecchia-nystrom_python.ipynb index 4d8d3d1..a28e61f 100644 --- a/large-n_Kriging-vecchia-nystrom_python.ipynb +++ b/large-n_Kriging-vecchia-nystrom_python.ipynb @@ -7,6 +7,13 @@ "source": [ "# Python · Kriging with `method=\"vecchia\"` / `\"nystrom\"` — large n\n", "\n", + "**Idea.** Vecchia and Nyström approximations make Kriging affordable for large n;\n", + "how much accuracy do they cost, and do they converge to the exact model as their\n", + "size parameter grows?\n", + "\n", + "**Reference.** The exact model at the same θ (limits m = n − 1 and k = n), and the\n", + "exact fit.\n", + "\n", "The exact fit costs O(n³). `Kriging` offers:\n", "\n", "* `method=\"vecchia(m)\"`: Vecchia likelihood with m conditioning neighbours\n", @@ -51,9 +58,9 @@ "from pathlib import Path\n", "\n", "if not Path(\"tools/known_issues.csv\").is_file():\n", - " if not Path(\"benchmarks/tools/known_issues.csv\").is_file():\n", - " subprocess.run([\"git\", \"clone\", \"--depth\", \"1\", \"https://github.com/libKriging/benchmarks\"], check=True)\n", - " os.chdir(\"benchmarks\")\n", + " if not Path(\"gallery/tools/known_issues.csv\").is_file():\n", + " subprocess.run([\"git\", \"clone\", \"--depth\", \"1\", \"https://github.com/libKriging/gallery\"], check=True)\n", + " os.chdir(\"gallery\")\n", " try:\n", " import pylibkriging # noqa: F401\n", " except ImportError:\n", @@ -246,10 +253,33 @@ "suite.table().round(4)" ] }, + { + "cell_type": "markdown", + "id": "c009", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "The metrics of this notebook, the number of checks per status, and every check that\n", + "did not pass (FAIL: error-level, fails the run; WARN: warn-level; KNOWN: listed in\n", + "`tools/known_issues.csv`, with its issue; SKIP: feature absent from the installed version).\n", + "All of them are also written to `results/large-n_Kriging-vecchia-nystrom_python.checks.csv` and `.metrics.csv`." + ] + }, { "cell_type": "code", "execution_count": null, - "id": "c009", + "id": "c010", + "metadata": {}, + "outputs": [], + "source": [ + "suite.summary()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c011", "metadata": {}, "outputs": [], "source": [ diff --git a/noisy-data_Kriging-nugget-noise_python.ipynb b/noisy-data_Kriging-nugget-noise_python.ipynb index c895332..4fe95ab 100644 --- a/noisy-data_Kriging-nugget-noise_python.ipynb +++ b/noisy-data_Kriging-nugget-noise_python.ipynb @@ -7,6 +7,13 @@ "source": [ "# Python · Kriging with noise (`noise=\"nugget\"`, known variances) — noisy observations\n", "\n", + "**Idea.** Kriging with noisy observations: an estimated homogeneous nugget, or known\n", + "(heteroscedastic) noise variances, must recover the noise level and predict the\n", + "noise-free function with calibrated intervals.\n", + "\n", + "**Reference.** The true noise variance, replicate averaging, and the interpolating\n", + "(noise-free) fit on the same data.\n", + "\n", "`Kriging` handles noisy responses through its `noise` argument: `\"nugget\"`\n", "estimates one homogeneous noise variance, a vector gives a known variance per\n", "observation (heteroscedastic case).\n", @@ -50,9 +57,9 @@ "from pathlib import Path\n", "\n", "if not Path(\"tools/known_issues.csv\").is_file():\n", - " if not Path(\"benchmarks/tools/known_issues.csv\").is_file():\n", - " subprocess.run([\"git\", \"clone\", \"--depth\", \"1\", \"https://github.com/libKriging/benchmarks\"], check=True)\n", - " os.chdir(\"benchmarks\")\n", + " if not Path(\"gallery/tools/known_issues.csv\").is_file():\n", + " subprocess.run([\"git\", \"clone\", \"--depth\", \"1\", \"https://github.com/libKriging/gallery\"], check=True)\n", + " os.chdir(\"gallery\")\n", " try:\n", " import pylibkriging # noqa: F401\n", " except ImportError:\n", @@ -249,10 +256,33 @@ " rtol=1e-4, atol=1e-6 * np.std(yr))" ] }, + { + "cell_type": "markdown", + "id": "c011", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "The metrics of this notebook, the number of checks per status, and every check that\n", + "did not pass (FAIL: error-level, fails the run; WARN: warn-level; KNOWN: listed in\n", + "`tools/known_issues.csv`, with its issue; SKIP: feature absent from the installed version).\n", + "All of them are also written to `results/noisy-data_Kriging-nugget-noise_python.checks.csv` and `.metrics.csv`." + ] + }, { "cell_type": "code", "execution_count": null, - "id": "c011", + "id": "c012", + "metadata": {}, + "outputs": [], + "source": [ + "suite.summary()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c013", "metadata": {}, "outputs": [], "source": [ diff --git a/objectives-LL-LOO-LMP-gradients_Kriging_python.ipynb b/objectives-LL-LOO-LMP-gradients_Kriging_python.ipynb index 09656f1..62f21bf 100644 --- a/objectives-LL-LOO-LMP-gradients_Kriging_python.ipynb +++ b/objectives-LL-LOO-LMP-gradients_Kriging_python.ipynb @@ -7,6 +7,13 @@ "source": [ "# Python · Kriging — objectives (LL, LOO, LMP), their gradients and optima\n", "\n", + "**Idea.** The three estimation criteria (likelihood, leave-one-out, marginal\n", + "posterior) must be computed and differentiated correctly, and the fitted θ̂ must be\n", + "an optimum of the criterion it maximises.\n", + "\n", + "**Reference.** Finite differences of each criterion, explicit leave-one-out refits,\n", + "and the criterion profile around θ̂.\n", + "\n", "The hyperparameters θ of `Kriging` are estimated by maximizing the\n", "log-likelihood (`LL`), maximizing the log marginal posterior (`LMP`), or\n", "minimizing the leave-one-out error (`LOO`). This notebook checks the machinery\n", @@ -48,9 +55,9 @@ "from pathlib import Path\n", "\n", "if not Path(\"tools/known_issues.csv\").is_file():\n", - " if not Path(\"benchmarks/tools/known_issues.csv\").is_file():\n", - " subprocess.run([\"git\", \"clone\", \"--depth\", \"1\", \"https://github.com/libKriging/benchmarks\"], check=True)\n", - " os.chdir(\"benchmarks\")\n", + " if not Path(\"gallery/tools/known_issues.csv\").is_file():\n", + " subprocess.run([\"git\", \"clone\", \"--depth\", \"1\", \"https://github.com/libKriging/gallery\"], check=True)\n", + " os.chdir(\"gallery\")\n", " try:\n", " import pylibkriging # noqa: F401\n", " except ImportError:\n", @@ -73,6 +80,7 @@ "metadata": {}, "outputs": [], "source": [ + "import matplotlib.pyplot as plt\n", "import numpy as np\n", "import pandas as pd\n", "import pylibkriging as lk\n", @@ -189,7 +197,11 @@ "id": "c008", "metadata": {}, "source": [ - "## `logLikelihood()` == `logLikelihoodFun(theta())` (and the same for LOO, LMP)" + "## In pictures: each criterion along θ = t · θ̂ (Branin)\n", + "\n", + "Each criterion fitted on its own, then evaluated along the ray t · θ̂ through its\n", + "optimum (t = 1, dashed line): a maximum for LL and LMP, a minimum for LOO.\n", + "The three optima are not the same θ, which is why the fitted models differ." ] }, { @@ -198,6 +210,38 @@ "id": "c009", "metadata": {}, "outputs": [], + "source": [ + "tf = TESTS[\"branin\"]\n", + "X, y, _, _ = design.dataset(tf, CASES[\"branin\"], seed=2)\n", + "ts = np.exp(np.linspace(np.log(0.2), np.log(5), 61))\n", + "fig, axes = plt.subplots(1, 3, figsize=(13, 3.6))\n", + "for ax, objective in zip(axes, [\"LL\", \"LOO\", \"LMP\"]):\n", + " k = lk.Kriging(y, X, \"matern5_2\", objective=objective)\n", + " theta = np.asarray(k.theta()).ravel()\n", + " f = objective_fun(k, objective)\n", + " vals = [f(t * theta, False)[0] for t in ts]\n", + " ax.semilogx(ts, vals); ax.axvline(1, color=\"k\", ls=\"--\", lw=.8)\n", + " ax.set_title(f\"{objective}, θ̂ = {np.round(theta, 3)}\"); ax.set_xlabel(\"t (θ = t · θ̂)\")\n", + " i = int(np.argmax(vals) if SIGN[objective] > 0 else np.argmin(vals))\n", + " suite.between(f\"branin/{objective}: optimum of the profile along t * theta_hat (t)\", ts[i], 0.8, 1.25,\n", + " severity=\"warn\")\n", + "axes[0].set_ylabel(\"criterion\"); plt.tight_layout(); plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "c010", + "metadata": {}, + "source": [ + "## `logLikelihood()` == `logLikelihoodFun(theta())` (and the same for LOO, LMP)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c011", + "metadata": {}, + "outputs": [], "source": [ "tf = TESTS[\"hartmann3\"]\n", "X, y, _, _ = design.dataset(tf, 40, seed=5)\n", @@ -209,7 +253,7 @@ }, { "cell_type": "markdown", - "id": "c010", + "id": "c012", "metadata": {}, "source": [ "## Closed-form leave-one-out vs explicit refits\n", @@ -223,7 +267,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c011", + "id": "c013", "metadata": {}, "outputs": [], "source": [ @@ -255,7 +299,7 @@ }, { "cell_type": "markdown", - "id": "c012", + "id": "c014", "metadata": {}, "source": [ "## Multistart\n", @@ -267,7 +311,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c013", + "id": "c015", "metadata": {}, "outputs": [], "source": [ @@ -286,10 +330,33 @@ "suite.table().loc[list(CASES)].unstack(\"package\")" ] }, + { + "cell_type": "markdown", + "id": "c016", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "The metrics of this notebook, the number of checks per status, and every check that\n", + "did not pass (FAIL: error-level, fails the run; WARN: warn-level; KNOWN: listed in\n", + "`tools/known_issues.csv`, with its issue; SKIP: feature absent from the installed version).\n", + "All of them are also written to `results/objectives-LL-LOO-LMP-gradients_Kriging_python.checks.csv` and `.metrics.csv`." + ] + }, { "cell_type": "code", "execution_count": null, - "id": "c014", + "id": "c017", + "metadata": {}, + "outputs": [], + "source": [ + "suite.summary()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c018", "metadata": {}, "outputs": [], "source": [ diff --git a/robustness-invalid-inputs_Kriging_python.ipynb b/robustness-invalid-inputs_Kriging_python.ipynb index b1cc8a0..fa122d2 100644 --- a/robustness-invalid-inputs_Kriging_python.ipynb +++ b/robustness-invalid-inputs_Kriging_python.ipynb @@ -7,6 +7,13 @@ "source": [ "# Python · Kriging — robustness: degenerate and adversarial inputs\n", "\n", + "**Idea.** What happens off the happy path: invalid, degenerate or ill-conditioned\n", + "inputs must be rejected or handled, results must not depend on the units, and the\n", + "prediction cost must grow linearly with the number of points.\n", + "\n", + "**Reference.** Equivariance identities (rescaled inputs / outputs give rescaled\n", + "results) and the O(q) cost of `predict`.\n", + "\n", "What a user eventually feeds a surrogate model, and what must happen:\n", "\n", "| input | expected behaviour |\n", @@ -42,9 +49,9 @@ "from pathlib import Path\n", "\n", "if not Path(\"tools/known_issues.csv\").is_file():\n", - " if not Path(\"benchmarks/tools/known_issues.csv\").is_file():\n", - " subprocess.run([\"git\", \"clone\", \"--depth\", \"1\", \"https://github.com/libKriging/benchmarks\"], check=True)\n", - " os.chdir(\"benchmarks\")\n", + " if not Path(\"gallery/tools/known_issues.csv\").is_file():\n", + " subprocess.run([\"git\", \"clone\", \"--depth\", \"1\", \"https://github.com/libKriging/gallery\"], check=True)\n", + " os.chdir(\"gallery\")\n", " try:\n", " import pylibkriging # noqa: F401\n", " except ImportError:\n", @@ -277,10 +284,33 @@ "plt.show()" ] }, + { + "cell_type": "markdown", + "id": "c013", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "The metrics of this notebook, the number of checks per status, and every check that\n", + "did not pass (FAIL: error-level, fails the run; WARN: warn-level; KNOWN: listed in\n", + "`tools/known_issues.csv`, with its issue; SKIP: feature absent from the installed version).\n", + "All of them are also written to `results/robustness-invalid-inputs_Kriging_python.checks.csv` and `.metrics.csv`." + ] + }, { "cell_type": "code", "execution_count": null, - "id": "c013", + "id": "c014", + "metadata": {}, + "outputs": [], + "source": [ + "suite.summary()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c015", "metadata": {}, "outputs": [], "source": [ diff --git a/save-load-pickle-copy-accessors_Kriging_python.ipynb b/save-load-pickle-copy-accessors_Kriging_python.ipynb index 349830f..fa98abe 100644 --- a/save-load-pickle-copy-accessors_Kriging_python.ipynb +++ b/save-load-pickle-copy-accessors_Kriging_python.ipynb @@ -7,6 +7,13 @@ "source": [ "# Python · Kriging — save/load, pickle, copy, accessors\n", "\n", + "**Idea.** A model must survive a round trip (file, pickle, copy) unchanged, and\n", + "expose its settings and hyperparameters: for every noise model, objective and fit\n", + "method of `Kriging`.\n", + "\n", + "**Reference.** The original model: predictions and hyperparameters must be identical\n", + "to the last bit (1e-12).\n", + "\n", "Persistence and introspection of `Kriging`, for each of its variants (noise\n", "models, objectives, approximate fit methods):\n", "\n", @@ -19,7 +26,10 @@ "\n", "The same persistence checks for the other classes are in their own notebooks\n", "(`*_WarpKriging_python`, `*_MLPKriging_python`, `*_NestedKriging_python`); the\n", - "scikit-learn estimators in `sklearn-estimators_Kriging_python.ipynb`." + "scikit-learn estimators in `sklearn-estimators_Kriging_python.ipynb`.\n", + "\n", + "An API notebook: its results are identities (to 1e-12), so they are given as\n", + "the table of checks of the Summary rather than as plots." ] }, { @@ -39,9 +49,9 @@ "from pathlib import Path\n", "\n", "if not Path(\"tools/known_issues.csv\").is_file():\n", - " if not Path(\"benchmarks/tools/known_issues.csv\").is_file():\n", - " subprocess.run([\"git\", \"clone\", \"--depth\", \"1\", \"https://github.com/libKriging/benchmarks\"], check=True)\n", - " os.chdir(\"benchmarks\")\n", + " if not Path(\"gallery/tools/known_issues.csv\").is_file():\n", + " subprocess.run([\"git\", \"clone\", \"--depth\", \"1\", \"https://github.com/libKriging/gallery\"], check=True)\n", + " os.chdir(\"gallery\")\n", " try:\n", " import pylibkriging # noqa: F401\n", " except ImportError:\n", @@ -180,10 +190,33 @@ "suite.true(\"is_theta_estim() after an optimized fit\", bool(k.is_theta_estim()))" ] }, + { + "cell_type": "markdown", + "id": "c009", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "The metrics of this notebook, the number of checks per status, and every check that\n", + "did not pass (FAIL: error-level, fails the run; WARN: warn-level; KNOWN: listed in\n", + "`tools/known_issues.csv`, with its issue; SKIP: feature absent from the installed version).\n", + "All of them are also written to `results/save-load-pickle-copy-accessors_Kriging_python.checks.csv` and `.metrics.csv`." + ] + }, { "cell_type": "code", "execution_count": null, - "id": "c009", + "id": "c010", + "metadata": {}, + "outputs": [], + "source": [ + "suite.summary()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c011", "metadata": {}, "outputs": [], "source": [ diff --git a/sklearn-estimators_Kriging_python.ipynb b/sklearn-estimators_Kriging_python.ipynb index a065132..9a96927 100644 --- a/sklearn-estimators_Kriging_python.ipynb +++ b/sklearn-estimators_Kriging_python.ipynb @@ -7,6 +7,12 @@ "source": [ "# Python · Kriging — scikit-learn estimators (`pylibkriging.sklearn`)\n", "\n", + "**Idea.** `pylibkriging.sklearn` makes `Kriging` a scikit-learn estimator, usable in\n", + "pipelines, grid searches and cross-validation.\n", + "\n", + "**Reference.** The native `Kriging` class on the same data (same predictions), and\n", + "scikit-learn's estimator conventions.\n", + "\n", "`KrigingRegressor` (the scikit-learn face of `Kriging`) on Hartmann 3:\n", "`fit`/`predict` (scikit-learn conventions), the same predictions as the native\n", "class, `get_params`/`set_params`/`clone`, use inside `Pipeline`,\n", @@ -30,9 +36,9 @@ "from pathlib import Path\n", "\n", "if not Path(\"tools/known_issues.csv\").is_file():\n", - " if not Path(\"benchmarks/tools/known_issues.csv\").is_file():\n", - " subprocess.run([\"git\", \"clone\", \"--depth\", \"1\", \"https://github.com/libKriging/benchmarks\"], check=True)\n", - " os.chdir(\"benchmarks\")\n", + " if not Path(\"gallery/tools/known_issues.csv\").is_file():\n", + " subprocess.run([\"git\", \"clone\", \"--depth\", \"1\", \"https://github.com/libKriging/gallery\"], check=True)\n", + " os.chdir(\"gallery\")\n", " try:\n", " import pylibkriging # noqa: F401\n", " except ImportError:\n", @@ -55,7 +61,9 @@ "metadata": {}, "outputs": [], "source": [ + "import matplotlib.pyplot as plt\n", "import numpy as np\n", + "import pandas as pd\n", "import pylibkriging as lk\n", "\n", "import lkbench\n", @@ -147,6 +155,14 @@ " gs = GridSearchCV(KrigingRegressor(), {\"kernel\": [\"gauss\", \"matern3_2\", \"matern5_2\"]}, cv=3).fit(X, y)\n", " suite.true(\"GridSearchCV picked a kernel\", gs.best_params_[\"kernel\"] in (\"gauss\", \"matern3_2\", \"matern5_2\"),\n", " detail=str(gs.best_params_))\n", + " cv = pd.DataFrame(gs.cv_results_)[[\"param_kernel\", \"mean_test_score\", \"std_test_score\", \"mean_fit_time\"]]\n", + " for _, r in cv.iterrows():\n", + " suite.metric(\"hartmann3/GridSearchCV\", str(r[\"param_kernel\"]), cv_r2=r[\"mean_test_score\"],\n", + " cv_r2_sd=r[\"std_test_score\"], fit_s=r[\"mean_fit_time\"])\n", + " plt.figure(figsize=(5, 3.2))\n", + " plt.bar(cv[\"param_kernel\"].astype(str), cv[\"mean_test_score\"], yerr=cv[\"std_test_score\"], capsize=4)\n", + " plt.ylabel(\"3-fold CV R²\"); plt.title(f\"GridSearchCV(KrigingRegressor): best = {gs.best_params_['kernel']}\")\n", + " plt.ylim(min(0, cv[\"mean_test_score\"].min() - .05), 1.02); plt.show()\n", " with suite.runs(\"cross_val_score\"):\n", " sc = cross_val_score(KrigingRegressor(kernel=\"matern5_2\"), X, y, cv=4)\n", " suite.finite(\"cross_val_score finite\", sc)\n", @@ -157,10 +173,33 @@ " suite.skip(\"sklearn estimators\", \"pylibkriging.sklearn or scikit-learn not available\")" ] }, + { + "cell_type": "markdown", + "id": "c007", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "The metrics of this notebook, the number of checks per status, and every check that\n", + "did not pass (FAIL: error-level, fails the run; WARN: warn-level; KNOWN: listed in\n", + "`tools/known_issues.csv`, with its issue; SKIP: feature absent from the installed version).\n", + "All of them are also written to `results/sklearn-estimators_Kriging_python.checks.csv` and `.metrics.csv`." + ] + }, { "cell_type": "code", "execution_count": null, - "id": "c007", + "id": "c008", + "metadata": {}, + "outputs": [], + "source": [ + "suite.summary()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c009", "metadata": {}, "outputs": [], "source": [ diff --git a/test-functions-reference-values_no-model_python.ipynb b/test-functions-reference-values_no-model_python.ipynb index 0b48e9a..0be7fba 100644 --- a/test-functions-reference-values_no-model_python.ipynb +++ b/test-functions-reference-values_no-model_python.ipynb @@ -7,7 +7,14 @@ "source": [ "# Python · no model — the test functions against their published reference values\n", "\n", - "Every benchmark of this repository relies on the test functions of\n", + "**Idea.** Every example of this gallery is only as good as its test functions:\n", + "before fitting anything, each of the 21 functions is evaluated at points where its\n", + "value is published, and its range and shape are plotted.\n", + "\n", + "**Reference.** Published optima and nominal values (Virtual Library of Simulation\n", + "Experiments, Surjanovic & Bingham).\n", + "\n", + "Every notebook of this gallery relies on the test functions of\n", "`lkbench.functions` (after the [Virtual Library of Simulation Experiments](https://www.sfu.ca/~ssurjano/)).\n", "A typo in one of them would silently bias every result built on it, so they are\n", "checked first, against the reference values published on the library pages\n", @@ -31,9 +38,9 @@ "from pathlib import Path\n", "\n", "if not Path(\"tools/known_issues.csv\").is_file():\n", - " if not Path(\"benchmarks/tools/known_issues.csv\").is_file():\n", - " subprocess.run([\"git\", \"clone\", \"--depth\", \"1\", \"https://github.com/libKriging/benchmarks\"], check=True)\n", - " os.chdir(\"benchmarks\")\n", + " if not Path(\"gallery/tools/known_issues.csv\").is_file():\n", + " subprocess.run([\"git\", \"clone\", \"--depth\", \"1\", \"https://github.com/libKriging/gallery\"], check=True)\n", + " os.chdir(\"gallery\")\n", " try:\n", " import pylibkriging # noqa: F401\n", " except ImportError:\n", @@ -138,7 +145,7 @@ "## Designs\n", "\n", "The designs are seeded: the same seed gives the same points (reproducibility of\n", - "every benchmark), and the optimized LHS is a Latin hypercube (one point per\n", + "every notebook), and the optimized LHS is a Latin hypercube (one point per\n", "stratum in each dimension)." ] }, @@ -177,10 +184,33 @@ "plt.tight_layout()" ] }, + { + "cell_type": "markdown", + "id": "c010", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "The metrics of this notebook, the number of checks per status, and every check that\n", + "did not pass (FAIL: error-level, fails the run; WARN: warn-level; KNOWN: listed in\n", + "`tools/known_issues.csv`, with its issue; SKIP: feature absent from the installed version).\n", + "All of them are also written to `results/test-functions-reference-values_no-model_python.checks.csv` and `.metrics.csv`." + ] + }, { "cell_type": "code", "execution_count": null, - "id": "c010", + "id": "c011", + "metadata": {}, + "outputs": [], + "source": [ + "suite.summary()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c012", "metadata": {}, "outputs": [], "source": [ diff --git a/tools/julia/checks.jl b/tools/julia/checks.jl index 7de62f8..9d51365 100644 --- a/tools/julia/checks.jl +++ b/tools/julia/checks.jl @@ -56,6 +56,18 @@ function write_rows(f, header, rows) end end +function summary!(s::Suite) + # end-of-notebook summary: metrics, count per status, every check that did not pass + for r in s.metrics + println(rpad(r[3], 28), rpad(r[4], 28), rpad(r[5], 12), r[6]) + end + st = [c[5] for c in s.checks] + println(length(st), " checks: ", join(["$k $(count(==(k), st))" for k in ("PASS", "FAIL", "WARN", "KNOWN", "SKIP")], ", ")) + for c in s.checks + c[5] == "PASS" || println(rpad(c[5], 7), rpad(c[4], 7), c[3], ": value ", c[6], ", expected ", c[7], " ", c[8]) + end +end + function finalize!(s::Suite) add_metric!(s, "_suite", "_"; elapsed_s=time() - s.t0) write_rows(joinpath(RESULTS_DIR, s.name * ".checks.csv"), diff --git a/tools/lkbench/__init__.py b/tools/lkbench/__init__.py index 82fbb65..9e4eb50 100644 --- a/tools/lkbench/__init__.py +++ b/tools/lkbench/__init__.py @@ -1,4 +1,4 @@ -"""Helpers shared by the libKriging benchmark notebooks.""" +"""Helpers shared by the notebooks of the libKriging gallery (test functions, designs, metrics, checks).""" from . import checks, compat, design, functions, metrics, persistence # noqa: F401 from .checks import Suite # noqa: F401 from .functions import TESTS # noqa: F401 diff --git a/tools/lkbench/checks.py b/tools/lkbench/checks.py index 05770f8..b94c9a9 100644 --- a/tools/lkbench/checks.py +++ b/tools/lkbench/checks.py @@ -42,7 +42,7 @@ def _find_root(): for d in [Path.cwd(), *Path.cwd().parents]: if (d / "tools" / "known_issues.csv").is_file(): return d - raise RuntimeError("libKriging benchmarks repository not found from the working directory " + raise RuntimeError("libKriging gallery repository not found from the working directory " f"{Path.cwd()}: run the notebooks from the repository root, or set LKBENCH_ROOT") @@ -163,6 +163,23 @@ def table(self): df["value"] = pd.to_numeric(df["value"], errors="coerce") return df.pivot_table(index=["case", "package"], columns="metric", values="value", sort=False) + def summary(self): + """End-of-notebook summary: the metrics table, the count per status, and every check + that did not pass (with its value, threshold and, for KNOWN, its issue).""" + import pandas as pd + try: + from IPython.display import display + except ImportError: # outside Jupyter + display = print + if self.metrics: + display(self.table().round(4)) + df = pd.DataFrame(self.checks, columns=CHECK_FIELDS) + counts = {s: int((df["status"] == s).sum()) for s in ("PASS", "FAIL", "WARN", "KNOWN", "SKIP")} + print(f"{len(df)} checks: " + ", ".join(f"{k} {v}" for k, v in counts.items())) + others = df[df["status"] != "PASS"][["status", "severity", "check", "value", "expected", "detail"]] + if len(others): + display(others.reset_index(drop=True)) + # -- output --------------------------------------------------------------- def finalize(self): diff --git a/tools/lkbench/functions.py b/tools/lkbench/functions.py index 33e52f9..3b606dd 100644 --- a/tools/lkbench/functions.py +++ b/tools/lkbench/functions.py @@ -4,7 +4,7 @@ Every function takes ``X`` of shape (n, d) in its *native* domain and returns a length-n vector. ``TESTS`` gathers the metadata (dimension, domain, a reference point with its known value) used by ``test-functions-reference-values_no-model_python.ipynb`` -to check the implementations themselves before anything is benchmarked on them. +to check the implementations themselves before any model is fitted on them. """ from dataclasses import dataclass, field diff --git a/tools/notebook_setup.py b/tools/notebook_setup.py index 43f585e..ea00798 100644 --- a/tools/notebook_setup.py +++ b/tools/notebook_setup.py @@ -8,9 +8,9 @@ from pathlib import Path if not Path("tools/known_issues.csv").is_file(): - if not Path("benchmarks/tools/known_issues.csv").is_file(): - subprocess.run(["git", "clone", "--depth", "1", "https://github.com/libKriging/benchmarks"], check=True) - os.chdir("benchmarks") + if not Path("gallery/tools/known_issues.csv").is_file(): + subprocess.run(["git", "clone", "--depth", "1", "https://github.com/libKriging/gallery"], check=True) + os.chdir("gallery") try: import pylibkriging # noqa: F401 except ImportError: diff --git a/tools/octave/lkb_summary.m b/tools/octave/lkb_summary.m new file mode 100644 index 0000000..275bb1c --- /dev/null +++ b/tools/octave/lkb_summary.m @@ -0,0 +1,13 @@ +function lkb_summary(s) + % End-of-notebook summary: metrics, count per status, every check that did not pass. + for i = 1:size(s.metrics, 1) + printf("%-28s %-28s %-12s %s\n", s.metrics{i, 3}, s.metrics{i, 4}, s.metrics{i, 5}, s.metrics{i, 6}); + end + st = s.checks(:, 5); + printf("%d checks: PASS %d, FAIL %d, WARN %d, KNOWN %d, SKIP %d\n", numel(st), sum(strcmp(st, "PASS")), ... + sum(strcmp(st, "FAIL")), sum(strcmp(st, "WARN")), sum(strcmp(st, "KNOWN")), sum(strcmp(st, "SKIP"))); + for i = find(~strcmp(st, "PASS"))' + printf("%-6s %-6s %s: value %s, expected %s %s\n", s.checks{i, 5}, s.checks{i, 4}, s.checks{i, 3}, ... + s.checks{i, 6}, s.checks{i, 7}, s.checks{i, 8}); + end +end diff --git a/tools/pyproject.toml b/tools/pyproject.toml index 3e980ef..513f897 100644 --- a/tools/pyproject.toml +++ b/tools/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta" [project] name = "lkbench" version = "0.1.0" -description = "Helpers (test functions, designs, metrics, checks) for the libKriging benchmark notebooks" +description = "Helpers (test functions, designs, metrics, checks) for the notebooks of the libKriging gallery" requires-python = ">=3.9" dependencies = ["numpy", "scipy", "pandas", "matplotlib", "jinja2"] # jinja2: DataFrame.style diff --git a/tools/r/checks.R b/tools/r/checks.R index 9b28a67..8eb536e 100644 --- a/tools/r/checks.R +++ b/tools/r/checks.R @@ -55,6 +55,21 @@ add_metric <- function(s, case, package, ...) { package = package, metric = n, value = .fmt(v[[n]]), stringsAsFactors = FALSE) } +summary_suite <- function(s) { + # end-of-notebook summary: metrics, count per status, every check that did not pass + if (length(s$metrics) > 0) { + me <- do.call(rbind, s$metrics) + me$value <- as.numeric(me$value) + print(reshape(me[, c("case", "package", "metric", "value")], idvar = c("case", "package"), + timevar = "metric", direction = "wide"), row.names = FALSE) + } + ch <- do.call(rbind, s$checks) + print(table(factor(ch$status, levels = c("PASS", "FAIL", "WARN", "KNOWN", "SKIP")))) + other <- ch[ch$status != "PASS", c("status", "severity", "check", "value", "expected", "detail")] + if (nrow(other) > 0) print(other, row.names = FALSE) + invisible(ch) +} + finalize <- function(s) { add_metric(s, "_suite", "_", elapsed_s = as.numeric(difftime(Sys.time(), s$t0, units = "secs"))) ch <- do.call(rbind, s$checks) diff --git a/tools/report.py b/tools/report.py index 95cc9f6..6ca65c4 100644 --- a/tools/report.py +++ b/tools/report.py @@ -153,7 +153,7 @@ def main(argv=None): checks = read_all(args.results, "*.checks.csv") metrics = read_all(args.results, "*.metrics.csv") - lines = ["# libKriging benchmark report\n"] + lines = ["# libKriging gallery: validation report\n"] exit_code = 0 # 1. status per notebook diff --git a/tools/src/aggregations-large-n_NestedKriging_python.py b/tools/src/aggregations-large-n_NestedKriging_python.py index 5466a74..ebb1499 100644 --- a/tools/src/aggregations-large-n_NestedKriging_python.py +++ b/tools/src/aggregations-large-n_NestedKriging_python.py @@ -1,6 +1,13 @@ # %% [markdown] # # Python · NestedKriging — aggregations, large n # +# **Idea.** NestedKriging splits a large design into groups and aggregates their +# predictions; how far is it from the exact predictor as the number of groups grows, +# and which aggregation is closest? +# +# **Reference.** The exact predictor at the same θ (limit: 1 group), and the exact +# fit. +# # `NestedKriging` partitions the design into groups, fits one `Kriging` # sub-model per group (common hyperparameters), and aggregates the sub-model # predictions: `NK` (nested kriging: kriging of the sub-predictors, a proper @@ -156,5 +163,16 @@ def timed(f): with suite.runs("NestedKriging: fit (persistence)"): lkbench.persistence.check(suite, "NestedKriging", lk.NestedKriging(y, X, "matern5_2", nb_groups=4, seed=1), Xt, pred) +# %% [markdown] +# ## Summary +# +# The metrics of this notebook, the number of checks per status, and every check that +# did not pass (FAIL: error-level, fails the run; WARN: warn-level; KNOWN: listed in +# `tools/known_issues.csv`, with its issue; SKIP: feature absent from the installed version). +# All of them are also written to `results/aggregations-large-n_NestedKriging_python.checks.csv` and `.metrics.csv`. + +# %% +suite.summary() + # %% suite.finalize() diff --git a/tools/src/cross-binding-reference_all-classes_python.py b/tools/src/cross-binding-reference_all-classes_python.py index 66de287..10045b4 100644 --- a/tools/src/cross-binding-reference_all-classes_python.py +++ b/tools/src/cross-binding-reference_all-classes_python.py @@ -1,6 +1,13 @@ # %% [markdown] # # Python · all classes — cross-binding consistency: the Python reference # +# **Idea.** All bindings wrap the same C++ library: on the same data with the same +# options, they must give the same numbers, for every model class and fit method. This +# notebook writes the Python values, the reference of the comparison. +# +# **Reference.** The R, Octave and Julia notebooks `cross-binding_all-classes_*`, +# compared by `tools/report.py`. +# # All bindings (Python, R, Octave, Julia) wrap the same C++ library: on the # same data with the same options they must return the same numbers. Each # language runs the protocol below on the shared datasets of `data/` and writes @@ -37,7 +44,9 @@ # %% import csv +import matplotlib.pyplot as plt import numpy as np +import pandas as pd import pylibkriging as lk import lkbench @@ -122,5 +131,37 @@ def load(name, stem): w.writerows(rows) print(len(rows), "values written") +# %% [markdown] +# ## The reference values +# +# Number of values per quantity and dataset (the report compares each of them +# with the other bindings), and the default fits they come from: predicted vs +# true values on the 50 test points. + +# %% +ref = pd.DataFrame(rows, columns=["lang", "dataset", "quantity", "index", "value"]) +ref["value"] = ref["value"].astype(float) +display(ref.groupby(["quantity", "dataset"])["value"].agg(["size", "first"]).unstack("dataset")) +fig, axes = plt.subplots(1, len(DATASETS), figsize=(12, 3.6)) +for ax, name in zip(axes, DATASETS): + yt = load(name, "yt").ravel()[:50] + m = ref[(ref.dataset == name) & (ref.quantity == "fit_mean")]["value"].to_numpy() + ax.plot(yt, m, "o", ms=4); ax.plot([yt.min(), yt.max()], [yt.min(), yt.max()], "k--", lw=.8) + q2 = 1 - np.sum((yt - m) ** 2) / np.sum((yt - yt.mean()) ** 2) + suite.metric(name, "pylibkriging default fit", q2=q2) + ax.set_title(f"{name}: Q² = {q2:.3f}"); ax.set_xlabel("true"); ax.set_ylabel("predicted") +plt.tight_layout(); plt.show() + +# %% [markdown] +# ## Summary +# +# The metrics of this notebook, the number of checks per status, and every check that +# did not pass (FAIL: error-level, fails the run; WARN: warn-level; KNOWN: listed in +# `tools/known_issues.csv`, with its issue; SKIP: feature absent from the installed version). +# All of them are also written to `results/cross-binding-reference_all-classes_python.checks.csv` and `.metrics.csv`. + +# %% +suite.summary() + # %% suite.finalize() diff --git a/tools/src/cross-binding_all-classes_R.R b/tools/src/cross-binding_all-classes_R.R index 0bdb655..26a00ed 100644 --- a/tools/src/cross-binding_all-classes_R.R +++ b/tools/src/cross-binding_all-classes_R.R @@ -1,6 +1,12 @@ # %% [markdown] # # R · all classes — cross-binding consistency: the R side # +# **Idea.** The R side of the cross-binding protocol: rlibkriging must give the +# numbers of pylibkriging on the same data, for every model class and fit method. +# +# **Reference.** The Python values (`cross-binding-reference_all- +# classes_python.ipynb`), compared by `tools/report.py`. +# # Same protocol as `cross-binding-reference_all-classes_python.ipynb` (see there), on the # shared datasets of `data/`; writes `results/xbind_R.csv`, compared with the # Python values by `tools/report.py`. @@ -78,5 +84,34 @@ out <- do.call(rbind, rows) write.csv(out, file.path(results_dir, "xbind_R.csv"), row.names = FALSE, quote = FALSE) cat(nrow(out), "values written\n") +# %% [markdown] +# ## The values written +# +# Number of values per quantity and dataset (each compared with Python by the +# report), and the default fits they come from: predicted vs true on the 50 test points. + +# %% +print(with(out, table(quantity, dataset))) +op <- par(mfrow = c(1, 3), mar = c(4, 4, 2, 1)) +for (name in c("branin", "hartmann3", "borehole")) { + yt <- as.numeric(read_data(name, "yt"))[1:50] + m <- as.numeric(out$value[out$dataset == name & out$quantity == "fit_mean"]) + q <- q2(yt, m); add_metric(s, name, "rlibkriging default fit", q2 = q) + plot(yt, m, xlab = "true", ylab = "predicted", main = sprintf("%s: Q2 = %.3f", name, q), pch = 16, cex = .7) + abline(0, 1, lty = 2) +} +par(op) + +# %% [markdown] +# ## Summary +# +# The metrics of this notebook, the number of checks per status, and every check that +# did not pass (FAIL: error-level, fails the run; WARN: warn-level; KNOWN: listed in +# `tools/known_issues.csv`, with its issue; SKIP: feature absent from the installed version). +# All of them are also written to `results/cross-binding_all-classes_R.checks.csv` and `.metrics.csv`. + +# %% +summary_suite(s) + # %% finalize(s) diff --git a/tools/src/cross-binding_all-classes_julia.jl b/tools/src/cross-binding_all-classes_julia.jl index 821cdbf..780b889 100644 --- a/tools/src/cross-binding_all-classes_julia.jl +++ b/tools/src/cross-binding_all-classes_julia.jl @@ -1,6 +1,12 @@ # %% [markdown] # # Julia · all classes — cross-binding consistency: the Julia side # +# **Idea.** The Julia side of the cross-binding protocol: JLibKriging must give the +# numbers of pylibkriging on the same data, for every model class and fit method. +# +# **Reference.** The Python values (`cross-binding-reference_all- +# classes_python.ipynb`), compared by `tools/report.py`. +# # Same protocol as `cross-binding-reference_all-classes_python.ipynb` (see there), on # the shared datasets of `data/`; writes `results/xbind_julia.csv`, compared with # the Python values by `tools/report.py`. @@ -81,5 +87,28 @@ open(joinpath(RESULTS_DIR, "xbind_julia.csv"), "w") do io end println(length(rows), " values written") +# %% [markdown] +# ## The values written +# +# First value of each quantity and dataset (each compared with Python by the +# report; plots are left to the Python reference, to avoid a plotting package here). + +# %% +for key in unique([(r[2], r[3]) for r in rows]) + vals = [r[5] for r in rows if (r[2], r[3]) == key] + println(rpad("$(key[1]) $(key[2])", 36), vals[1], " (", length(vals), " values)") +end + +# %% [markdown] +# ## Summary +# +# The metrics of this notebook, the number of checks per status, and every check that +# did not pass (FAIL: error-level, fails the run; WARN: warn-level; KNOWN: listed in +# `tools/known_issues.csv`, with its issue; SKIP: feature absent from the installed version). +# All of them are also written to `results/cross-binding_all-classes_julia.checks.csv` and `.metrics.csv`. + +# %% +summary!(s) + # %% finalize!(s) diff --git a/tools/src/cross-binding_all-classes_octave.m b/tools/src/cross-binding_all-classes_octave.m index a562aed..c572709 100644 --- a/tools/src/cross-binding_all-classes_octave.m +++ b/tools/src/cross-binding_all-classes_octave.m @@ -1,6 +1,12 @@ % %% [markdown] % # Octave · all classes — cross-binding consistency: the Octave side % +% **Idea.** The Octave side of the cross-binding protocol: mLibKriging must give the +% numbers of pylibkriging on the same data, for every model class and fit method. +% +% **Reference.** The Python values (`cross-binding-reference_all- +% classes_python.ipynb`), compared by `tools/report.py`. +% % Same protocol as `cross-binding-reference_all-classes_python.ipynb` (see there), on the % shared datasets of `data/`; writes `results/xbind_octave.csv`, compared with % the Python values by `tools/report.py`. mLibKriging must be on the path @@ -97,5 +103,30 @@ fclose(fid); printf("%d values written\n", size(out, 1)); +% %% [markdown] +% ## The values written +% +% First value of each quantity and dataset (each compared with Python by the +% report; plots are left to the Python reference, Octave's graphics toolkit not +% being available on every runner). + +% %% +keys = strcat(out(:, 2), " ", out(:, 3)); +[u, first] = unique(keys, "first"); +for i = 1:numel(u) + printf("%-36s %s (%d values)\n", u{i}, out{first(i), 5}, sum(strcmp(keys, u{i}))); +end + +% %% [markdown] +% ## Summary +% +% The metrics of this notebook, the number of checks per status, and every check that +% did not pass (FAIL: error-level, fails the run; WARN: warn-level; KNOWN: listed in +% `tools/known_issues.csv`, with its issue; SKIP: feature absent from the installed version). +% All of them are also written to `results/cross-binding_all-classes_octave.checks.csv` and `.metrics.csv`. + +% %% +lkb_summary(s); + % %% lkb_finalize(s); diff --git a/tools/src/deep-kernel-projected-branin_MLPKriging_python.py b/tools/src/deep-kernel-projected-branin_MLPKriging_python.py index 037ae12..7ca0393 100644 --- a/tools/src/deep-kernel-projected-branin_MLPKriging_python.py +++ b/tools/src/deep-kernel-projected-branin_MLPKriging_python.py @@ -1,6 +1,12 @@ # %% [markdown] # # Python · MLPKriging — deep kernel on a 10D function with a 2D structure # +# **Idea.** A deep kernel (an MLP feature map before the kernel) can find a low- +# dimensional structure that a per-axis kernel cannot: Branin of a 2D projection of 10 +# inputs. +# +# **Reference.** Plain `Kriging` on the same data. +# # f(x) = Branin(Wx) for x ∈ [0,1]¹⁰ and a fixed 2 × 10 projection W: the # function only varies in a 2D subspace, which a joint feature map (an MLP from # ℝ¹⁰ to ℝ², then a stationary kernel) can find, whereas a per-axis kernel @@ -84,5 +90,16 @@ def projected(X): lkbench.persistence.check(suite, "MLPKriging", lk.MLPKriging(y, X, hidden_dims=[8], d_out=2, activation="tanh", kernel="matern5_2"), Xt, pred) +# %% [markdown] +# ## Summary +# +# The metrics of this notebook, the number of checks per status, and every check that +# did not pass (FAIL: error-level, fails the run; WARN: warn-level; KNOWN: listed in +# `tools/known_issues.csv`, with its issue; SKIP: feature absent from the installed version). +# All of them are also written to `results/deep-kernel-projected-branin_MLPKriging_python.checks.csv` and `.metrics.csv`. + +# %% +suite.summary() + # %% suite.finalize() diff --git a/tools/src/high-dimension-normalize-screening_Kriging_python.py b/tools/src/high-dimension-normalize-screening_Kriging_python.py index e332dd0..a4044fd 100644 --- a/tools/src/high-dimension-normalize-screening_Kriging_python.py +++ b/tools/src/high-dimension-normalize-screening_Kriging_python.py @@ -1,6 +1,13 @@ # %% [markdown] # # Python · Kriging — moderate and high dimension: engineering functions, `normalize`, screening # +# **Idea.** From 6 to 20 inputs on engineering functions: accuracy, the effect of +# `normalize` on inputs in physical units, and the ability of the fitted ranges θ to +# tell influential inputs from inactive ones. +# +# **Reference.** The same model on inputs rescaled by hand, and the known active +# inputs of the Welch function. +# # | function | d | character | # |---|---|---| # | OTL circuit | 6 | smooth, inputs on very different scales | @@ -96,5 +103,16 @@ def pred(k, X): plt.title("Welch (20D): red = most influential inputs, gray = inactive (x8, x16)"); plt.show() pd.Series(theta, index=[f"x{i + 1}" for i in range(20)]).round(3).to_frame("theta").T +# %% [markdown] +# ## Summary +# +# The metrics of this notebook, the number of checks per status, and every check that +# did not pass (FAIL: error-level, fails the run; WARN: warn-level; KNOWN: listed in +# `tools/known_issues.csv`, with its issue; SKIP: feature absent from the installed version). +# All of them are also written to `results/high-dimension-normalize-screening_Kriging_python.checks.csv` and `.metrics.csv`. + +# %% +suite.summary() + # %% suite.finalize() diff --git a/tools/src/kernels-trends-interpolation_Kriging_python.py b/tools/src/kernels-trends-interpolation_Kriging_python.py index c5f295b..2272316 100644 --- a/tools/src/kernels-trends-interpolation_Kriging_python.py +++ b/tools/src/kernels-trends-interpolation_Kriging_python.py @@ -1,6 +1,14 @@ # %% [markdown] # # Python · Kriging — kernels × trends, interpolation, predict (covariance, derivatives) # +# **Idea.** The basic promise of noise-free Kriging, over the whole grid of kernels +# and trends: it interpolates the data, its predictive variance vanishes at the design +# points, its predicted derivatives are the derivatives of its prediction, and it is +# accurate on smooth 1D to 3D functions. +# +# **Reference.** Finite differences of the prediction, the data themselves, and per- +# function Q² floors. +# # Plain noise-free `Kriging`, every kernel (`gauss`, `exp`, `matern3_2`, # `matern5_2`) crossed with every trend (`constant`, `linear`, `interactive`, # `quadratic`) on functions of different regularity: @@ -190,5 +198,16 @@ def predict(model, X, **kw): ax.set_title(fname) ax.legend() +# %% [markdown] +# ## Summary +# +# The metrics of this notebook, the number of checks per status, and every check that +# did not pass (FAIL: error-level, fails the run; WARN: warn-level; KNOWN: listed in +# `tools/known_issues.csv`, with its issue; SKIP: feature absent from the installed version). +# All of them are also written to `results/kernels-trends-interpolation_Kriging_python.checks.csv` and `.metrics.csv`. + +# %% +suite.summary() + # %% suite.finalize() diff --git a/tools/src/large-n_Kriging-vecchia-nystrom_python.py b/tools/src/large-n_Kriging-vecchia-nystrom_python.py index 71fde68..4ee296e 100644 --- a/tools/src/large-n_Kriging-vecchia-nystrom_python.py +++ b/tools/src/large-n_Kriging-vecchia-nystrom_python.py @@ -1,6 +1,13 @@ # %% [markdown] # # Python · Kriging with `method="vecchia"` / `"nystrom"` — large n # +# **Idea.** Vecchia and Nyström approximations make Kriging affordable for large n; +# how much accuracy do they cost, and do they converge to the exact model as their +# size parameter grows? +# +# **Reference.** The exact model at the same θ (limits m = n − 1 and k = n), and the +# exact fit. +# # The exact fit costs O(n³). `Kriging` offers: # # * `method="vecchia(m)"`: Vecchia likelihood with m conditioning neighbours @@ -161,5 +168,16 @@ def sod(Xa, ya): suite.less(f"{case}/{label}: fit time / exact fit time", t / t_exact, 1.0, severity="warn") suite.table().round(4) +# %% [markdown] +# ## Summary +# +# The metrics of this notebook, the number of checks per status, and every check that +# did not pass (FAIL: error-level, fails the run; WARN: warn-level; KNOWN: listed in +# `tools/known_issues.csv`, with its issue; SKIP: feature absent from the installed version). +# All of them are also written to `results/large-n_Kriging-vecchia-nystrom_python.checks.csv` and `.metrics.csv`. + +# %% +suite.summary() + # %% suite.finalize() diff --git a/tools/src/noisy-data_Kriging-nugget-noise_python.py b/tools/src/noisy-data_Kriging-nugget-noise_python.py index bf9ab12..84a198f 100644 --- a/tools/src/noisy-data_Kriging-nugget-noise_python.py +++ b/tools/src/noisy-data_Kriging-nugget-noise_python.py @@ -1,6 +1,13 @@ # %% [markdown] # # Python · Kriging with noise (`noise="nugget"`, known variances) — noisy observations # +# **Idea.** Kriging with noisy observations: an estimated homogeneous nugget, or known +# (heteroscedastic) noise variances, must recover the noise level and predict the +# noise-free function with calibrated intervals. +# +# **Reference.** The true noise variance, replicate averaging, and the interpolating +# (noise-free) fit on the same data. +# # `Kriging` handles noisy responses through its `noise` argument: `"nugget"` # estimates one homogeneous noise variance, a vector gives a known variance per # observation (heteroscedastic case). @@ -152,5 +159,16 @@ def pred(k, X): suite.close("replicates == averages with variance/r (stdev)", pred(k_rep, Xt)[1], pred(k_avg, Xt)[1], rtol=1e-4, atol=1e-6 * np.std(yr)) +# %% [markdown] +# ## Summary +# +# The metrics of this notebook, the number of checks per status, and every check that +# did not pass (FAIL: error-level, fails the run; WARN: warn-level; KNOWN: listed in +# `tools/known_issues.csv`, with its issue; SKIP: feature absent from the installed version). +# All of them are also written to `results/noisy-data_Kriging-nugget-noise_python.checks.csv` and `.metrics.csv`. + +# %% +suite.summary() + # %% suite.finalize() diff --git a/tools/src/objectives-LL-LOO-LMP-gradients_Kriging_python.py b/tools/src/objectives-LL-LOO-LMP-gradients_Kriging_python.py index 4a986d2..8fa8b61 100644 --- a/tools/src/objectives-LL-LOO-LMP-gradients_Kriging_python.py +++ b/tools/src/objectives-LL-LOO-LMP-gradients_Kriging_python.py @@ -1,6 +1,13 @@ # %% [markdown] # # Python · Kriging — objectives (LL, LOO, LMP), their gradients and optima # +# **Idea.** The three estimation criteria (likelihood, leave-one-out, marginal +# posterior) must be computed and differentiated correctly, and the fitted θ̂ must be +# an optimum of the criterion it maximises. +# +# **Reference.** Finite differences of each criterion, explicit leave-one-out refits, +# and the criterion profile around θ̂. +# # The hyperparameters θ of `Kriging` are estimated by maximizing the # log-likelihood (`LL`), maximizing the log marginal posterior (`LMP`), or # minimizing the leave-one-out error (`LOO`). This notebook checks the machinery @@ -25,6 +32,7 @@ # Functions: Branin (2D), Hartmann 3 (3D), Ishigami (3D), Friedman (5D). # %% +import matplotlib.pyplot as plt import numpy as np import pandas as pd import pylibkriging as lk @@ -104,6 +112,30 @@ def objective_fun(k, objective): tab[["function", "objective"]] = tab["case"].str.split("/", expand=True) tab.pivot_table(index="function", columns="objective", values=["q2", "coverage95"]).round(3) +# %% [markdown] +# ## In pictures: each criterion along θ = t · θ̂ (Branin) +# +# Each criterion fitted on its own, then evaluated along the ray t · θ̂ through its +# optimum (t = 1, dashed line): a maximum for LL and LMP, a minimum for LOO. +# The three optima are not the same θ, which is why the fitted models differ. + +# %% +tf = TESTS["branin"] +X, y, _, _ = design.dataset(tf, CASES["branin"], seed=2) +ts = np.exp(np.linspace(np.log(0.2), np.log(5), 61)) +fig, axes = plt.subplots(1, 3, figsize=(13, 3.6)) +for ax, objective in zip(axes, ["LL", "LOO", "LMP"]): + k = lk.Kriging(y, X, "matern5_2", objective=objective) + theta = np.asarray(k.theta()).ravel() + f = objective_fun(k, objective) + vals = [f(t * theta, False)[0] for t in ts] + ax.semilogx(ts, vals); ax.axvline(1, color="k", ls="--", lw=.8) + ax.set_title(f"{objective}, θ̂ = {np.round(theta, 3)}"); ax.set_xlabel("t (θ = t · θ̂)") + i = int(np.argmax(vals) if SIGN[objective] > 0 else np.argmin(vals)) + suite.between(f"branin/{objective}: optimum of the profile along t * theta_hat (t)", ts[i], 0.8, 1.25, + severity="warn") +axes[0].set_ylabel("criterion"); plt.tight_layout(); plt.show() + # %% [markdown] # ## `logLikelihood()` == `logLikelihoodFun(theta())` (and the same for LOO, LMP) @@ -170,5 +202,16 @@ def objective_fun(k, objective): severity="warn") suite.table().loc[list(CASES)].unstack("package") +# %% [markdown] +# ## Summary +# +# The metrics of this notebook, the number of checks per status, and every check that +# did not pass (FAIL: error-level, fails the run; WARN: warn-level; KNOWN: listed in +# `tools/known_issues.csv`, with its issue; SKIP: feature absent from the installed version). +# All of them are also written to `results/objectives-LL-LOO-LMP-gradients_Kriging_python.checks.csv` and `.metrics.csv`. + +# %% +suite.summary() + # %% suite.finalize() diff --git a/tools/src/robustness-invalid-inputs_Kriging_python.py b/tools/src/robustness-invalid-inputs_Kriging_python.py index 75490d7..fbdcb25 100644 --- a/tools/src/robustness-invalid-inputs_Kriging_python.py +++ b/tools/src/robustness-invalid-inputs_Kriging_python.py @@ -1,6 +1,13 @@ # %% [markdown] # # Python · Kriging — robustness: degenerate and adversarial inputs # +# **Idea.** What happens off the happy path: invalid, degenerate or ill-conditioned +# inputs must be rejected or handled, results must not depend on the units, and the +# prediction cost must grow linearly with the number of points. +# +# **Reference.** Equivariance identities (rescaled inputs / outputs give rescaled +# results) and the O(q) cost of `predict`. +# # What a user eventually feeds a surrogate model, and what must happen: # # | input | expected behaviour | @@ -168,5 +175,16 @@ def finite_model(check, make, Xn=Xt, severity="error"): plt.xlabel("q (prediction points)"); plt.ylabel("time (s)"); plt.legend(); plt.grid(True, which="both", alpha=.3) plt.show() +# %% [markdown] +# ## Summary +# +# The metrics of this notebook, the number of checks per status, and every check that +# did not pass (FAIL: error-level, fails the run; WARN: warn-level; KNOWN: listed in +# `tools/known_issues.csv`, with its issue; SKIP: feature absent from the installed version). +# All of them are also written to `results/robustness-invalid-inputs_Kriging_python.checks.csv` and `.metrics.csv`. + +# %% +suite.summary() + # %% suite.finalize() diff --git a/tools/src/save-load-pickle-copy-accessors_Kriging_python.py b/tools/src/save-load-pickle-copy-accessors_Kriging_python.py index e609406..c474fb6 100644 --- a/tools/src/save-load-pickle-copy-accessors_Kriging_python.py +++ b/tools/src/save-load-pickle-copy-accessors_Kriging_python.py @@ -1,6 +1,13 @@ # %% [markdown] # # Python · Kriging — save/load, pickle, copy, accessors # +# **Idea.** A model must survive a round trip (file, pickle, copy) unchanged, and +# expose its settings and hyperparameters: for every noise model, objective and fit +# method of `Kriging`. +# +# **Reference.** The original model: predictions and hyperparameters must be identical +# to the last bit (1e-12). +# # Persistence and introspection of `Kriging`, for each of its variants (noise # models, objectives, approximate fit methods): # @@ -14,6 +21,9 @@ # The same persistence checks for the other classes are in their own notebooks # (`*_WarpKriging_python`, `*_MLPKriging_python`, `*_NestedKriging_python`); the # scikit-learn estimators in `sklearn-estimators_Kriging_python.ipynb`. +# +# An API notebook: its results are identities (to 1e-12), so they are given as +# the table of checks of the Summary rather than as plots. # %% import copy @@ -95,5 +105,16 @@ def predict_any(k, Xn): suite.true("summary() is a non-empty string", isinstance(k.summary(), str) and len(k.summary()) > 20) suite.true("is_theta_estim() after an optimized fit", bool(k.is_theta_estim())) +# %% [markdown] +# ## Summary +# +# The metrics of this notebook, the number of checks per status, and every check that +# did not pass (FAIL: error-level, fails the run; WARN: warn-level; KNOWN: listed in +# `tools/known_issues.csv`, with its issue; SKIP: feature absent from the installed version). +# All of them are also written to `results/save-load-pickle-copy-accessors_Kriging_python.checks.csv` and `.metrics.csv`. + +# %% +suite.summary() + # %% suite.finalize() diff --git a/tools/src/sklearn-estimators_Kriging_python.py b/tools/src/sklearn-estimators_Kriging_python.py index 80df91d..f4078c4 100644 --- a/tools/src/sklearn-estimators_Kriging_python.py +++ b/tools/src/sklearn-estimators_Kriging_python.py @@ -1,13 +1,21 @@ # %% [markdown] # # Python · Kriging — scikit-learn estimators (`pylibkriging.sklearn`) # +# **Idea.** `pylibkriging.sklearn` makes `Kriging` a scikit-learn estimator, usable in +# pipelines, grid searches and cross-validation. +# +# **Reference.** The native `Kriging` class on the same data (same predictions), and +# scikit-learn's estimator conventions. +# # `KrigingRegressor` (the scikit-learn face of `Kriging`) on Hartmann 3: # `fit`/`predict` (scikit-learn conventions), the same predictions as the native # class, `get_params`/`set_params`/`clone`, use inside `Pipeline`, # `GridSearchCV`, `cross_val_score`, and `sample_y`. # %% +import matplotlib.pyplot as plt import numpy as np +import pandas as pd import pylibkriging as lk import lkbench @@ -75,6 +83,14 @@ def mean_sd(k, Xn): gs = GridSearchCV(KrigingRegressor(), {"kernel": ["gauss", "matern3_2", "matern5_2"]}, cv=3).fit(X, y) suite.true("GridSearchCV picked a kernel", gs.best_params_["kernel"] in ("gauss", "matern3_2", "matern5_2"), detail=str(gs.best_params_)) + cv = pd.DataFrame(gs.cv_results_)[["param_kernel", "mean_test_score", "std_test_score", "mean_fit_time"]] + for _, r in cv.iterrows(): + suite.metric("hartmann3/GridSearchCV", str(r["param_kernel"]), cv_r2=r["mean_test_score"], + cv_r2_sd=r["std_test_score"], fit_s=r["mean_fit_time"]) + plt.figure(figsize=(5, 3.2)) + plt.bar(cv["param_kernel"].astype(str), cv["mean_test_score"], yerr=cv["std_test_score"], capsize=4) + plt.ylabel("3-fold CV R²"); plt.title(f"GridSearchCV(KrigingRegressor): best = {gs.best_params_['kernel']}") + plt.ylim(min(0, cv["mean_test_score"].min() - .05), 1.02); plt.show() with suite.runs("cross_val_score"): sc = cross_val_score(KrigingRegressor(kernel="matern5_2"), X, y, cv=4) suite.finite("cross_val_score finite", sc) @@ -84,5 +100,16 @@ def mean_sd(k, Xn): else: suite.skip("sklearn estimators", "pylibkriging.sklearn or scikit-learn not available") +# %% [markdown] +# ## Summary +# +# The metrics of this notebook, the number of checks per status, and every check that +# did not pass (FAIL: error-level, fails the run; WARN: warn-level; KNOWN: listed in +# `tools/known_issues.csv`, with its issue; SKIP: feature absent from the installed version). +# All of them are also written to `results/sklearn-estimators_Kriging_python.checks.csv` and `.metrics.csv`. + +# %% +suite.summary() + # %% suite.finalize() diff --git a/tools/src/test-functions-reference-values_no-model_python.py b/tools/src/test-functions-reference-values_no-model_python.py index c355439..34b6be5 100644 --- a/tools/src/test-functions-reference-values_no-model_python.py +++ b/tools/src/test-functions-reference-values_no-model_python.py @@ -1,7 +1,14 @@ # %% [markdown] # # Python · no model — the test functions against their published reference values # -# Every benchmark of this repository relies on the test functions of +# **Idea.** Every example of this gallery is only as good as its test functions: +# before fitting anything, each of the 21 functions is evaluated at points where its +# value is published, and its range and shape are plotted. +# +# **Reference.** Published optima and nominal values (Virtual Library of Simulation +# Experiments, Surjanovic & Bingham). +# +# Every notebook of this gallery relies on the test functions of # `lkbench.functions` (after the [Virtual Library of Simulation Experiments](https://www.sfu.ca/~ssurjano/)). # A typo in one of them would silently bias every result built on it, so they are # checked first, against the reference values published on the library pages @@ -61,7 +68,7 @@ # ## Designs # # The designs are seeded: the same seed gives the same points (reproducibility of -# every benchmark), and the optimized LHS is a Latin hypercube (one point per +# every notebook), and the optimized LHS is a Latin hypercube (one point per # stratum in each dimension). # %% @@ -85,5 +92,16 @@ ax.set_title(name) plt.tight_layout() +# %% [markdown] +# ## Summary +# +# The metrics of this notebook, the number of checks per status, and every check that +# did not pass (FAIL: error-level, fails the run; WARN: warn-level; KNOWN: listed in +# `tools/known_issues.csv`, with its issue; SKIP: feature absent from the installed version). +# All of them are also written to `results/test-functions-reference-values_no-model_python.checks.csv` and `.metrics.csv`. + +# %% +suite.summary() + # %% suite.finalize() diff --git a/tools/src/update-simulate_Kriging_python.py b/tools/src/update-simulate_Kriging_python.py index fdd5eca..0ab15a0 100644 --- a/tools/src/update-simulate_Kriging_python.py +++ b/tools/src/update-simulate_Kriging_python.py @@ -1,6 +1,12 @@ # %% [markdown] # # Python · Kriging — `update`, `simulate`, `update_simulate` # +# **Idea.** Adding observations (`update`) and drawing conditional paths (`simulate`, +# `update_simulate`) are shortcuts: they must give what the long way gives. +# +# **Reference.** A model fitted from scratch on all the data, and Monte Carlo moments +# against the predictive mean and covariance. +# # Sequential use of a fitted model (sequential design, Bayesian optimization): # # * `update(y_u, X_u, refit=False)` adds observations through a Cholesky @@ -158,5 +164,16 @@ def fixed_params(k): ax.set_title(title); ax.set_xlabel("x") axes[1].plot(Xu, yu, "rs", label="new points"); axes[1].legend(); plt.show() +# %% [markdown] +# ## Summary +# +# The metrics of this notebook, the number of checks per status, and every check that +# did not pass (FAIL: error-level, fails the run; WARN: warn-level; KNOWN: listed in +# `tools/known_issues.csv`, with its issue; SKIP: feature absent from the installed version). +# All of them are also written to `results/update-simulate_Kriging_python.checks.csv` and `.metrics.csv`. + +# %% +suite.summary() + # %% suite.finalize() diff --git a/tools/src/vs-DiceKriging-RobustGaSP_Kriging_R.R b/tools/src/vs-DiceKriging-RobustGaSP_Kriging_R.R index 373a231..1d6e73a 100644 --- a/tools/src/vs-DiceKriging-RobustGaSP_Kriging_R.R +++ b/tools/src/vs-DiceKriging-RobustGaSP_Kriging_R.R @@ -1,6 +1,12 @@ # %% [markdown] # # R · Kriging vs DiceKriging and RobustGaSP # +# **Idea.** libKriging's `Kriging` follows DiceKriging: at the same hyperparameters it +# must give DiceKriging's numbers, and its default fit must be as accurate as the R +# references. +# +# **Reference.** DiceKriging (exact equality) and RobustGaSP (accuracy, cost). +# # DiceKriging is the reference implementation libKriging's `Kriging` follows # (same product-form kernels and range parametrization, same concentrated # likelihood). Hence two levels of comparison: @@ -101,7 +107,14 @@ for (name in DATASETS) { } } res <- do.call(rbind, results) -print(reshape(res[, c("dataset", "package", "q2")], idvar = "dataset", timevar = "package", direction = "wide")) +print(reshape(res[, c("dataset", "package", "q2", "fit_s")], idvar = "dataset", timevar = "package", direction = "wide")) +op <- par(mfrow = c(1, 2), mar = c(4, 4, 2, 1)) +q <- tapply(res$q2, list(res$package, res$dataset), mean) +barplot(pmax(q, 0), beside = TRUE, legend.text = rownames(q), args.legend = list(x = "bottomright", cex = .7), + ylim = c(0, 1), ylab = "Q2 (test set)", main = "Default fits: accuracy") +t <- tapply(res$fit_s, list(res$package, res$dataset), mean) +barplot(t, beside = TRUE, log = "y", ylab = "fit time (s)", main = "Default fits: cost") +par(op) # %% for (name in DATASETS) { @@ -111,5 +124,16 @@ for (name in DATASETS) { check_greater(s, paste0(name, ": Q2(rlibkriging BFGS10) > 0.9"), r$q2[r$package == "rlibkriging BFGS10"], 0.9) } +# %% [markdown] +# ## Summary +# +# The metrics of this notebook, the number of checks per status, and every check that +# did not pass (FAIL: error-level, fails the run; WARN: warn-level; KNOWN: listed in +# `tools/known_issues.csv`, with its issue; SKIP: feature absent from the installed version). +# All of them are also written to `results/vs-DiceKriging-RobustGaSP_Kriging_R.checks.csv` and `.metrics.csv`. + +# %% +summary_suite(s) + # %% finalize(s) diff --git a/tools/src/vs-GaussianProcesses_Kriging_julia.jl b/tools/src/vs-GaussianProcesses_Kriging_julia.jl index 1abaada..19f957b 100644 --- a/tools/src/vs-GaussianProcesses_Kriging_julia.jl +++ b/tools/src/vs-GaussianProcesses_Kriging_julia.jl @@ -1,6 +1,11 @@ # %% [markdown] # # Julia · Kriging vs GaussianProcesses.jl # +# **Idea.** JLibKriging against the main Julia GP package: default fits on the shared +# datasets. +# +# **Reference.** GaussianProcesses.jl (accuracy and cost). +# # Default fits on the shared datasets of `data/` (Branin, Hartmann 3, Borehole): # JLibKriging (`matern5_2`, constant trend, single start and 10 restarts) against # [GaussianProcesses.jl](https://github.com/STOR-i/GaussianProcesses.jl) @@ -67,5 +72,16 @@ end have_gp || add_check!(s, "GaussianProcesses.jl available", false; value="absent", severity="warn") foreach(r -> println(join(r[3:6], " ")), s.metrics) +# %% [markdown] +# ## Summary +# +# The metrics of this notebook, the number of checks per status, and every check that +# did not pass (FAIL: error-level, fails the run; WARN: warn-level; KNOWN: listed in +# `tools/known_issues.csv`, with its issue; SKIP: feature absent from the installed version). +# All of them are also written to `results/vs-GaussianProcesses_Kriging_julia.checks.csv` and `.metrics.csv`. + +# %% +summary!(s) + # %% finalize!(s) diff --git a/tools/src/vs-STK_Kriging_octave.m b/tools/src/vs-STK_Kriging_octave.m index bac5706..e46f8e0 100644 --- a/tools/src/vs-STK_Kriging_octave.m +++ b/tools/src/vs-STK_Kriging_octave.m @@ -1,6 +1,12 @@ % %% [markdown] % # Octave · Kriging vs STK % +% **Idea.** The Octave/MATLAB reference toolbox, STK: at the same hyperparameters +% (Gaussian kernel) both must give the same predictor, and their default fits are +% compared. +% +% **Reference.** STK (exact equality, then accuracy and cost). +% % [STK](https://github.com/stk-kriging/stk) (Small Toolbox for Kriging) is the % reference kriging toolbox for Octave/MATLAB. % @@ -99,5 +105,16 @@ end disp(s.metrics(:, 3:6)) +% %% [markdown] +% ## Summary +% +% The metrics of this notebook, the number of checks per status, and every check that +% did not pass (FAIL: error-level, fails the run; WARN: warn-level; KNOWN: listed in +% `tools/known_issues.csv`, with its issue; SKIP: feature absent from the installed version). +% All of them are also written to `results/vs-STK_Kriging_octave.checks.csv` and `.metrics.csv`. + +% %% +lkb_summary(s); + % %% lkb_finalize(s); diff --git a/tools/src/vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python.py b/tools/src/vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python.py index f71be9c..78defd7 100644 --- a/tools/src/vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python.py +++ b/tools/src/vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python.py @@ -1,6 +1,12 @@ # %% [markdown] # # Python · Kriging vs scikit-learn, SMT, OpenTURNS, GPy # +# **Idea.** How does libKriging compare with the Python GP packages, first as an +# implementation (same hyperparameters → same numbers), then as a tool (default fits: +# accuracy, calibration, cost) on 10 functions? +# +# **Reference.** Scikit-learn, SMT, OpenTURNS and GPy. +# # Two kinds of comparisons, which should not be confused: # # 1. **Same model, same hyperparameters → same numbers.** With θ, σ², β fixed, @@ -239,5 +245,16 @@ def p(Z): -0.05, detail=f"best: {others.idxmax()}") if len(others) else None suite.greater(f"{fname}: Q2(pylibkriging BFGS10) > 0.5", s["pylibkriging BFGS10"], 0.5) +# %% [markdown] +# ## Summary +# +# The metrics of this notebook, the number of checks per status, and every check that +# did not pass (FAIL: error-level, fails the run; WARN: warn-level; KNOWN: listed in +# `tools/known_issues.csv`, with its issue; SKIP: feature absent from the installed version). +# All of them are also written to `results/vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python.checks.csv` and `.metrics.csv`. + +# %% +suite.summary() + # %% suite.finalize() diff --git a/tools/src/warpings-categorical-ordinal_WarpKriging_python.py b/tools/src/warpings-categorical-ordinal_WarpKriging_python.py index ed6485a..95b1c2c 100644 --- a/tools/src/warpings-categorical-ordinal_WarpKriging_python.py +++ b/tools/src/warpings-categorical-ordinal_WarpKriging_python.py @@ -1,6 +1,13 @@ # %% [markdown] # # Python · WarpKriging — non-stationarity, categorical and ordinal inputs # +# **Idea.** Input warpings handle non-stationarity and categorical or ordinal inputs; +# each warping must beat, or at least match, plain Kriging where it is meant to help, +# without making the predictive variance overconfident. +# +# **Reference.** Plain `Kriging` on the same data (accuracy and NLPD), and `none` +# warping = `Kriging`. +# # `WarpKriging` learns a transform per input variable jointly with the GP # hyperparameters. Problems where a plain stationary GP is the wrong model: # @@ -192,5 +199,16 @@ def f_ord(Xo): with suite.runs("WarpKriging: fit (persistence)"): lkbench.persistence.check(suite, "WarpKriging", lk.WarpKriging(y, X, ["kumaraswamy", "none", "affine"], "matern5_2"), Xt, pred) +# %% [markdown] +# ## Summary +# +# The metrics of this notebook, the number of checks per status, and every check that +# did not pass (FAIL: error-level, fails the run; WARN: warn-level; KNOWN: listed in +# `tools/known_issues.csv`, with its issue; SKIP: feature absent from the installed version). +# All of them are also written to `results/warpings-categorical-ordinal_WarpKriging_python.checks.csv` and `.metrics.csv`. + +# %% +suite.summary() + # %% suite.finalize() diff --git a/update-simulate_Kriging_python.ipynb b/update-simulate_Kriging_python.ipynb index 86b2ccb..9af5731 100644 --- a/update-simulate_Kriging_python.ipynb +++ b/update-simulate_Kriging_python.ipynb @@ -7,6 +7,12 @@ "source": [ "# Python · Kriging — `update`, `simulate`, `update_simulate`\n", "\n", + "**Idea.** Adding observations (`update`) and drawing conditional paths (`simulate`,\n", + "`update_simulate`) are shortcuts: they must give what the long way gives.\n", + "\n", + "**Reference.** A model fitted from scratch on all the data, and Monte Carlo moments\n", + "against the predictive mean and covariance.\n", + "\n", "Sequential use of a fitted model (sequential design, Bayesian optimization):\n", "\n", "* `update(y_u, X_u, refit=False)` adds observations through a Cholesky\n", @@ -41,9 +47,9 @@ "from pathlib import Path\n", "\n", "if not Path(\"tools/known_issues.csv\").is_file():\n", - " if not Path(\"benchmarks/tools/known_issues.csv\").is_file():\n", - " subprocess.run([\"git\", \"clone\", \"--depth\", \"1\", \"https://github.com/libKriging/benchmarks\"], check=True)\n", - " os.chdir(\"benchmarks\")\n", + " if not Path(\"gallery/tools/known_issues.csv\").is_file():\n", + " subprocess.run([\"git\", \"clone\", \"--depth\", \"1\", \"https://github.com/libKriging/gallery\"], check=True)\n", + " os.chdir(\"gallery\")\n", " try:\n", " import pylibkriging # noqa: F401\n", " except ImportError:\n", @@ -267,10 +273,33 @@ "axes[1].plot(Xu, yu, \"rs\", label=\"new points\"); axes[1].legend(); plt.show()" ] }, + { + "cell_type": "markdown", + "id": "c013", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "The metrics of this notebook, the number of checks per status, and every check that\n", + "did not pass (FAIL: error-level, fails the run; WARN: warn-level; KNOWN: listed in\n", + "`tools/known_issues.csv`, with its issue; SKIP: feature absent from the installed version).\n", + "All of them are also written to `results/update-simulate_Kriging_python.checks.csv` and `.metrics.csv`." + ] + }, { "cell_type": "code", "execution_count": null, - "id": "c013", + "id": "c014", + "metadata": {}, + "outputs": [], + "source": [ + "suite.summary()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c015", "metadata": {}, "outputs": [], "source": [ diff --git a/vs-DiceKriging-RobustGaSP_Kriging_R.ipynb b/vs-DiceKriging-RobustGaSP_Kriging_R.ipynb index 804bc82..d8a3e4e 100644 --- a/vs-DiceKriging-RobustGaSP_Kriging_R.ipynb +++ b/vs-DiceKriging-RobustGaSP_Kriging_R.ipynb @@ -7,6 +7,12 @@ "source": [ "# R · Kriging vs DiceKriging and RobustGaSP\n", "\n", + "**Idea.** libKriging's `Kriging` follows DiceKriging: at the same hyperparameters it\n", + "must give DiceKriging's numbers, and its default fit must be as accurate as the R\n", + "references.\n", + "\n", + "**Reference.** DiceKriging (exact equality) and RobustGaSP (accuracy, cost).\n", + "\n", "DiceKriging is the reference implementation libKriging's `Kriging` follows\n", "(same product-form kernels and range parametrization, same concentrated\n", "likelihood). Hence two levels of comparison:\n", @@ -150,7 +156,14 @@ " }\n", "}\n", "res <- do.call(rbind, results)\n", - "print(reshape(res[, c(\"dataset\", \"package\", \"q2\")], idvar = \"dataset\", timevar = \"package\", direction = \"wide\"))" + "print(reshape(res[, c(\"dataset\", \"package\", \"q2\", \"fit_s\")], idvar = \"dataset\", timevar = \"package\", direction = \"wide\"))\n", + "op <- par(mfrow = c(1, 2), mar = c(4, 4, 2, 1))\n", + "q <- tapply(res$q2, list(res$package, res$dataset), mean)\n", + "barplot(pmax(q, 0), beside = TRUE, legend.text = rownames(q), args.legend = list(x = \"bottomright\", cex = .7),\n", + " ylim = c(0, 1), ylab = \"Q2 (test set)\", main = \"Default fits: accuracy\")\n", + "t <- tapply(res$fit_s, list(res$package, res$dataset), mean)\n", + "barplot(t, beside = TRUE, log = \"y\", ylab = \"fit time (s)\", main = \"Default fits: cost\")\n", + "par(op)" ] }, { @@ -168,10 +181,33 @@ "}" ] }, + { + "cell_type": "markdown", + "id": "c009", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "The metrics of this notebook, the number of checks per status, and every check that\n", + "did not pass (FAIL: error-level, fails the run; WARN: warn-level; KNOWN: listed in\n", + "`tools/known_issues.csv`, with its issue; SKIP: feature absent from the installed version).\n", + "All of them are also written to `results/vs-DiceKriging-RobustGaSP_Kriging_R.checks.csv` and `.metrics.csv`." + ] + }, { "cell_type": "code", "execution_count": null, - "id": "c009", + "id": "c010", + "metadata": {}, + "outputs": [], + "source": [ + "summary_suite(s)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c011", "metadata": {}, "outputs": [], "source": [ diff --git a/vs-GaussianProcesses_Kriging_julia.ipynb b/vs-GaussianProcesses_Kriging_julia.ipynb index 5a99dd3..697874e 100644 --- a/vs-GaussianProcesses_Kriging_julia.ipynb +++ b/vs-GaussianProcesses_Kriging_julia.ipynb @@ -7,6 +7,11 @@ "source": [ "# Julia · Kriging vs GaussianProcesses.jl\n", "\n", + "**Idea.** JLibKriging against the main Julia GP package: default fits on the shared\n", + "datasets.\n", + "\n", + "**Reference.** GaussianProcesses.jl (accuracy and cost).\n", + "\n", "Default fits on the shared datasets of `data/` (Branin, Hartmann 3, Borehole):\n", "JLibKriging (`matern5_2`, constant trend, single start and 10 restarts) against\n", "[GaussianProcesses.jl](https://github.com/STOR-i/GaussianProcesses.jl)\n", @@ -105,10 +110,33 @@ "foreach(r -> println(join(r[3:6], \" \")), s.metrics)" ] }, + { + "cell_type": "markdown", + "id": "c006", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "The metrics of this notebook, the number of checks per status, and every check that\n", + "did not pass (FAIL: error-level, fails the run; WARN: warn-level; KNOWN: listed in\n", + "`tools/known_issues.csv`, with its issue; SKIP: feature absent from the installed version).\n", + "All of them are also written to `results/vs-GaussianProcesses_Kriging_julia.checks.csv` and `.metrics.csv`." + ] + }, { "cell_type": "code", "execution_count": null, - "id": "c006", + "id": "c007", + "metadata": {}, + "outputs": [], + "source": [ + "summary!(s)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c008", "metadata": {}, "outputs": [], "source": [ diff --git a/vs-STK_Kriging_octave.ipynb b/vs-STK_Kriging_octave.ipynb index 0605a12..d992ff6 100644 --- a/vs-STK_Kriging_octave.ipynb +++ b/vs-STK_Kriging_octave.ipynb @@ -7,6 +7,12 @@ "source": [ "# Octave · Kriging vs STK\n", "\n", + "**Idea.** The Octave/MATLAB reference toolbox, STK: at the same hyperparameters\n", + "(Gaussian kernel) both must give the same predictor, and their default fits are\n", + "compared.\n", + "\n", + "**Reference.** STK (exact equality, then accuracy and cost).\n", + "\n", "[STK](https://github.com/stk-kriging/stk) (Small Toolbox for Kriging) is the\n", "reference kriging toolbox for Octave/MATLAB.\n", "\n", @@ -149,10 +155,33 @@ "disp(s.metrics(:, 3:6))" ] }, + { + "cell_type": "markdown", + "id": "c008", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "The metrics of this notebook, the number of checks per status, and every check that\n", + "did not pass (FAIL: error-level, fails the run; WARN: warn-level; KNOWN: listed in\n", + "`tools/known_issues.csv`, with its issue; SKIP: feature absent from the installed version).\n", + "All of them are also written to `results/vs-STK_Kriging_octave.checks.csv` and `.metrics.csv`." + ] + }, { "cell_type": "code", "execution_count": null, - "id": "c008", + "id": "c009", + "metadata": {}, + "outputs": [], + "source": [ + "lkb_summary(s);" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c010", "metadata": {}, "outputs": [], "source": [ diff --git a/vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python.ipynb b/vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python.ipynb index 022e337..d6f60ea 100644 --- a/vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python.ipynb +++ b/vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python.ipynb @@ -7,6 +7,12 @@ "source": [ "# Python · Kriging vs scikit-learn, SMT, OpenTURNS, GPy\n", "\n", + "**Idea.** How does libKriging compare with the Python GP packages, first as an\n", + "implementation (same hyperparameters → same numbers), then as a tool (default fits:\n", + "accuracy, calibration, cost) on 10 functions?\n", + "\n", + "**Reference.** Scikit-learn, SMT, OpenTURNS and GPy.\n", + "\n", "Two kinds of comparisons, which should not be confused:\n", "\n", "1. **Same model, same hyperparameters → same numbers.** With θ, σ², β fixed,\n", @@ -42,9 +48,9 @@ "from pathlib import Path\n", "\n", "if not Path(\"tools/known_issues.csv\").is_file():\n", - " if not Path(\"benchmarks/tools/known_issues.csv\").is_file():\n", - " subprocess.run([\"git\", \"clone\", \"--depth\", \"1\", \"https://github.com/libKriging/benchmarks\"], check=True)\n", - " os.chdir(\"benchmarks\")\n", + " if not Path(\"gallery/tools/known_issues.csv\").is_file():\n", + " subprocess.run([\"git\", \"clone\", \"--depth\", \"1\", \"https://github.com/libKriging/gallery\"], check=True)\n", + " os.chdir(\"gallery\")\n", " try:\n", " import pylibkriging # noqa: F401\n", " except ImportError:\n", @@ -352,10 +358,33 @@ " suite.greater(f\"{fname}: Q2(pylibkriging BFGS10) > 0.5\", s[\"pylibkriging BFGS10\"], 0.5)" ] }, + { + "cell_type": "markdown", + "id": "c013", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "The metrics of this notebook, the number of checks per status, and every check that\n", + "did not pass (FAIL: error-level, fails the run; WARN: warn-level; KNOWN: listed in\n", + "`tools/known_issues.csv`, with its issue; SKIP: feature absent from the installed version).\n", + "All of them are also written to `results/vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python.checks.csv` and `.metrics.csv`." + ] + }, { "cell_type": "code", "execution_count": null, - "id": "c013", + "id": "c014", + "metadata": {}, + "outputs": [], + "source": [ + "suite.summary()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c015", "metadata": {}, "outputs": [], "source": [ diff --git a/warpings-categorical-ordinal_WarpKriging_python.ipynb b/warpings-categorical-ordinal_WarpKriging_python.ipynb index 7760dc4..04d8ef7 100644 --- a/warpings-categorical-ordinal_WarpKriging_python.ipynb +++ b/warpings-categorical-ordinal_WarpKriging_python.ipynb @@ -7,6 +7,13 @@ "source": [ "# Python · WarpKriging — non-stationarity, categorical and ordinal inputs\n", "\n", + "**Idea.** Input warpings handle non-stationarity and categorical or ordinal inputs;\n", + "each warping must beat, or at least match, plain Kriging where it is meant to help,\n", + "without making the predictive variance overconfident.\n", + "\n", + "**Reference.** Plain `Kriging` on the same data (accuracy and NLPD), and `none`\n", + "warping = `Kriging`.\n", + "\n", "`WarpKriging` learns a transform per input variable jointly with the GP\n", "hyperparameters. Problems where a plain stationary GP is the wrong model:\n", "\n", @@ -39,9 +46,9 @@ "from pathlib import Path\n", "\n", "if not Path(\"tools/known_issues.csv\").is_file():\n", - " if not Path(\"benchmarks/tools/known_issues.csv\").is_file():\n", - " subprocess.run([\"git\", \"clone\", \"--depth\", \"1\", \"https://github.com/libKriging/benchmarks\"], check=True)\n", - " os.chdir(\"benchmarks\")\n", + " if not Path(\"gallery/tools/known_issues.csv\").is_file():\n", + " subprocess.run([\"git\", \"clone\", \"--depth\", \"1\", \"https://github.com/libKriging/gallery\"], check=True)\n", + " os.chdir(\"gallery\")\n", " try:\n", " import pylibkriging # noqa: F401\n", " except ImportError:\n", @@ -320,10 +327,33 @@ " lkbench.persistence.check(suite, \"WarpKriging\", lk.WarpKriging(y, X, [\"kumaraswamy\", \"none\", \"affine\"], \"matern5_2\"), Xt, pred)" ] }, + { + "cell_type": "markdown", + "id": "c016", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "The metrics of this notebook, the number of checks per status, and every check that\n", + "did not pass (FAIL: error-level, fails the run; WARN: warn-level; KNOWN: listed in\n", + "`tools/known_issues.csv`, with its issue; SKIP: feature absent from the installed version).\n", + "All of them are also written to `results/warpings-categorical-ordinal_WarpKriging_python.checks.csv` and `.metrics.csv`." + ] + }, { "cell_type": "code", "execution_count": null, - "id": "c016", + "id": "c017", + "metadata": {}, + "outputs": [], + "source": [ + "suite.summary()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c018", "metadata": {}, "outputs": [], "source": [ From 036c1fea7afd4534c439c46f945584d68173c359 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 15 Oct 2026 20:32:27 +0000 Subject: [PATCH 08/16] Rename the helper package lkbench to lkgallery tools/lkbench -> tools/lkgallery (package name in tools/pyproject.toml), the imports of every notebook, and the environment variables LKBENCH_ROOT / LKBENCH_RESULTS -> LKGALLERY_ROOT / LKGALLERY_RESULTS (Python, R, Octave and Julia helpers). Checked locally: Python, R and Octave notebooks run, cross-binding 49/49 per binding. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01QiFEE514dpwS57mcFWDeZG --- .binder/requirements.txt | 4 ++-- README.md | 4 ++-- aggregations-large-n_NestedKriging_python.ipynb | 10 +++++----- cross-binding-reference_all-classes_python.ipynb | 10 +++++----- cross-binding_all-classes_octave.ipynb | 2 +- deep-kernel-projected-branin_MLPKriging_python.ipynb | 10 +++++----- ...-dimension-normalize-screening_Kriging_python.ipynb | 8 ++++---- kernels-trends-interpolation_Kriging_python.ipynb | 8 ++++---- large-n_Kriging-vecchia-nystrom_python.ipynb | 8 ++++---- noisy-data_Kriging-nugget-noise_python.ipynb | 8 ++++---- objectives-LL-LOO-LMP-gradients_Kriging_python.ipynb | 8 ++++---- robustness-invalid-inputs_Kriging_python.ipynb | 8 ++++---- save-load-pickle-copy-accessors_Kriging_python.ipynb | 10 +++++----- sklearn-estimators_Kriging_python.ipynb | 8 ++++---- test-functions-reference-values_no-model_python.ipynb | 10 +++++----- tools/install/python.sh | 2 +- tools/julia/checks.jl | 4 ++-- tools/{lkbench => lkgallery}/__init__.py | 0 tools/{lkbench => lkgallery}/checks.py | 10 +++++----- tools/{lkbench => lkgallery}/compat.py | 0 tools/{lkbench => lkgallery}/design.py | 0 tools/{lkbench => lkgallery}/functions.py | 0 tools/{lkbench => lkgallery}/metrics.py | 0 tools/{lkbench => lkgallery}/persistence.py | 0 tools/make_data.py | 2 +- tools/notebook_setup.py | 2 +- tools/octave/lkb_finalize.m | 2 +- tools/octave/lkb_suite.m | 2 +- tools/pyproject.toml | 4 ++-- tools/r/checks.R | 4 ++-- tools/src/aggregations-large-n_NestedKriging_python.py | 8 ++++---- .../src/cross-binding-reference_all-classes_python.py | 8 ++++---- tools/src/cross-binding_all-classes_octave.m | 2 +- .../deep-kernel-projected-branin_MLPKriging_python.py | 8 ++++---- ...igh-dimension-normalize-screening_Kriging_python.py | 6 +++--- .../src/kernels-trends-interpolation_Kriging_python.py | 6 +++--- tools/src/large-n_Kriging-vecchia-nystrom_python.py | 6 +++--- tools/src/noisy-data_Kriging-nugget-noise_python.py | 6 +++--- .../objectives-LL-LOO-LMP-gradients_Kriging_python.py | 6 +++--- tools/src/robustness-invalid-inputs_Kriging_python.py | 6 +++--- .../save-load-pickle-copy-accessors_Kriging_python.py | 8 ++++---- tools/src/sklearn-estimators_Kriging_python.py | 6 +++--- .../test-functions-reference-values_no-model_python.py | 8 ++++---- tools/src/update-simulate_Kriging_python.py | 6 +++--- .../src/vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python.py | 6 +++--- .../warpings-categorical-ordinal_WarpKriging_python.py | 8 ++++---- update-simulate_Kriging_python.ipynb | 8 ++++---- vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python.ipynb | 8 ++++---- warpings-categorical-ordinal_WarpKriging_python.ipynb | 10 +++++----- 49 files changed, 139 insertions(+), 139 deletions(-) rename tools/{lkbench => lkgallery}/__init__.py (100%) rename tools/{lkbench => lkgallery}/checks.py (96%) rename tools/{lkbench => lkgallery}/compat.py (100%) rename tools/{lkbench => lkgallery}/design.py (100%) rename tools/{lkbench => lkgallery}/functions.py (100%) rename tools/{lkbench => lkgallery}/metrics.py (100%) rename tools/{lkbench => lkgallery}/persistence.py (100%) diff --git a/.binder/requirements.txt b/.binder/requirements.txt index 0a370dc..7122539 100644 --- a/.binder/requirements.txt +++ b/.binder/requirements.txt @@ -1,6 +1,6 @@ # Binder (mybinder.org) environment for the Python notebooks: the released pylibkriging -# (PyPI wheels exist for CPython 3.10 to 3.12, hence runtime.txt) and lkbench's -# dependencies; lkbench itself is found in tools/ by the notebooks' setup cell, and the +# (PyPI wheels exist for CPython 3.10 to 3.12, hence runtime.txt) and lkgallery's +# dependencies; lkgallery itself is found in tools/ by the notebooks' setup cell, and the # comparison packages are installed by the notebooks that use them. pylibkriging numpy diff --git a/README.md b/README.md index df3c74d..5f66a51 100644 --- a/README.md +++ b/README.md @@ -92,7 +92,7 @@ Notebooks are named `__.ipynb`. Those comparing with another package install it themselves, in their first cells (pip, CRAN, `Pkg.add`, or a clone of STK); a package that cannot be installed is reported (warn-level) and its comparison skipped. Test functions: -[`tools/lkbench/functions.py`](tools/lkbench/functions.py), after the +[`tools/lkgallery/functions.py`](tools/lkgallery/functions.py), after the [Virtual Library of Simulation Experiments](https://www.sfu.ca/~ssurjano/) (Surjanovic & Bingham). Everything that is not a notebook is under [`tools/`](tools) (sources of the notebooks, helpers, installation of the bindings, @@ -122,7 +122,7 @@ and the installation of rlibkriging (compiled from CRAN sources) and of JLibKrig ## Checks and statuses -Each notebook records its checks with `lkbench.Suite` (Python) or the small +Each notebook records its checks with `lkgallery.Suite` (Python) or the small equivalents of `tools/r`, `tools/octave` and `tools/julia`, under a suite named after the notebook, and writes `results/.checks.csv` and `results/.metrics.csv`. diff --git a/aggregations-large-n_NestedKriging_python.ipynb b/aggregations-large-n_NestedKriging_python.ipynb index d69597f..3e41435 100644 --- a/aggregations-large-n_NestedKriging_python.ipynb +++ b/aggregations-large-n_NestedKriging_python.ipynb @@ -66,7 +66,7 @@ " \"(PyPI has wheels for CPython 3.10 to 3.12 only). On Colab: Runtime > Change runtime version > \"\n", " \"2026.07 (Python 3.12), then run again.\\n\" + r.stderr[-500:])\n", " importlib.invalidate_caches()\n", - "sys.path.insert(0, str(Path(\"tools\").resolve())) # lkbench, when not installed (Colab, Binder)\n", + "sys.path.insert(0, str(Path(\"tools\").resolve())) # lkgallery, when not installed (Colab, Binder)\n", "print(\"working directory:\", os.getcwd())" ] }, @@ -84,10 +84,10 @@ "import pandas as pd\n", "import pylibkriging as lk\n", "\n", - "import lkbench\n", - "from lkbench import TESTS, design, metrics, compat\n", + "import lkgallery\n", + "from lkgallery import TESTS, design, metrics, compat\n", "\n", - "suite = lkbench.Suite(\"aggregations-large-n_NestedKriging_python\")\n", + "suite = lkgallery.Suite(\"aggregations-large-n_NestedKriging_python\")\n", "print(\"pylibkriging\", compat.version())\n", "if not compat.has(\"NestedKriging\"):\n", " suite.skip(\"NestedKriging\", \"not available in this pylibkriging\")\n", @@ -257,7 +257,7 @@ "if compat.has(\"NestedKriging\"):\n", " X, y, Xt, yt = design.dataset(TESTS[\"hartmann3\"], 40, n_test=100, seed=61)\n", " with suite.runs(\"NestedKriging: fit (persistence)\"):\n", - " lkbench.persistence.check(suite, \"NestedKriging\", lk.NestedKriging(y, X, \"matern5_2\", nb_groups=4, seed=1), Xt, pred)" + " lkgallery.persistence.check(suite, \"NestedKriging\", lk.NestedKriging(y, X, \"matern5_2\", nb_groups=4, seed=1), Xt, pred)" ] }, { diff --git a/cross-binding-reference_all-classes_python.ipynb b/cross-binding-reference_all-classes_python.ipynb index 1d37ad3..8d6912e 100644 --- a/cross-binding-reference_all-classes_python.ipynb +++ b/cross-binding-reference_all-classes_python.ipynb @@ -79,7 +79,7 @@ " \"(PyPI has wheels for CPython 3.10 to 3.12 only). On Colab: Runtime > Change runtime version > \"\n", " \"2026.07 (Python 3.12), then run again.\\n\" + r.stderr[-500:])\n", " importlib.invalidate_caches()\n", - "sys.path.insert(0, str(Path(\"tools\").resolve())) # lkbench, when not installed (Colab, Binder)\n", + "sys.path.insert(0, str(Path(\"tools\").resolve())) # lkgallery, when not installed (Colab, Binder)\n", "print(\"working directory:\", os.getcwd())" ] }, @@ -97,11 +97,11 @@ "import pandas as pd\n", "import pylibkriging as lk\n", "\n", - "import lkbench\n", - "from lkbench import compat\n", - "from lkbench.checks import DATA_DIR as DATA, RESULTS_DIR\n", + "import lkgallery\n", + "from lkgallery import compat\n", + "from lkgallery.checks import DATA_DIR as DATA, RESULTS_DIR\n", "\n", - "suite = lkbench.Suite(\"cross-binding-reference_all-classes_python\")\n", + "suite = lkgallery.Suite(\"cross-binding-reference_all-classes_python\")\n", "print(\"pylibkriging\", compat.version())\n", "DATASETS = [\"branin\", \"hartmann3\", \"borehole\"]\n", "rows = []\n", diff --git a/cross-binding_all-classes_octave.ipynb b/cross-binding_all-classes_octave.ipynb index 27aa097..f9e2f75 100644 --- a/cross-binding_all-classes_octave.ipynb +++ b/cross-binding_all-classes_octave.ipynb @@ -132,7 +132,7 @@ "metadata": {}, "outputs": [], "source": [ - "d = getenv(\"LKBENCH_RESULTS\"); if isempty(d), d = \"results\"; end\n", + "d = getenv(\"LKGALLERY_RESULTS\"); if isempty(d), d = \"results\"; end\n", "if ~exist(d, \"dir\"), mkdir(d); end\n", "fid = fopen(fullfile(d, \"xbind_octave.csv\"), \"w\");\n", "fprintf(fid, \"lang,dataset,quantity,index,value\\n\");\n", diff --git a/deep-kernel-projected-branin_MLPKriging_python.ipynb b/deep-kernel-projected-branin_MLPKriging_python.ipynb index 11ec6a4..40e963c 100644 --- a/deep-kernel-projected-branin_MLPKriging_python.ipynb +++ b/deep-kernel-projected-branin_MLPKriging_python.ipynb @@ -55,7 +55,7 @@ " \"(PyPI has wheels for CPython 3.10 to 3.12 only). On Colab: Runtime > Change runtime version > \"\n", " \"2026.07 (Python 3.12), then run again.\\n\" + r.stderr[-500:])\n", " importlib.invalidate_caches()\n", - "sys.path.insert(0, str(Path(\"tools\").resolve())) # lkbench, when not installed (Colab, Binder)\n", + "sys.path.insert(0, str(Path(\"tools\").resolve())) # lkgallery, when not installed (Colab, Binder)\n", "print(\"working directory:\", os.getcwd())" ] }, @@ -72,10 +72,10 @@ "import numpy as np\n", "import pylibkriging as lk\n", "\n", - "import lkbench\n", - "from lkbench import TESTS, design, metrics, compat\n", + "import lkgallery\n", + "from lkgallery import TESTS, design, metrics, compat\n", "\n", - "suite = lkbench.Suite(\"deep-kernel-projected-branin_MLPKriging_python\")\n", + "suite = lkgallery.Suite(\"deep-kernel-projected-branin_MLPKriging_python\")\n", "print(\"pylibkriging\", compat.version())\n", "\n", "\n", @@ -154,7 +154,7 @@ "if compat.has(\"MLPKriging\"):\n", " X, y, Xt, yt = design.dataset(TESTS[\"hartmann3\"], 40, n_test=100, seed=61)\n", " with suite.runs(\"MLPKriging: fit (persistence)\"):\n", - " lkbench.persistence.check(suite, \"MLPKriging\", lk.MLPKriging(y, X, hidden_dims=[8], d_out=2, activation=\"tanh\",\n", + " lkgallery.persistence.check(suite, \"MLPKriging\", lk.MLPKriging(y, X, hidden_dims=[8], d_out=2, activation=\"tanh\",\n", " kernel=\"matern5_2\"), Xt, pred)" ] }, diff --git a/high-dimension-normalize-screening_Kriging_python.ipynb b/high-dimension-normalize-screening_Kriging_python.ipynb index b3ce1a6..4d0b034 100644 --- a/high-dimension-normalize-screening_Kriging_python.ipynb +++ b/high-dimension-normalize-screening_Kriging_python.ipynb @@ -67,7 +67,7 @@ " \"(PyPI has wheels for CPython 3.10 to 3.12 only). On Colab: Runtime > Change runtime version > \"\n", " \"2026.07 (Python 3.12), then run again.\\n\" + r.stderr[-500:])\n", " importlib.invalidate_caches()\n", - "sys.path.insert(0, str(Path(\"tools\").resolve())) # lkbench, when not installed (Colab, Binder)\n", + "sys.path.insert(0, str(Path(\"tools\").resolve())) # lkgallery, when not installed (Colab, Binder)\n", "print(\"working directory:\", os.getcwd())" ] }, @@ -85,10 +85,10 @@ "import pandas as pd\n", "import pylibkriging as lk\n", "\n", - "import lkbench\n", - "from lkbench import TESTS, design, metrics, compat\n", + "import lkgallery\n", + "from lkgallery import TESTS, design, metrics, compat\n", "\n", - "suite = lkbench.Suite(\"high-dimension-normalize-screening_Kriging_python\")\n", + "suite = lkgallery.Suite(\"high-dimension-normalize-screening_Kriging_python\")\n", "print(\"pylibkriging\", compat.version())\n", "\n", "\n", diff --git a/kernels-trends-interpolation_Kriging_python.ipynb b/kernels-trends-interpolation_Kriging_python.ipynb index efd2b70..fb93860 100644 --- a/kernels-trends-interpolation_Kriging_python.ipynb +++ b/kernels-trends-interpolation_Kriging_python.ipynb @@ -79,7 +79,7 @@ " \"(PyPI has wheels for CPython 3.10 to 3.12 only). On Colab: Runtime > Change runtime version > \"\n", " \"2026.07 (Python 3.12), then run again.\\n\" + r.stderr[-500:])\n", " importlib.invalidate_caches()\n", - "sys.path.insert(0, str(Path(\"tools\").resolve())) # lkbench, when not installed (Colab, Binder)\n", + "sys.path.insert(0, str(Path(\"tools\").resolve())) # lkgallery, when not installed (Colab, Binder)\n", "print(\"working directory:\", os.getcwd())" ] }, @@ -97,10 +97,10 @@ "import matplotlib.pyplot as plt\n", "import pylibkriging as lk\n", "\n", - "import lkbench\n", - "from lkbench import TESTS, design, metrics, compat\n", + "import lkgallery\n", + "from lkgallery import TESTS, design, metrics, compat\n", "\n", - "suite = lkbench.Suite(\"kernels-trends-interpolation_Kriging_python\")\n", + "suite = lkgallery.Suite(\"kernels-trends-interpolation_Kriging_python\")\n", "print(\"pylibkriging\", compat.version())\n", "\n", "KERNELS = [\"gauss\", \"exp\", \"matern3_2\", \"matern5_2\"]\n", diff --git a/large-n_Kriging-vecchia-nystrom_python.ipynb b/large-n_Kriging-vecchia-nystrom_python.ipynb index a28e61f..cf72cc4 100644 --- a/large-n_Kriging-vecchia-nystrom_python.ipynb +++ b/large-n_Kriging-vecchia-nystrom_python.ipynb @@ -72,7 +72,7 @@ " \"(PyPI has wheels for CPython 3.10 to 3.12 only). On Colab: Runtime > Change runtime version > \"\n", " \"2026.07 (Python 3.12), then run again.\\n\" + r.stderr[-500:])\n", " importlib.invalidate_caches()\n", - "sys.path.insert(0, str(Path(\"tools\").resolve())) # lkbench, when not installed (Colab, Binder)\n", + "sys.path.insert(0, str(Path(\"tools\").resolve())) # lkgallery, when not installed (Colab, Binder)\n", "print(\"working directory:\", os.getcwd())" ] }, @@ -90,10 +90,10 @@ "import pandas as pd\n", "import pylibkriging as lk\n", "\n", - "import lkbench\n", - "from lkbench import TESTS, design, metrics, compat\n", + "import lkgallery\n", + "from lkgallery import TESTS, design, metrics, compat\n", "\n", - "suite = lkbench.Suite(\"large-n_Kriging-vecchia-nystrom_python\")\n", + "suite = lkgallery.Suite(\"large-n_Kriging-vecchia-nystrom_python\")\n", "print(\"pylibkriging\", compat.version())\n", "HAS_METHOD = compat.has(\"method\")\n", "if not HAS_METHOD:\n", diff --git a/noisy-data_Kriging-nugget-noise_python.ipynb b/noisy-data_Kriging-nugget-noise_python.ipynb index 4fe95ab..bc98d4e 100644 --- a/noisy-data_Kriging-nugget-noise_python.ipynb +++ b/noisy-data_Kriging-nugget-noise_python.ipynb @@ -71,7 +71,7 @@ " \"(PyPI has wheels for CPython 3.10 to 3.12 only). On Colab: Runtime > Change runtime version > \"\n", " \"2026.07 (Python 3.12), then run again.\\n\" + r.stderr[-500:])\n", " importlib.invalidate_caches()\n", - "sys.path.insert(0, str(Path(\"tools\").resolve())) # lkbench, when not installed (Colab, Binder)\n", + "sys.path.insert(0, str(Path(\"tools\").resolve())) # lkgallery, when not installed (Colab, Binder)\n", "print(\"working directory:\", os.getcwd())" ] }, @@ -87,10 +87,10 @@ "import matplotlib.pyplot as plt\n", "import pylibkriging as lk\n", "\n", - "import lkbench\n", - "from lkbench import TESTS, design, metrics, compat\n", + "import lkgallery\n", + "from lkgallery import TESTS, design, metrics, compat\n", "\n", - "suite = lkbench.Suite(\"noisy-data_Kriging-nugget-noise_python\")\n", + "suite = lkgallery.Suite(\"noisy-data_Kriging-nugget-noise_python\")\n", "print(\"pylibkriging\", compat.version())\n", "\n", "\n", diff --git a/objectives-LL-LOO-LMP-gradients_Kriging_python.ipynb b/objectives-LL-LOO-LMP-gradients_Kriging_python.ipynb index 62f21bf..5705554 100644 --- a/objectives-LL-LOO-LMP-gradients_Kriging_python.ipynb +++ b/objectives-LL-LOO-LMP-gradients_Kriging_python.ipynb @@ -69,7 +69,7 @@ " \"(PyPI has wheels for CPython 3.10 to 3.12 only). On Colab: Runtime > Change runtime version > \"\n", " \"2026.07 (Python 3.12), then run again.\\n\" + r.stderr[-500:])\n", " importlib.invalidate_caches()\n", - "sys.path.insert(0, str(Path(\"tools\").resolve())) # lkbench, when not installed (Colab, Binder)\n", + "sys.path.insert(0, str(Path(\"tools\").resolve())) # lkgallery, when not installed (Colab, Binder)\n", "print(\"working directory:\", os.getcwd())" ] }, @@ -85,10 +85,10 @@ "import pandas as pd\n", "import pylibkriging as lk\n", "\n", - "import lkbench\n", - "from lkbench import TESTS, design, metrics, compat\n", + "import lkgallery\n", + "from lkgallery import TESTS, design, metrics, compat\n", "\n", - "suite = lkbench.Suite(\"objectives-LL-LOO-LMP-gradients_Kriging_python\")\n", + "suite = lkgallery.Suite(\"objectives-LL-LOO-LMP-gradients_Kriging_python\")\n", "print(\"pylibkriging\", compat.version())\n", "CASES = {\"branin\": 25, \"hartmann3\": 40, \"ishigami\": 60, \"friedman\": 50}\n", "KERNELS = [\"gauss\", \"exp\", \"matern3_2\", \"matern5_2\"]\n", diff --git a/robustness-invalid-inputs_Kriging_python.ipynb b/robustness-invalid-inputs_Kriging_python.ipynb index fa122d2..da28053 100644 --- a/robustness-invalid-inputs_Kriging_python.ipynb +++ b/robustness-invalid-inputs_Kriging_python.ipynb @@ -63,7 +63,7 @@ " \"(PyPI has wheels for CPython 3.10 to 3.12 only). On Colab: Runtime > Change runtime version > \"\n", " \"2026.07 (Python 3.12), then run again.\\n\" + r.stderr[-500:])\n", " importlib.invalidate_caches()\n", - "sys.path.insert(0, str(Path(\"tools\").resolve())) # lkbench, when not installed (Colab, Binder)\n", + "sys.path.insert(0, str(Path(\"tools\").resolve())) # lkgallery, when not installed (Colab, Binder)\n", "print(\"working directory:\", os.getcwd())" ] }, @@ -80,10 +80,10 @@ "import numpy as np\n", "import pylibkriging as lk\n", "\n", - "import lkbench\n", - "from lkbench import TESTS, design, compat\n", + "import lkgallery\n", + "from lkgallery import TESTS, design, compat\n", "\n", - "suite = lkbench.Suite(\"robustness-invalid-inputs_Kriging_python\")\n", + "suite = lkgallery.Suite(\"robustness-invalid-inputs_Kriging_python\")\n", "print(\"pylibkriging\", compat.version())\n", "tf = TESTS[\"branin\"]\n", "X, y, Xt, yt = design.dataset(tf, 20, n_test=200, seed=71)\n", diff --git a/save-load-pickle-copy-accessors_Kriging_python.ipynb b/save-load-pickle-copy-accessors_Kriging_python.ipynb index fa98abe..f3b2d89 100644 --- a/save-load-pickle-copy-accessors_Kriging_python.ipynb +++ b/save-load-pickle-copy-accessors_Kriging_python.ipynb @@ -63,7 +63,7 @@ " \"(PyPI has wheels for CPython 3.10 to 3.12 only). On Colab: Runtime > Change runtime version > \"\n", " \"2026.07 (Python 3.12), then run again.\\n\" + r.stderr[-500:])\n", " importlib.invalidate_caches()\n", - "sys.path.insert(0, str(Path(\"tools\").resolve())) # lkbench, when not installed (Colab, Binder)\n", + "sys.path.insert(0, str(Path(\"tools\").resolve())) # lkgallery, when not installed (Colab, Binder)\n", "print(\"working directory:\", os.getcwd())" ] }, @@ -79,10 +79,10 @@ "import numpy as np\n", "import pylibkriging as lk\n", "\n", - "import lkbench\n", - "from lkbench import TESTS, design, compat\n", + "import lkgallery\n", + "from lkgallery import TESTS, design, compat\n", "\n", - "suite = lkbench.Suite(\"save-load-pickle-copy-accessors_Kriging_python\")\n", + "suite = lkgallery.Suite(\"save-load-pickle-copy-accessors_Kriging_python\")\n", "print(\"pylibkriging\", compat.version())\n", "\n", "tf = TESTS[\"hartmann3\"]\n", @@ -131,7 +131,7 @@ "source": [ "for name, make in MODELS.items():\n", " with suite.runs(f\"{name}: fit\"):\n", - " lkbench.persistence.check(suite, name, make(), Xt, predict_any)" + " lkgallery.persistence.check(suite, name, make(), Xt, predict_any)" ] }, { diff --git a/sklearn-estimators_Kriging_python.ipynb b/sklearn-estimators_Kriging_python.ipynb index 9a96927..e9c055d 100644 --- a/sklearn-estimators_Kriging_python.ipynb +++ b/sklearn-estimators_Kriging_python.ipynb @@ -50,7 +50,7 @@ " \"(PyPI has wheels for CPython 3.10 to 3.12 only). On Colab: Runtime > Change runtime version > \"\n", " \"2026.07 (Python 3.12), then run again.\\n\" + r.stderr[-500:])\n", " importlib.invalidate_caches()\n", - "sys.path.insert(0, str(Path(\"tools\").resolve())) # lkbench, when not installed (Colab, Binder)\n", + "sys.path.insert(0, str(Path(\"tools\").resolve())) # lkgallery, when not installed (Colab, Binder)\n", "print(\"working directory:\", os.getcwd())" ] }, @@ -66,10 +66,10 @@ "import pandas as pd\n", "import pylibkriging as lk\n", "\n", - "import lkbench\n", - "from lkbench import TESTS, design, compat\n", + "import lkgallery\n", + "from lkgallery import TESTS, design, compat\n", "\n", - "suite = lkbench.Suite(\"sklearn-estimators_Kriging_python\")\n", + "suite = lkgallery.Suite(\"sklearn-estimators_Kriging_python\")\n", "print(\"pylibkriging\", compat.version())\n", "\n", "tf = TESTS[\"hartmann3\"]\n", diff --git a/test-functions-reference-values_no-model_python.ipynb b/test-functions-reference-values_no-model_python.ipynb index 0be7fba..3d5d3d2 100644 --- a/test-functions-reference-values_no-model_python.ipynb +++ b/test-functions-reference-values_no-model_python.ipynb @@ -15,7 +15,7 @@ "Experiments, Surjanovic & Bingham).\n", "\n", "Every notebook of this gallery relies on the test functions of\n", - "`lkbench.functions` (after the [Virtual Library of Simulation Experiments](https://www.sfu.ca/~ssurjano/)).\n", + "`lkgallery.functions` (after the [Virtual Library of Simulation Experiments](https://www.sfu.ca/~ssurjano/)).\n", "A typo in one of them would silently bias every result built on it, so they are\n", "checked first, against the reference values published on the library pages\n", "(global optima, nominal points) and against the known output ranges." @@ -52,7 +52,7 @@ " \"(PyPI has wheels for CPython 3.10 to 3.12 only). On Colab: Runtime > Change runtime version > \"\n", " \"2026.07 (Python 3.12), then run again.\\n\" + r.stderr[-500:])\n", " importlib.invalidate_caches()\n", - "sys.path.insert(0, str(Path(\"tools\").resolve())) # lkbench, when not installed (Colab, Binder)\n", + "sys.path.insert(0, str(Path(\"tools\").resolve())) # lkgallery, when not installed (Colab, Binder)\n", "print(\"working directory:\", os.getcwd())" ] }, @@ -67,10 +67,10 @@ "import pandas as pd\n", "import matplotlib.pyplot as plt\n", "\n", - "import lkbench\n", - "from lkbench import TESTS, design\n", + "import lkgallery\n", + "from lkgallery import TESTS, design\n", "\n", - "suite = lkbench.Suite(\"test-functions-reference-values_no-model_python\")" + "suite = lkgallery.Suite(\"test-functions-reference-values_no-model_python\")" ] }, { diff --git a/tools/install/python.sh b/tools/install/python.sh index 5411b5c..1547e46 100755 --- a/tools/install/python.sh +++ b/tools/install/python.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Install pylibkriging, then the notebooks' helpers (tools/lkbench) and runner. +# Install pylibkriging, then the notebooks' helpers (tools/lkgallery) and runner. # tools/install/python.sh wheel LIBKRIGING_SRC OUT_DIR only build a wheel from a libKriging checkout # tools/install/python.sh master WHEEL_DIR|LIBKRIGING_SRC install the wheel of WHEEL_DIR, or build it first # tools/install/python.sh release the last release, from PyPI diff --git a/tools/julia/checks.jl b/tools/julia/checks.jl index 9d51365..01b50b3 100644 --- a/tools/julia/checks.jl +++ b/tools/julia/checks.jl @@ -1,8 +1,8 @@ -# Same CSV outputs as lkbench/checks.py (see there), for the Julia notebooks. +# Same CSV outputs as lkgallery/checks.py (see there), for the Julia notebooks. # Paths are relative to the repository root, where the notebooks are executed. using DelimitedFiles, Printf -const RESULTS_DIR = get(ENV, "LKBENCH_RESULTS", "results") +const RESULTS_DIR = get(ENV, "LKGALLERY_RESULTS", "results") mkpath(RESULTS_DIR) function read_known() diff --git a/tools/lkbench/__init__.py b/tools/lkgallery/__init__.py similarity index 100% rename from tools/lkbench/__init__.py rename to tools/lkgallery/__init__.py diff --git a/tools/lkbench/checks.py b/tools/lkgallery/checks.py similarity index 96% rename from tools/lkbench/checks.py rename to tools/lkgallery/checks.py index b94c9a9..d8aafd6 100644 --- a/tools/lkbench/checks.py +++ b/tools/lkgallery/checks.py @@ -33,21 +33,21 @@ def _find_root(): - """The repository root: $LKBENCH_ROOT, or the first directory, from the working + """The repository root: $LKGALLERY_ROOT, or the first directory, from the working directory up, that holds tools/known_issues.csv (notebooks run from the root). Not derived from this file's location, which is site-packages for a non-editable install: the known issues would then be silently ignored.""" - if os.environ.get("LKBENCH_ROOT"): - return Path(os.environ["LKBENCH_ROOT"]).resolve() + if os.environ.get("LKGALLERY_ROOT"): + return Path(os.environ["LKGALLERY_ROOT"]).resolve() for d in [Path.cwd(), *Path.cwd().parents]: if (d / "tools" / "known_issues.csv").is_file(): return d raise RuntimeError("libKriging gallery repository not found from the working directory " - f"{Path.cwd()}: run the notebooks from the repository root, or set LKBENCH_ROOT") + f"{Path.cwd()}: run the notebooks from the repository root, or set LKGALLERY_ROOT") ROOT = _find_root() -RESULTS_DIR = Path(os.environ.get("LKBENCH_RESULTS", ROOT / "results")) +RESULTS_DIR = Path(os.environ.get("LKGALLERY_RESULTS", ROOT / "results")) KNOWN_ISSUES_FILE = ROOT / "tools" / "known_issues.csv" DATA_DIR = ROOT / "data" diff --git a/tools/lkbench/compat.py b/tools/lkgallery/compat.py similarity index 100% rename from tools/lkbench/compat.py rename to tools/lkgallery/compat.py diff --git a/tools/lkbench/design.py b/tools/lkgallery/design.py similarity index 100% rename from tools/lkbench/design.py rename to tools/lkgallery/design.py diff --git a/tools/lkbench/functions.py b/tools/lkgallery/functions.py similarity index 100% rename from tools/lkbench/functions.py rename to tools/lkgallery/functions.py diff --git a/tools/lkbench/metrics.py b/tools/lkgallery/metrics.py similarity index 100% rename from tools/lkbench/metrics.py rename to tools/lkgallery/metrics.py diff --git a/tools/lkbench/persistence.py b/tools/lkgallery/persistence.py similarity index 100% rename from tools/lkbench/persistence.py rename to tools/lkgallery/persistence.py diff --git a/tools/make_data.py b/tools/make_data.py index 62294c5..6dd966b 100644 --- a/tools/make_data.py +++ b/tools/make_data.py @@ -9,7 +9,7 @@ import numpy as np -from lkbench import TESTS, design +from lkgallery import TESTS, design DATA = Path(__file__).resolve().parents[1] / "data" DATASETS = [("branin", 25), ("hartmann3", 40), ("borehole", 80)] diff --git a/tools/notebook_setup.py b/tools/notebook_setup.py index ea00798..a71a002 100644 --- a/tools/notebook_setup.py +++ b/tools/notebook_setup.py @@ -22,5 +22,5 @@ "(PyPI has wheels for CPython 3.10 to 3.12 only). On Colab: Runtime > Change runtime version > " "2026.07 (Python 3.12), then run again.\n" + r.stderr[-500:]) importlib.invalidate_caches() -sys.path.insert(0, str(Path("tools").resolve())) # lkbench, when not installed (Colab, Binder) +sys.path.insert(0, str(Path("tools").resolve())) # lkgallery, when not installed (Colab, Binder) print("working directory:", os.getcwd()) diff --git a/tools/octave/lkb_finalize.m b/tools/octave/lkb_finalize.m index 5a90860..110d8c6 100644 --- a/tools/octave/lkb_finalize.m +++ b/tools/octave/lkb_finalize.m @@ -1,6 +1,6 @@ function lkb_finalize(s) s = lkb_metric(s, "_suite", "_", "elapsed_s", toc(s.t0)); - d = getenv("LKBENCH_RESULTS"); if isempty(d), d = "results"; end + d = getenv("LKGALLERY_RESULTS"); if isempty(d), d = "results"; end if ~exist(d, "dir"), mkdir(d); end write_csv(fullfile(d, [s.name ".checks.csv"]), {"suite", "lang", "check", "severity", "status", "value", "expected", "detail"}, s.checks); write_csv(fullfile(d, [s.name ".metrics.csv"]), {"suite", "lang", "case", "package", "metric", "value"}, s.metrics); diff --git a/tools/octave/lkb_suite.m b/tools/octave/lkb_suite.m index 1cecb2e..3820d7e 100644 --- a/tools/octave/lkb_suite.m +++ b/tools/octave/lkb_suite.m @@ -1,5 +1,5 @@ function s = lkb_suite(name) - % Same CSV outputs as lkbench/checks.py (see there), for the Octave notebooks. + % Same CSV outputs as lkgallery/checks.py (see there), for the Octave notebooks. % A suite is a struct passed around by value: s = lkb_close(s, ...). % Paths are relative to the repository root, where the notebooks are executed. s.name = name; diff --git a/tools/pyproject.toml b/tools/pyproject.toml index 513f897..fb31f56 100644 --- a/tools/pyproject.toml +++ b/tools/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=64"] build-backend = "setuptools.build_meta" [project] -name = "lkbench" +name = "lkgallery" version = "0.1.0" description = "Helpers (test functions, designs, metrics, checks) for the notebooks of the libKriging gallery" requires-python = ">=3.9" @@ -15,4 +15,4 @@ dependencies = ["numpy", "scipy", "pandas", "matplotlib", "jinja2"] # jinja2: D notebooks = ["nbclient", "nbformat", "ipykernel"] [tool.setuptools] -packages = ["lkbench"] +packages = ["lkgallery"] diff --git a/tools/r/checks.R b/tools/r/checks.R index 8eb536e..a7f0994 100644 --- a/tools/r/checks.R +++ b/tools/r/checks.R @@ -1,7 +1,7 @@ -# Same CSV outputs as lkbench/checks.py (see there), for the R notebooks. +# Same CSV outputs as lkgallery/checks.py (see there), for the R notebooks. # Statuses: PASS, FAIL (error-level), WARN (warn-level), SKIP, KNOWN (listed in tools/known_issues.csv). # Paths are relative to the repository root, where the notebooks are executed. -results_dir <- Sys.getenv("LKBENCH_RESULTS", "results") +results_dir <- Sys.getenv("LKGALLERY_RESULTS", "results") dir.create(results_dir, showWarnings = FALSE, recursive = TRUE) known_file <- file.path("tools", "known_issues.csv") if (!file.exists(known_file)) stop("run the notebooks from the repository root (", known_file, " not found)") diff --git a/tools/src/aggregations-large-n_NestedKriging_python.py b/tools/src/aggregations-large-n_NestedKriging_python.py index ebb1499..22827f1 100644 --- a/tools/src/aggregations-large-n_NestedKriging_python.py +++ b/tools/src/aggregations-large-n_NestedKriging_python.py @@ -36,10 +36,10 @@ import pandas as pd import pylibkriging as lk -import lkbench -from lkbench import TESTS, design, metrics, compat +import lkgallery +from lkgallery import TESTS, design, metrics, compat -suite = lkbench.Suite("aggregations-large-n_NestedKriging_python") +suite = lkgallery.Suite("aggregations-large-n_NestedKriging_python") print("pylibkriging", compat.version()) if not compat.has("NestedKriging"): suite.skip("NestedKriging", "not available in this pylibkriging") @@ -161,7 +161,7 @@ def timed(f): if compat.has("NestedKriging"): X, y, Xt, yt = design.dataset(TESTS["hartmann3"], 40, n_test=100, seed=61) with suite.runs("NestedKriging: fit (persistence)"): - lkbench.persistence.check(suite, "NestedKriging", lk.NestedKriging(y, X, "matern5_2", nb_groups=4, seed=1), Xt, pred) + lkgallery.persistence.check(suite, "NestedKriging", lk.NestedKriging(y, X, "matern5_2", nb_groups=4, seed=1), Xt, pred) # %% [markdown] # ## Summary diff --git a/tools/src/cross-binding-reference_all-classes_python.py b/tools/src/cross-binding-reference_all-classes_python.py index 10045b4..c83a08e 100644 --- a/tools/src/cross-binding-reference_all-classes_python.py +++ b/tools/src/cross-binding-reference_all-classes_python.py @@ -49,11 +49,11 @@ import pandas as pd import pylibkriging as lk -import lkbench -from lkbench import compat -from lkbench.checks import DATA_DIR as DATA, RESULTS_DIR +import lkgallery +from lkgallery import compat +from lkgallery.checks import DATA_DIR as DATA, RESULTS_DIR -suite = lkbench.Suite("cross-binding-reference_all-classes_python") +suite = lkgallery.Suite("cross-binding-reference_all-classes_python") print("pylibkriging", compat.version()) DATASETS = ["branin", "hartmann3", "borehole"] rows = [] diff --git a/tools/src/cross-binding_all-classes_octave.m b/tools/src/cross-binding_all-classes_octave.m index c572709..4f5b96d 100644 --- a/tools/src/cross-binding_all-classes_octave.m +++ b/tools/src/cross-binding_all-classes_octave.m @@ -93,7 +93,7 @@ m = kk.predict(Xt); out = put(out, name, "nested_fit_mean", m); % %% -d = getenv("LKBENCH_RESULTS"); if isempty(d), d = "results"; end +d = getenv("LKGALLERY_RESULTS"); if isempty(d), d = "results"; end if ~exist(d, "dir"), mkdir(d); end fid = fopen(fullfile(d, "xbind_octave.csv"), "w"); fprintf(fid, "lang,dataset,quantity,index,value\n"); diff --git a/tools/src/deep-kernel-projected-branin_MLPKriging_python.py b/tools/src/deep-kernel-projected-branin_MLPKriging_python.py index 7ca0393..d5f210b 100644 --- a/tools/src/deep-kernel-projected-branin_MLPKriging_python.py +++ b/tools/src/deep-kernel-projected-branin_MLPKriging_python.py @@ -24,10 +24,10 @@ import numpy as np import pylibkriging as lk -import lkbench -from lkbench import TESTS, design, metrics, compat +import lkgallery +from lkgallery import TESTS, design, metrics, compat -suite = lkbench.Suite("deep-kernel-projected-branin_MLPKriging_python") +suite = lkgallery.Suite("deep-kernel-projected-branin_MLPKriging_python") print("pylibkriging", compat.version()) @@ -87,7 +87,7 @@ def projected(X): if compat.has("MLPKriging"): X, y, Xt, yt = design.dataset(TESTS["hartmann3"], 40, n_test=100, seed=61) with suite.runs("MLPKriging: fit (persistence)"): - lkbench.persistence.check(suite, "MLPKriging", lk.MLPKriging(y, X, hidden_dims=[8], d_out=2, activation="tanh", + lkgallery.persistence.check(suite, "MLPKriging", lk.MLPKriging(y, X, hidden_dims=[8], d_out=2, activation="tanh", kernel="matern5_2"), Xt, pred) # %% [markdown] diff --git a/tools/src/high-dimension-normalize-screening_Kriging_python.py b/tools/src/high-dimension-normalize-screening_Kriging_python.py index a4044fd..1720b54 100644 --- a/tools/src/high-dimension-normalize-screening_Kriging_python.py +++ b/tools/src/high-dimension-normalize-screening_Kriging_python.py @@ -37,10 +37,10 @@ import pandas as pd import pylibkriging as lk -import lkbench -from lkbench import TESTS, design, metrics, compat +import lkgallery +from lkgallery import TESTS, design, metrics, compat -suite = lkbench.Suite("high-dimension-normalize-screening_Kriging_python") +suite = lkgallery.Suite("high-dimension-normalize-screening_Kriging_python") print("pylibkriging", compat.version()) diff --git a/tools/src/kernels-trends-interpolation_Kriging_python.py b/tools/src/kernels-trends-interpolation_Kriging_python.py index 2272316..d0eee20 100644 --- a/tools/src/kernels-trends-interpolation_Kriging_python.py +++ b/tools/src/kernels-trends-interpolation_Kriging_python.py @@ -49,10 +49,10 @@ import matplotlib.pyplot as plt import pylibkriging as lk -import lkbench -from lkbench import TESTS, design, metrics, compat +import lkgallery +from lkgallery import TESTS, design, metrics, compat -suite = lkbench.Suite("kernels-trends-interpolation_Kriging_python") +suite = lkgallery.Suite("kernels-trends-interpolation_Kriging_python") print("pylibkriging", compat.version()) KERNELS = ["gauss", "exp", "matern3_2", "matern5_2"] diff --git a/tools/src/large-n_Kriging-vecchia-nystrom_python.py b/tools/src/large-n_Kriging-vecchia-nystrom_python.py index 4ee296e..dc0fab0 100644 --- a/tools/src/large-n_Kriging-vecchia-nystrom_python.py +++ b/tools/src/large-n_Kriging-vecchia-nystrom_python.py @@ -42,10 +42,10 @@ import pandas as pd import pylibkriging as lk -import lkbench -from lkbench import TESTS, design, metrics, compat +import lkgallery +from lkgallery import TESTS, design, metrics, compat -suite = lkbench.Suite("large-n_Kriging-vecchia-nystrom_python") +suite = lkgallery.Suite("large-n_Kriging-vecchia-nystrom_python") print("pylibkriging", compat.version()) HAS_METHOD = compat.has("method") if not HAS_METHOD: diff --git a/tools/src/noisy-data_Kriging-nugget-noise_python.py b/tools/src/noisy-data_Kriging-nugget-noise_python.py index 84a198f..93978c0 100644 --- a/tools/src/noisy-data_Kriging-nugget-noise_python.py +++ b/tools/src/noisy-data_Kriging-nugget-noise_python.py @@ -39,10 +39,10 @@ import matplotlib.pyplot as plt import pylibkriging as lk -import lkbench -from lkbench import TESTS, design, metrics, compat +import lkgallery +from lkgallery import TESTS, design, metrics, compat -suite = lkbench.Suite("noisy-data_Kriging-nugget-noise_python") +suite = lkgallery.Suite("noisy-data_Kriging-nugget-noise_python") print("pylibkriging", compat.version()) diff --git a/tools/src/objectives-LL-LOO-LMP-gradients_Kriging_python.py b/tools/src/objectives-LL-LOO-LMP-gradients_Kriging_python.py index 8fa8b61..6c3760b 100644 --- a/tools/src/objectives-LL-LOO-LMP-gradients_Kriging_python.py +++ b/tools/src/objectives-LL-LOO-LMP-gradients_Kriging_python.py @@ -37,10 +37,10 @@ import pandas as pd import pylibkriging as lk -import lkbench -from lkbench import TESTS, design, metrics, compat +import lkgallery +from lkgallery import TESTS, design, metrics, compat -suite = lkbench.Suite("objectives-LL-LOO-LMP-gradients_Kriging_python") +suite = lkgallery.Suite("objectives-LL-LOO-LMP-gradients_Kriging_python") print("pylibkriging", compat.version()) CASES = {"branin": 25, "hartmann3": 40, "ishigami": 60, "friedman": 50} KERNELS = ["gauss", "exp", "matern3_2", "matern5_2"] diff --git a/tools/src/robustness-invalid-inputs_Kriging_python.py b/tools/src/robustness-invalid-inputs_Kriging_python.py index fbdcb25..e4d4355 100644 --- a/tools/src/robustness-invalid-inputs_Kriging_python.py +++ b/tools/src/robustness-invalid-inputs_Kriging_python.py @@ -32,10 +32,10 @@ import numpy as np import pylibkriging as lk -import lkbench -from lkbench import TESTS, design, compat +import lkgallery +from lkgallery import TESTS, design, compat -suite = lkbench.Suite("robustness-invalid-inputs_Kriging_python") +suite = lkgallery.Suite("robustness-invalid-inputs_Kriging_python") print("pylibkriging", compat.version()) tf = TESTS["branin"] X, y, Xt, yt = design.dataset(tf, 20, n_test=200, seed=71) diff --git a/tools/src/save-load-pickle-copy-accessors_Kriging_python.py b/tools/src/save-load-pickle-copy-accessors_Kriging_python.py index c474fb6..7895da3 100644 --- a/tools/src/save-load-pickle-copy-accessors_Kriging_python.py +++ b/tools/src/save-load-pickle-copy-accessors_Kriging_python.py @@ -31,10 +31,10 @@ import numpy as np import pylibkriging as lk -import lkbench -from lkbench import TESTS, design, compat +import lkgallery +from lkgallery import TESTS, design, compat -suite = lkbench.Suite("save-load-pickle-copy-accessors_Kriging_python") +suite = lkgallery.Suite("save-load-pickle-copy-accessors_Kriging_python") print("pylibkriging", compat.version()) tf = TESTS["hartmann3"] @@ -71,7 +71,7 @@ def predict_any(k, Xn): # %% for name, make in MODELS.items(): with suite.runs(f"{name}: fit"): - lkbench.persistence.check(suite, name, make(), Xt, predict_any) + lkgallery.persistence.check(suite, name, make(), Xt, predict_any) # %% [markdown] # A copy is independent: updating it leaves the original model unchanged. diff --git a/tools/src/sklearn-estimators_Kriging_python.py b/tools/src/sklearn-estimators_Kriging_python.py index f4078c4..56c3aff 100644 --- a/tools/src/sklearn-estimators_Kriging_python.py +++ b/tools/src/sklearn-estimators_Kriging_python.py @@ -18,10 +18,10 @@ import pandas as pd import pylibkriging as lk -import lkbench -from lkbench import TESTS, design, compat +import lkgallery +from lkgallery import TESTS, design, compat -suite = lkbench.Suite("sklearn-estimators_Kriging_python") +suite = lkgallery.Suite("sklearn-estimators_Kriging_python") print("pylibkriging", compat.version()) tf = TESTS["hartmann3"] diff --git a/tools/src/test-functions-reference-values_no-model_python.py b/tools/src/test-functions-reference-values_no-model_python.py index 34b6be5..36e8eda 100644 --- a/tools/src/test-functions-reference-values_no-model_python.py +++ b/tools/src/test-functions-reference-values_no-model_python.py @@ -9,7 +9,7 @@ # Experiments, Surjanovic & Bingham). # # Every notebook of this gallery relies on the test functions of -# `lkbench.functions` (after the [Virtual Library of Simulation Experiments](https://www.sfu.ca/~ssurjano/)). +# `lkgallery.functions` (after the [Virtual Library of Simulation Experiments](https://www.sfu.ca/~ssurjano/)). # A typo in one of them would silently bias every result built on it, so they are # checked first, against the reference values published on the library pages # (global optima, nominal points) and against the known output ranges. @@ -19,10 +19,10 @@ import pandas as pd import matplotlib.pyplot as plt -import lkbench -from lkbench import TESTS, design +import lkgallery +from lkgallery import TESTS, design -suite = lkbench.Suite("test-functions-reference-values_no-model_python") +suite = lkgallery.Suite("test-functions-reference-values_no-model_python") # %% [markdown] # ## Reference values diff --git a/tools/src/update-simulate_Kriging_python.py b/tools/src/update-simulate_Kriging_python.py index 0ab15a0..a2738a9 100644 --- a/tools/src/update-simulate_Kriging_python.py +++ b/tools/src/update-simulate_Kriging_python.py @@ -28,10 +28,10 @@ import numpy as np import pylibkriging as lk -import lkbench -from lkbench import TESTS, design, compat +import lkgallery +from lkgallery import TESTS, design, compat -suite = lkbench.Suite("update-simulate_Kriging_python") +suite = lkgallery.Suite("update-simulate_Kriging_python") print("pylibkriging", compat.version()) diff --git a/tools/src/vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python.py b/tools/src/vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python.py index 78defd7..aa3fe69 100644 --- a/tools/src/vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python.py +++ b/tools/src/vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python.py @@ -54,11 +54,11 @@ import pandas as pd import pylibkriging as lk -import lkbench -from lkbench import TESTS, design, metrics, compat +import lkgallery +from lkgallery import TESTS, design, metrics, compat warnings.filterwarnings("ignore") -suite = lkbench.Suite("vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python") +suite = lkgallery.Suite("vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python") sk = compat.optional("sklearn.gaussian_process") smt = compat.optional("smt.surrogate_models") ot = compat.optional("openturns") diff --git a/tools/src/warpings-categorical-ordinal_WarpKriging_python.py b/tools/src/warpings-categorical-ordinal_WarpKriging_python.py index 95b1c2c..1ba28fd 100644 --- a/tools/src/warpings-categorical-ordinal_WarpKriging_python.py +++ b/tools/src/warpings-categorical-ordinal_WarpKriging_python.py @@ -28,10 +28,10 @@ import matplotlib.pyplot as plt import pylibkriging as lk -import lkbench -from lkbench import TESTS, design, metrics, compat +import lkgallery +from lkgallery import TESTS, design, metrics, compat -suite = lkbench.Suite("warpings-categorical-ordinal_WarpKriging_python") +suite = lkgallery.Suite("warpings-categorical-ordinal_WarpKriging_python") print("pylibkriging", compat.version()) if not compat.has("WarpKriging"): suite.skip("WarpKriging", "not available in this pylibkriging") @@ -197,7 +197,7 @@ def f_ord(Xo): if compat.has("WarpKriging"): X, y, Xt, yt = design.dataset(TESTS["hartmann3"], 40, n_test=100, seed=61) with suite.runs("WarpKriging: fit (persistence)"): - lkbench.persistence.check(suite, "WarpKriging", lk.WarpKriging(y, X, ["kumaraswamy", "none", "affine"], "matern5_2"), Xt, pred) + lkgallery.persistence.check(suite, "WarpKriging", lk.WarpKriging(y, X, ["kumaraswamy", "none", "affine"], "matern5_2"), Xt, pred) # %% [markdown] # ## Summary diff --git a/update-simulate_Kriging_python.ipynb b/update-simulate_Kriging_python.ipynb index 9af5731..a5c9241 100644 --- a/update-simulate_Kriging_python.ipynb +++ b/update-simulate_Kriging_python.ipynb @@ -61,7 +61,7 @@ " \"(PyPI has wheels for CPython 3.10 to 3.12 only). On Colab: Runtime > Change runtime version > \"\n", " \"2026.07 (Python 3.12), then run again.\\n\" + r.stderr[-500:])\n", " importlib.invalidate_caches()\n", - "sys.path.insert(0, str(Path(\"tools\").resolve())) # lkbench, when not installed (Colab, Binder)\n", + "sys.path.insert(0, str(Path(\"tools\").resolve())) # lkgallery, when not installed (Colab, Binder)\n", "print(\"working directory:\", os.getcwd())" ] }, @@ -76,10 +76,10 @@ "import numpy as np\n", "import pylibkriging as lk\n", "\n", - "import lkbench\n", - "from lkbench import TESTS, design, compat\n", + "import lkgallery\n", + "from lkgallery import TESTS, design, compat\n", "\n", - "suite = lkbench.Suite(\"update-simulate_Kriging_python\")\n", + "suite = lkgallery.Suite(\"update-simulate_Kriging_python\")\n", "print(\"pylibkriging\", compat.version())\n", "\n", "\n", diff --git a/vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python.ipynb b/vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python.ipynb index d6f60ea..b76b8d6 100644 --- a/vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python.ipynb +++ b/vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python.ipynb @@ -62,7 +62,7 @@ " \"(PyPI has wheels for CPython 3.10 to 3.12 only). On Colab: Runtime > Change runtime version > \"\n", " \"2026.07 (Python 3.12), then run again.\\n\" + r.stderr[-500:])\n", " importlib.invalidate_caches()\n", - "sys.path.insert(0, str(Path(\"tools\").resolve())) # lkbench, when not installed (Colab, Binder)\n", + "sys.path.insert(0, str(Path(\"tools\").resolve())) # lkgallery, when not installed (Colab, Binder)\n", "print(\"working directory:\", os.getcwd())" ] }, @@ -114,11 +114,11 @@ "import pandas as pd\n", "import pylibkriging as lk\n", "\n", - "import lkbench\n", - "from lkbench import TESTS, design, metrics, compat\n", + "import lkgallery\n", + "from lkgallery import TESTS, design, metrics, compat\n", "\n", "warnings.filterwarnings(\"ignore\")\n", - "suite = lkbench.Suite(\"vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python\")\n", + "suite = lkgallery.Suite(\"vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python\")\n", "sk = compat.optional(\"sklearn.gaussian_process\")\n", "smt = compat.optional(\"smt.surrogate_models\")\n", "ot = compat.optional(\"openturns\")\n", diff --git a/warpings-categorical-ordinal_WarpKriging_python.ipynb b/warpings-categorical-ordinal_WarpKriging_python.ipynb index 04d8ef7..b32e75c 100644 --- a/warpings-categorical-ordinal_WarpKriging_python.ipynb +++ b/warpings-categorical-ordinal_WarpKriging_python.ipynb @@ -60,7 +60,7 @@ " \"(PyPI has wheels for CPython 3.10 to 3.12 only). On Colab: Runtime > Change runtime version > \"\n", " \"2026.07 (Python 3.12), then run again.\\n\" + r.stderr[-500:])\n", " importlib.invalidate_caches()\n", - "sys.path.insert(0, str(Path(\"tools\").resolve())) # lkbench, when not installed (Colab, Binder)\n", + "sys.path.insert(0, str(Path(\"tools\").resolve())) # lkgallery, when not installed (Colab, Binder)\n", "print(\"working directory:\", os.getcwd())" ] }, @@ -76,10 +76,10 @@ "import matplotlib.pyplot as plt\n", "import pylibkriging as lk\n", "\n", - "import lkbench\n", - "from lkbench import TESTS, design, metrics, compat\n", + "import lkgallery\n", + "from lkgallery import TESTS, design, metrics, compat\n", "\n", - "suite = lkbench.Suite(\"warpings-categorical-ordinal_WarpKriging_python\")\n", + "suite = lkgallery.Suite(\"warpings-categorical-ordinal_WarpKriging_python\")\n", "print(\"pylibkriging\", compat.version())\n", "if not compat.has(\"WarpKriging\"):\n", " suite.skip(\"WarpKriging\", \"not available in this pylibkriging\")\n", @@ -324,7 +324,7 @@ "if compat.has(\"WarpKriging\"):\n", " X, y, Xt, yt = design.dataset(TESTS[\"hartmann3\"], 40, n_test=100, seed=61)\n", " with suite.runs(\"WarpKriging: fit (persistence)\"):\n", - " lkbench.persistence.check(suite, \"WarpKriging\", lk.WarpKriging(y, X, [\"kumaraswamy\", \"none\", \"affine\"], \"matern5_2\"), Xt, pred)" + " lkgallery.persistence.check(suite, \"WarpKriging\", lk.WarpKriging(y, X, [\"kumaraswamy\", \"none\", \"affine\"], \"matern5_2\"), Xt, pred)" ] }, { From 7341666d042e5d10a2e2f0654d6e773b92932873 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 15 Oct 2026 20:59:16 +0000 Subject: [PATCH 09/16] One helper file per language, tools//gallery., with aligned names - Python: the lkgallery package (6 modules, installed from tools/pyproject.toml) becomes the single file tools/python/gallery.py, loaded with "import gallery" (the setup cell of every notebook puts tools/python on sys.path); design / metrics / compat / persistence remain as namespaces; the repository root is found from the file's own location ($LKGALLERY_ROOT overrides); tools/python/requirements.txt replaces the package's dependencies (tools/install/python.sh, make_data.py updated); - R: tools/r/checks.R -> tools/R/gallery.R (source()), Octave: the seven lkb_*.m and read_data.m -> tools/octave/gallery.m, a script of functions (source()), Julia: tools/julia/checks.jl -> tools/julia/gallery.jl (include()); - same names everywhere: suite / Suite, check, check_true, check_less, check_greater, check_between, check_close, check_finite, check_runs, check_skip, metric, suite_summary, suite_finalize, read_data, rmse, q2 (Suite methods in Python: suite.check_close(...) is check_close(s, ...) in R and Octave, check_close!(s, ...) in Julia); the notebooks use them; - README: the table of the helpers of each language. Run locally against libKriging master 41d97bc: every Python notebook, the R and Octave notebooks (cross-binding 49/49 per binding, 0 FAIL); not run: Julia, vs-DiceKriging (CRAN unreachable here). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01QiFEE514dpwS57mcFWDeZG --- .binder/requirements.txt | 4 +- README.md | 24 +- ...gations-large-n_NestedKriging_python.ipynb | 42 +- ...binding-reference_all-classes_python.ipynb | 16 +- cross-binding_all-classes_R.ipynb | 10 +- cross-binding_all-classes_julia.ipynb | 6 +- cross-binding_all-classes_octave.ipynb | 10 +- ...l-projected-branin_MLPKriging_python.ipynb | 24 +- ...n-normalize-screening_Kriging_python.ipynb | 18 +- ...-trends-interpolation_Kriging_python.ipynb | 46 +- large-n_Kriging-vecchia-nystrom_python.ipynb | 40 +- noisy-data_Kriging-nugget-noise_python.ipynb | 40 +- ...-LL-LOO-LMP-gradients_Kriging_python.ipynb | 28 +- ...stness-invalid-inputs_Kriging_python.ipynb | 62 +- ...pickle-copy-accessors_Kriging_python.ipynb | 36 +- sklearn-estimators_Kriging_python.ipynb | 36 +- ...ons-reference-values_no-model_python.ipynb | 22 +- tools/R/gallery.R | 138 ++++ tools/install/python.sh | 5 +- tools/julia/checks.jl | 84 --- tools/julia/gallery.jl | 136 ++++ tools/lkgallery/__init__.py | 4 - tools/lkgallery/checks.py | 199 ------ tools/lkgallery/compat.py | 61 -- tools/lkgallery/design.py | 31 - tools/lkgallery/functions.py | 241 ------- tools/lkgallery/metrics.py | 50 -- tools/lkgallery/persistence.py | 50 -- tools/make_data.py | 4 +- tools/notebook_setup.py | 4 +- tools/octave/gallery.m | 190 +++++ tools/octave/lkb_check.m | 14 - tools/octave/lkb_close.m | 15 - tools/octave/lkb_finalize.m | 24 - tools/octave/lkb_metric.m | 3 - tools/octave/lkb_suite.m | 39 - tools/octave/lkb_summary.m | 13 - tools/octave/read_data.m | 3 - tools/pyproject.toml | 18 - tools/python/gallery.py | 674 ++++++++++++++++++ tools/python/requirements.txt | 11 + tools/r/checks.R | 87 --- ...gregations-large-n_NestedKriging_python.py | 38 +- ...ss-binding-reference_all-classes_python.py | 12 +- tools/src/cross-binding_all-classes_R.R | 10 +- tools/src/cross-binding_all-classes_julia.jl | 6 +- tools/src/cross-binding_all-classes_octave.m | 10 +- ...rnel-projected-branin_MLPKriging_python.py | 20 +- ...sion-normalize-screening_Kriging_python.py | 14 +- ...els-trends-interpolation_Kriging_python.py | 42 +- .../large-n_Kriging-vecchia-nystrom_python.py | 36 +- .../noisy-data_Kriging-nugget-noise_python.py | 36 +- ...ves-LL-LOO-LMP-gradients_Kriging_python.py | 24 +- ...obustness-invalid-inputs_Kriging_python.py | 58 +- ...ad-pickle-copy-accessors_Kriging_python.py | 32 +- .../src/sklearn-estimators_Kriging_python.py | 32 +- ...ctions-reference-values_no-model_python.py | 18 +- tools/src/update-simulate_Kriging_python.py | 42 +- .../src/vs-DiceKriging-RobustGaSP_Kriging_R.R | 12 +- .../src/vs-GaussianProcesses_Kriging_julia.jl | 20 +- tools/src/vs-STK_Kriging_octave.m | 25 +- ...klearn-SMT-OpenTURNS-GPy_Kriging_python.py | 30 +- ...-categorical-ordinal_WarpKriging_python.py | 40 +- update-simulate_Kriging_python.ipynb | 46 +- vs-DiceKriging-RobustGaSP_Kriging_R.ipynb | 12 +- vs-GaussianProcesses_Kriging_julia.ipynb | 20 +- vs-STK_Kriging_octave.ipynb | 25 +- ...arn-SMT-OpenTURNS-GPy_Kriging_python.ipynb | 34 +- ...tegorical-ordinal_WarpKriging_python.ipynb | 44 +- 69 files changed, 1764 insertions(+), 1536 deletions(-) create mode 100644 tools/R/gallery.R delete mode 100644 tools/julia/checks.jl create mode 100644 tools/julia/gallery.jl delete mode 100644 tools/lkgallery/__init__.py delete mode 100644 tools/lkgallery/checks.py delete mode 100644 tools/lkgallery/compat.py delete mode 100644 tools/lkgallery/design.py delete mode 100644 tools/lkgallery/functions.py delete mode 100644 tools/lkgallery/metrics.py delete mode 100644 tools/lkgallery/persistence.py create mode 100644 tools/octave/gallery.m delete mode 100644 tools/octave/lkb_check.m delete mode 100644 tools/octave/lkb_close.m delete mode 100644 tools/octave/lkb_finalize.m delete mode 100644 tools/octave/lkb_metric.m delete mode 100644 tools/octave/lkb_suite.m delete mode 100644 tools/octave/lkb_summary.m delete mode 100644 tools/octave/read_data.m delete mode 100644 tools/pyproject.toml create mode 100644 tools/python/gallery.py create mode 100644 tools/python/requirements.txt delete mode 100644 tools/r/checks.R diff --git a/.binder/requirements.txt b/.binder/requirements.txt index 7122539..9969ea4 100644 --- a/.binder/requirements.txt +++ b/.binder/requirements.txt @@ -1,6 +1,6 @@ # Binder (mybinder.org) environment for the Python notebooks: the released pylibkriging -# (PyPI wheels exist for CPython 3.10 to 3.12, hence runtime.txt) and lkgallery's -# dependencies; lkgallery itself is found in tools/ by the notebooks' setup cell, and the +# (PyPI wheels exist for CPython 3.10 to 3.12, hence runtime.txt) and the dependencies of +# tools/python/gallery.py, which the notebooks load from the repository; the # comparison packages are installed by the notebooks that use them. pylibkriging numpy diff --git a/README.md b/README.md index 5f66a51..de0ce1b 100644 --- a/README.md +++ b/README.md @@ -92,7 +92,7 @@ Notebooks are named `__.ipynb`. Those comparing with another package install it themselves, in their first cells (pip, CRAN, `Pkg.add`, or a clone of STK); a package that cannot be installed is reported (warn-level) and its comparison skipped. Test functions: -[`tools/lkgallery/functions.py`](tools/lkgallery/functions.py), after the +[`tools/python/gallery.py`](tools/python/gallery.py) (section 1), after the [Virtual Library of Simulation Experiments](https://www.sfu.ca/~ssurjano/) (Surjanovic & Bingham). Everything that is not a notebook is under [`tools/`](tools) (sources of the notebooks, helpers, installation of the bindings, @@ -122,10 +122,24 @@ and the installation of rlibkriging (compiled from CRAN sources) and of JLibKrig ## Checks and statuses -Each notebook records its checks with `lkgallery.Suite` (Python) or the small -equivalents of `tools/r`, `tools/octave` and `tools/julia`, under a suite named -after the notebook, and writes `results/.checks.csv` and -`results/.metrics.csv`. +Each notebook records its checks in a suite named after the notebook, and writes +`results/.checks.csv` and `results/.metrics.csv`, with the helpers +of one file per language, loaded by the notebook (not installed): + +| | Python | R | Octave | Julia | +|---|---|---|---|---| +| file | [`tools/python/gallery.py`](tools/python/gallery.py) | [`tools/R/gallery.R`](tools/R/gallery.R) | [`tools/octave/gallery.m`](tools/octave/gallery.m) | [`tools/julia/gallery.jl`](tools/julia/gallery.jl) | +| loaded by | `import gallery` (path set by the setup cell) | `source("tools/R/gallery.R")` | `source("tools/octave/gallery.m")` | `include("tools/julia/gallery.jl")` | +| a suite | `suite = gallery.Suite(name)` | `s <- suite(name)` | `s = suite(name)` | `s = Suite(name)` | +| a check | `suite.check(name, ok, ...)` | `check(s, name, ok, ...)` | `s = check(s, name, ok, ...)` | `check!(s, name, ok; ...)` | +| usual checks | `suite.check_close`, `check_less`, `check_greater`, `check_between`, `check_true`, `check_finite`, `check_runs`, `check_skip`, `check_raises` | `check_close(s, ...)`, ... | `s = check_close(s, ...)`, ... (no `check_runs`: `try`) | `check_close!(s, ...)`, ..., `check_runs!(s, name) do ... end` | +| metrics | `suite.metric(case, package, q2=...)` | `metric(s, case, package, q2 = ...)` | `s = metric(s, case, package, "q2", v)` | `metric!(s, case, package; q2=...)` | +| end of notebook | `suite.summary()`, `suite.finalize()` | `suite_summary(s)`, `suite_finalize(s)` | `suite_summary(s)`, `suite_finalize(s)` | `suite_summary(s)`, `suite_finalize!(s)` | +| data, errors | `gallery.read_data`, `metrics.rmse`, `metrics.q2`, `metrics.nlpd`, ... | `read_data`, `rmse`, `q2`, `nlpd`, `coverage` | `read_data`, `rmse`, `q2` | `read_data`, `rmse`, `q2` | + +The Python file also holds the test functions (`TESTS`), the designs (`design.lhs`, +`design.dataset`), what the installed pylibkriging offers (`compat.has`) and the +persistence checks (`persistence.check`), used by the Python notebooks only. | status | meaning | fails CI | |---|---|---| diff --git a/aggregations-large-n_NestedKriging_python.ipynb b/aggregations-large-n_NestedKriging_python.ipynb index 3e41435..b7ef835 100644 --- a/aggregations-large-n_NestedKriging_python.ipynb +++ b/aggregations-large-n_NestedKriging_python.ipynb @@ -44,7 +44,7 @@ "source": [ "# Setup for running this notebook outside the repository (Colab, Binder, a lone copy):\n", "# nothing to do in the repository itself (CI, local runs). Otherwise the repository\n", - "# (datasets of data/, helpers of tools/) is cloned and the released pylibkriging installed.\n", + "# (datasets of data/, helpers of tools/python/gallery.py) is cloned and the released pylibkriging installed.\n", "import importlib\n", "import os\n", "import subprocess\n", @@ -66,7 +66,7 @@ " \"(PyPI has wheels for CPython 3.10 to 3.12 only). On Colab: Runtime > Change runtime version > \"\n", " \"2026.07 (Python 3.12), then run again.\\n\" + r.stderr[-500:])\n", " importlib.invalidate_caches()\n", - "sys.path.insert(0, str(Path(\"tools\").resolve())) # lkgallery, when not installed (Colab, Binder)\n", + "sys.path.insert(0, str(Path(\"tools/python\").resolve())) # for \"import gallery\" (tools/python/gallery.py)\n", "print(\"working directory:\", os.getcwd())" ] }, @@ -84,13 +84,13 @@ "import pandas as pd\n", "import pylibkriging as lk\n", "\n", - "import lkgallery\n", - "from lkgallery import TESTS, design, metrics, compat\n", + "import gallery\n", + "from gallery import TESTS, design, metrics, compat\n", "\n", - "suite = lkgallery.Suite(\"aggregations-large-n_NestedKriging_python\")\n", + "suite = gallery.Suite(\"aggregations-large-n_NestedKriging_python\")\n", "print(\"pylibkriging\", compat.version())\n", "if not compat.has(\"NestedKriging\"):\n", - " suite.skip(\"NestedKriging\", \"not available in this pylibkriging\")\n", + " suite.check_skip(\"NestedKriging\", \"not available in this pylibkriging\")\n", "\n", "\n", "def pred(k, X, **kw):\n", @@ -132,7 +132,7 @@ " rows = []\n", " for agg in [\"NK\", \"rBCM\"]:\n", " for g in [1, 2, 4, 8, 16, 32]:\n", - " with suite.runs(f\"nested {agg}, {g} groups, fixed theta: fit + predict\"):\n", + " with suite.check_runs(f\"nested {agg}, {g} groups, fixed theta: fit + predict\"):\n", " k, t = timed(lambda: lk.NestedKriging(y, X, \"matern5_2\", nb_groups=g, aggregation=agg, seed=1,\n", " optim=\"none\", parameters=fixed))\n", " m = pred(k, Xt)[0]\n", @@ -141,11 +141,11 @@ " q2=metrics.q2(yt, m)))\n", " conv = pd.DataFrame(rows)\n", " nk = conv[conv[\"aggregation\"] == \"NK\"]\n", - " suite.less(\"nested NK, 1 group == exact predictor (RMS gap / sd)\", nk[\"mean_gap\"].iloc[0], 1e-6)\n", - " suite.less(\"nested NK, 32 groups: RMS gap to the exact predictor / sd\", nk[\"mean_gap\"].iloc[-1], 0.1,\n", + " suite.check_less(\"nested NK, 1 group == exact predictor (RMS gap / sd)\", nk[\"mean_gap\"].iloc[0], 1e-6)\n", + " suite.check_less(\"nested NK, 32 groups: RMS gap to the exact predictor / sd\", nk[\"mean_gap\"].iloc[-1], 0.1,\n", " severity=\"warn\")\n", " rb = conv[conv[\"aggregation\"] == \"rBCM\"]\n", - " suite.true(\"nested NK closer to the exact predictor than rBCM (2 to 32 groups)\",\n", + " suite.check_true(\"nested NK closer to the exact predictor than rBCM (2 to 32 groups)\",\n", " bool(np.all(nk[\"mean_gap\"].to_numpy()[1:] <= rb[\"mean_gap\"].to_numpy()[1:])), severity=\"warn\")\n", " fig, axes = plt.subplots(1, 2, figsize=(10, 3.8))\n", " for agg, g in conv.groupby(\"aggregation\"):\n", @@ -186,18 +186,18 @@ " label = f\"nested {agg} (20 groups)\"\n", " for n in (n_mid, n_big):\n", " case = f\"{fname}/n={n}\"\n", - " with suite.runs(f\"{case}/{label}: fit + predict\"):\n", + " with suite.check_runs(f\"{case}/{label}: fit + predict\"):\n", " k, t = timed(lambda: lk.NestedKriging(y[:n], X[:n], \"matern5_2\", nb_groups=20, aggregation=agg,\n", " seed=1))\n", " m, s = pred(k, Xt)\n", - " suite.finite(f\"{case}/{label}: finite predictions\", m)\n", + " suite.check_finite(f\"{case}/{label}: finite predictions\", m)\n", " res = metrics.all_metrics(yt, m, s)\n", " suite.metric(case, label, fit_s=t, **res)\n", - " suite.greater(f\"{case}/{label}: Q2\", res[\"q2\"], FLOORS[fname] - 0.05)\n", + " suite.check_greater(f\"{case}/{label}: Q2\", res[\"q2\"], FLOORS[fname] - 0.05)\n", " if n == n_mid:\n", - " suite.greater(f\"{case}/{label}: Q2 - Q2(exact)\", res[\"q2\"] - res_e[\"q2\"], -0.02,\n", + " suite.check_greater(f\"{case}/{label}: Q2 - Q2(exact)\", res[\"q2\"] - res_e[\"q2\"], -0.02,\n", " severity=\"warn\")\n", - " suite.less(f\"{case}/{label}: fit time / exact fit time\", t / t_exact, 1.0, severity=\"warn\")\n", + " suite.check_less(f\"{case}/{label}: fit time / exact fit time\", t / t_exact, 1.0, severity=\"warn\")\n", " display(suite.table().round(4))" ] }, @@ -220,17 +220,17 @@ " tf = TESTS[\"branin\"]\n", " X, y, Xt, yt = design.dataset(tf, 200, n_test=500, seed=53)\n", " for agg in [\"NK\", \"PoE\", \"gPoE\", \"BCM\", \"rBCM\"]:\n", - " with suite.runs(f\"nested {agg}: fit\"):\n", + " with suite.check_runs(f\"nested {agg}: fit\"):\n", " k = lk.NestedKriging(y, X, \"matern5_2\", nb_groups=8, aggregation=agg, seed=2)\n", " m, s = (np.ravel(a) for a in k.predict(Xt, return_stdev=True)[:2])\n", - " suite.true(f\"nested {agg}: stdev >= 0\", np.all(s >= 0))\n", + " suite.check_true(f\"nested {agg}: stdev >= 0\", np.all(s >= 0))\n", " suite.metric(\"branin/nested\", agg, **metrics.all_metrics(yt, m, s))\n", " if agg == \"NK\":\n", " mX = np.ravel(k.predict(X, return_stdev=True)[0])\n", - " suite.less(\"nested NK: interpolates (max |m(X)-y| / sd(y))\", np.max(np.abs(mX - y)) / np.std(y),\n", + " suite.check_less(\"nested NK: interpolates (max |m(X)-y| / sd(y))\", np.max(np.abs(mX - y)) / np.std(y),\n", " 1e-4)\n", " display(suite.table().loc[\"branin/nested\"].round(4))\n", - " suite.raises(\"nested NK with a linear trend is rejected\",\n", + " suite.check_raises(\"nested NK with a linear trend is rejected\",\n", " lambda: lk.NestedKriging(y, X, \"matern5_2\", nb_groups=4, aggregation=\"NK\", regmodel=\"linear\"),\n", " severity=\"warn\")" ] @@ -256,8 +256,8 @@ "source": [ "if compat.has(\"NestedKriging\"):\n", " X, y, Xt, yt = design.dataset(TESTS[\"hartmann3\"], 40, n_test=100, seed=61)\n", - " with suite.runs(\"NestedKriging: fit (persistence)\"):\n", - " lkgallery.persistence.check(suite, \"NestedKriging\", lk.NestedKriging(y, X, \"matern5_2\", nb_groups=4, seed=1), Xt, pred)" + " with suite.check_runs(\"NestedKriging: fit (persistence)\"):\n", + " gallery.persistence.check(suite, \"NestedKriging\", lk.NestedKriging(y, X, \"matern5_2\", nb_groups=4, seed=1), Xt, pred)" ] }, { diff --git a/cross-binding-reference_all-classes_python.ipynb b/cross-binding-reference_all-classes_python.ipynb index 8d6912e..b6c9026 100644 --- a/cross-binding-reference_all-classes_python.ipynb +++ b/cross-binding-reference_all-classes_python.ipynb @@ -57,7 +57,7 @@ "source": [ "# Setup for running this notebook outside the repository (Colab, Binder, a lone copy):\n", "# nothing to do in the repository itself (CI, local runs). Otherwise the repository\n", - "# (datasets of data/, helpers of tools/) is cloned and the released pylibkriging installed.\n", + "# (datasets of data/, helpers of tools/python/gallery.py) is cloned and the released pylibkriging installed.\n", "import importlib\n", "import os\n", "import subprocess\n", @@ -79,7 +79,7 @@ " \"(PyPI has wheels for CPython 3.10 to 3.12 only). On Colab: Runtime > Change runtime version > \"\n", " \"2026.07 (Python 3.12), then run again.\\n\" + r.stderr[-500:])\n", " importlib.invalidate_caches()\n", - "sys.path.insert(0, str(Path(\"tools\").resolve())) # lkgallery, when not installed (Colab, Binder)\n", + "sys.path.insert(0, str(Path(\"tools/python\").resolve())) # for \"import gallery\" (tools/python/gallery.py)\n", "print(\"working directory:\", os.getcwd())" ] }, @@ -97,11 +97,11 @@ "import pandas as pd\n", "import pylibkriging as lk\n", "\n", - "import lkgallery\n", - "from lkgallery import compat\n", - "from lkgallery.checks import DATA_DIR as DATA, RESULTS_DIR\n", + "import gallery\n", + "from gallery import compat\n", + "from gallery import DATA_DIR as DATA, RESULTS_DIR\n", "\n", - "suite = lkgallery.Suite(\"cross-binding-reference_all-classes_python\")\n", + "suite = gallery.Suite(\"cross-binding-reference_all-classes_python\")\n", "print(\"pylibkriging\", compat.version())\n", "DATASETS = [\"branin\", \"hartmann3\", \"borehole\"]\n", "rows = []\n", @@ -139,7 +139,7 @@ " ll, grad, _ = kf.logLikelihoodFun(theta0.ravel(), True)\n", " put(name, \"llfun\", ll)\n", " put(name, \"llfun_grad\", grad)\n", - " suite.close(f\"{name}: logLikelihood() == logLikelihoodFun(theta0)\", kf.logLikelihood(), ll, rtol=1e-10)\n", + " suite.check_close(f\"{name}: logLikelihood() == logLikelihoodFun(theta0)\", kf.logLikelihood(), ll, rtol=1e-10)\n", " k = lk.Kriging(y, X, \"matern5_2\")\n", " put(name, \"fit_theta\", k.theta())\n", " put(name, \"fit_ll\", k.logLikelihood())\n", @@ -182,7 +182,7 @@ " put(name, f\"{meth}_fixed_ll\", ka.logLikelihood(method=meth))\n", " put(name, f\"{meth}_fixed_mean\", ka.predict(Xt, return_stdev=False, method=meth)[0])\n", "if not compat.has(\"method\"):\n", - " suite.skip(\"vecchia / nystrom quantities\", \"pylibkriging without the method= argument\")\n", + " suite.check_skip(\"vecchia / nystrom quantities\", \"pylibkriging without the method= argument\")\n", "kw = lk.WarpKriging(y, X, [\"kumaraswamy\", \"kumaraswamy\"], \"matern5_2\")\n", "put(name, \"warp_fit_mean\", kw.predict(Xt, return_stdev=False)[0])\n", "km = lk.MLPKriging(y, X, hidden_dims=[8], d_out=2, activation=\"tanh\", kernel=\"matern5_2\", optim=\"BFGS10\")\n", diff --git a/cross-binding_all-classes_R.ipynb b/cross-binding_all-classes_R.ipynb index 48c3c6e..ddca363 100644 --- a/cross-binding_all-classes_R.ipynb +++ b/cross-binding_all-classes_R.ipynb @@ -26,8 +26,8 @@ "outputs": [], "source": [ "suppressPackageStartupMessages(library(rlibkriging))\n", - "source(\"tools/r/checks.R\")\n", - "s <- new_suite(\"cross-binding_all-classes_R\")\n", + "source(\"tools/R/gallery.R\")\n", + "s <- suite(\"cross-binding_all-classes_R\")\n", "cat(\"rlibkriging\", as.character(packageVersion(\"rlibkriging\")), \"\\n\")\n", "rows <- list()\n", "put <- function(dataset, quantity, values) {\n", @@ -147,7 +147,7 @@ "for (name in c(\"branin\", \"hartmann3\", \"borehole\")) {\n", " yt <- as.numeric(read_data(name, \"yt\"))[1:50]\n", " m <- as.numeric(out$value[out$dataset == name & out$quantity == \"fit_mean\"])\n", - " q <- q2(yt, m); add_metric(s, name, \"rlibkriging default fit\", q2 = q)\n", + " q <- q2(yt, m); metric(s, name, \"rlibkriging default fit\", q2 = q)\n", " plot(yt, m, xlab = \"true\", ylab = \"predicted\", main = sprintf(\"%s: Q2 = %.3f\", name, q), pch = 16, cex = .7)\n", " abline(0, 1, lty = 2)\n", "}\n", @@ -174,7 +174,7 @@ "metadata": {}, "outputs": [], "source": [ - "summary_suite(s)" + "suite_summary(s)" ] }, { @@ -184,7 +184,7 @@ "metadata": {}, "outputs": [], "source": [ - "finalize(s)" + "suite_finalize(s)" ] } ], diff --git a/cross-binding_all-classes_julia.ipynb b/cross-binding_all-classes_julia.ipynb index dfa1879..85f00df 100644 --- a/cross-binding_all-classes_julia.ipynb +++ b/cross-binding_all-classes_julia.ipynb @@ -30,7 +30,7 @@ "outputs": [], "source": [ "import JLibKriging as LK\n", - "include(\"tools/julia/checks.jl\")\n", + "include(\"tools/julia/gallery.jl\")\n", "s = Suite(\"cross-binding_all-classes_julia\")\n", "rows = Vector{Vector{String}}()\n", "record!(dataset, quantity, values) =\n", @@ -171,7 +171,7 @@ "metadata": {}, "outputs": [], "source": [ - "summary!(s)" + "suite_summary(s)" ] }, { @@ -181,7 +181,7 @@ "metadata": {}, "outputs": [], "source": [ - "finalize!(s)" + "suite_finalize!(s)" ] } ], diff --git a/cross-binding_all-classes_octave.ipynb b/cross-binding_all-classes_octave.ipynb index f9e2f75..f806f0d 100644 --- a/cross-binding_all-classes_octave.ipynb +++ b/cross-binding_all-classes_octave.ipynb @@ -26,8 +26,8 @@ "metadata": {}, "outputs": [], "source": [ - "addpath(fullfile(pwd, \"tools\", \"octave\"));\n", - "s = lkb_suite(\"cross-binding_all-classes_octave\");\n", + "source(\"tools/octave/gallery.m\");\n", + "s = suite(\"cross-binding_all-classes_octave\");\n", "out = {};\n", "function out = put(out, dataset, quantity, values)\n", " v = double(values(:));\n", @@ -62,7 +62,7 @@ " [llf, grad] = kf.logLikelihoodFun(theta0', true, false);\n", " out = put(out, name, \"llfun\", llf);\n", " out = put(out, name, \"llfun_grad\", grad);\n", - " s = lkb_close(s, [name \": logLikelihood() == logLikelihoodFun(theta0)\"], ll, llf, 1e-10);\n", + " s = check_close(s, [name \": logLikelihood() == logLikelihoodFun(theta0)\"], ll, llf, 1e-10);\n", " k = Kriging(y, X, \"matern5_2\");\n", " v = k.theta(); out = put(out, name, \"fit_theta\", v);\n", " llk = k.logLikelihood();\n", @@ -189,7 +189,7 @@ "metadata": {}, "outputs": [], "source": [ - "lkb_summary(s);" + "suite_summary(s);" ] }, { @@ -199,7 +199,7 @@ "metadata": {}, "outputs": [], "source": [ - "lkb_finalize(s);" + "suite_finalize(s);" ] } ], diff --git a/deep-kernel-projected-branin_MLPKriging_python.ipynb b/deep-kernel-projected-branin_MLPKriging_python.ipynb index 40e963c..6db067c 100644 --- a/deep-kernel-projected-branin_MLPKriging_python.ipynb +++ b/deep-kernel-projected-branin_MLPKriging_python.ipynb @@ -33,7 +33,7 @@ "source": [ "# Setup for running this notebook outside the repository (Colab, Binder, a lone copy):\n", "# nothing to do in the repository itself (CI, local runs). Otherwise the repository\n", - "# (datasets of data/, helpers of tools/) is cloned and the released pylibkriging installed.\n", + "# (datasets of data/, helpers of tools/python/gallery.py) is cloned and the released pylibkriging installed.\n", "import importlib\n", "import os\n", "import subprocess\n", @@ -55,7 +55,7 @@ " \"(PyPI has wheels for CPython 3.10 to 3.12 only). On Colab: Runtime > Change runtime version > \"\n", " \"2026.07 (Python 3.12), then run again.\\n\" + r.stderr[-500:])\n", " importlib.invalidate_caches()\n", - "sys.path.insert(0, str(Path(\"tools\").resolve())) # lkgallery, when not installed (Colab, Binder)\n", + "sys.path.insert(0, str(Path(\"tools/python\").resolve())) # for \"import gallery\" (tools/python/gallery.py)\n", "print(\"working directory:\", os.getcwd())" ] }, @@ -72,10 +72,10 @@ "import numpy as np\n", "import pylibkriging as lk\n", "\n", - "import lkgallery\n", - "from lkgallery import TESTS, design, metrics, compat\n", + "import gallery\n", + "from gallery import TESTS, design, metrics, compat\n", "\n", - "suite = lkgallery.Suite(\"deep-kernel-projected-branin_MLPKriging_python\")\n", + "suite = gallery.Suite(\"deep-kernel-projected-branin_MLPKriging_python\")\n", "print(\"pylibkriging\", compat.version())\n", "\n", "\n", @@ -110,26 +110,26 @@ " for act in [\"tanh\", \"selu\"]:\n", " for optim in [\"BFGS\", \"BFGS10\"]:\n", " case = f\"MLPKriging({act}, {optim})\"\n", - " with suite.runs(f\"{case}: fit\"):\n", + " with suite.check_runs(f\"{case}: fit\"):\n", " t0 = time.time()\n", " km = lk.MLPKriging(y, X, hidden_dims=[16, 8], d_out=2, activation=act, kernel=\"matern5_2\",\n", " optim=optim)\n", " fit_s = time.time() - t0\n", " m, s = pred(km, Xt)\n", - " suite.finite(f\"{case}: predictions finite\", m)\n", + " suite.check_finite(f\"{case}: predictions finite\", m)\n", " res = metrics.all_metrics(yt, m, s)\n", " suite.metric(\"projected_branin_10d\", f\"MLPKriging {act} {optim}\", fit_s=fit_s,\n", " ll=float(km.logLikelihood()), **res)\n", " if optim == \"BFGS10\":\n", - " suite.greater(f\"{case}: Q2\", res[\"q2\"], 0.8)\n", + " suite.check_greater(f\"{case}: Q2\", res[\"q2\"], 0.8)\n", " else:\n", - " suite.greater(f\"{case}: Q2 - Q2(Kriging)\", res[\"q2\"] - res_k[\"q2\"], -0.1, severity=\"warn\")\n", + " suite.check_greater(f\"{case}: Q2 - Q2(Kriging)\", res[\"q2\"] - res_k[\"q2\"], -0.1, severity=\"warn\")\n", " tab = suite.table().loc[\"projected_branin_10d\"]\n", " tab[\"q2\"].plot.barh(figsize=(6, 3)); plt.xlabel(\"Q² (test set)\"); plt.xlim(min(0, tab[\"q2\"].min()), 1)\n", " plt.title(\"Branin(Wx), x in [0,1]^10\"); plt.show()\n", " display(tab.round(4))\n", "else:\n", - " suite.skip(\"MLPKriging\", \"not available in this pylibkriging\")" + " suite.check_skip(\"MLPKriging\", \"not available in this pylibkriging\")" ] }, { @@ -153,8 +153,8 @@ "source": [ "if compat.has(\"MLPKriging\"):\n", " X, y, Xt, yt = design.dataset(TESTS[\"hartmann3\"], 40, n_test=100, seed=61)\n", - " with suite.runs(\"MLPKriging: fit (persistence)\"):\n", - " lkgallery.persistence.check(suite, \"MLPKriging\", lk.MLPKriging(y, X, hidden_dims=[8], d_out=2, activation=\"tanh\",\n", + " with suite.check_runs(\"MLPKriging: fit (persistence)\"):\n", + " gallery.persistence.check(suite, \"MLPKriging\", lk.MLPKriging(y, X, hidden_dims=[8], d_out=2, activation=\"tanh\",\n", " kernel=\"matern5_2\"), Xt, pred)" ] }, diff --git a/high-dimension-normalize-screening_Kriging_python.ipynb b/high-dimension-normalize-screening_Kriging_python.ipynb index 4d0b034..cdbc8ec 100644 --- a/high-dimension-normalize-screening_Kriging_python.ipynb +++ b/high-dimension-normalize-screening_Kriging_python.ipynb @@ -45,7 +45,7 @@ "source": [ "# Setup for running this notebook outside the repository (Colab, Binder, a lone copy):\n", "# nothing to do in the repository itself (CI, local runs). Otherwise the repository\n", - "# (datasets of data/, helpers of tools/) is cloned and the released pylibkriging installed.\n", + "# (datasets of data/, helpers of tools/python/gallery.py) is cloned and the released pylibkriging installed.\n", "import importlib\n", "import os\n", "import subprocess\n", @@ -67,7 +67,7 @@ " \"(PyPI has wheels for CPython 3.10 to 3.12 only). On Colab: Runtime > Change runtime version > \"\n", " \"2026.07 (Python 3.12), then run again.\\n\" + r.stderr[-500:])\n", " importlib.invalidate_caches()\n", - "sys.path.insert(0, str(Path(\"tools\").resolve())) # lkgallery, when not installed (Colab, Binder)\n", + "sys.path.insert(0, str(Path(\"tools/python\").resolve())) # for \"import gallery\" (tools/python/gallery.py)\n", "print(\"working directory:\", os.getcwd())" ] }, @@ -85,10 +85,10 @@ "import pandas as pd\n", "import pylibkriging as lk\n", "\n", - "import lkgallery\n", - "from lkgallery import TESTS, design, metrics, compat\n", + "import gallery\n", + "from gallery import TESTS, design, metrics, compat\n", "\n", - "suite = lkgallery.Suite(\"high-dimension-normalize-screening_Kriging_python\")\n", + "suite = gallery.Suite(\"high-dimension-normalize-screening_Kriging_python\")\n", "print(\"pylibkriging\", compat.version())\n", "\n", "\n", @@ -129,14 +129,14 @@ " fit_s = time.time() - t0\n", " res = metrics.all_metrics(yt, *pred(k, Ut))\n", " suite.metric(fname, \"unit cube\", fit_s=fit_s, **res)\n", - " suite.greater(f\"{fname}: Q2 (unit cube, matern5_2)\", res[\"q2\"], floor)\n", + " suite.check_greater(f\"{fname}: Q2 (unit cube, matern5_2)\", res[\"q2\"], floor)\n", " # same problem in native units, rescaled internally\n", " Xn, Xtn = tf.scale(U), tf.scale(Ut)\n", - " with suite.runs(f\"{fname}: fit on native inputs with normalize=True\"):\n", + " with suite.check_runs(f\"{fname}: fit on native inputs with normalize=True\"):\n", " kn = lk.Kriging(y, Xn, \"matern5_2\", normalize=True)\n", " res_n = metrics.all_metrics(yt, *pred(kn, Xtn))\n", " suite.metric(fname, \"native + normalize\", **res_n)\n", - " suite.greater(f\"{fname}: Q2(normalize) - Q2(unit cube)\", res_n[\"q2\"] - res[\"q2\"], -0.02)\n", + " suite.check_greater(f\"{fname}: Q2(normalize) - Q2(unit cube)\", res_n[\"q2\"] - res[\"q2\"], -0.02)\n", "tab = suite.table()\n", "tab.round(4)" ] @@ -166,7 +166,7 @@ "theta = np.ravel(k.theta())\n", "inactive = [7, 15] # 0-based indices of x8, x16\n", "active = [0, 3, 4, 11, 18, 19] # x1, x4, x5, x12, x19, x20\n", - "suite.greater(\"welch: min theta(inactive) / max theta(active)\", theta[inactive].min() / theta[active].max(), 1.0,\n", + "suite.check_greater(\"welch: min theta(inactive) / max theta(active)\", theta[inactive].min() / theta[active].max(), 1.0,\n", " severity=\"warn\")\n", "colors = [\"tab:red\" if i in active else \"tab:gray\" if i in inactive else \"tab:blue\" for i in range(20)]\n", "plt.figure(figsize=(9, 3))\n", diff --git a/kernels-trends-interpolation_Kriging_python.ipynb b/kernels-trends-interpolation_Kriging_python.ipynb index fb93860..7242018 100644 --- a/kernels-trends-interpolation_Kriging_python.ipynb +++ b/kernels-trends-interpolation_Kriging_python.ipynb @@ -57,7 +57,7 @@ "source": [ "# Setup for running this notebook outside the repository (Colab, Binder, a lone copy):\n", "# nothing to do in the repository itself (CI, local runs). Otherwise the repository\n", - "# (datasets of data/, helpers of tools/) is cloned and the released pylibkriging installed.\n", + "# (datasets of data/, helpers of tools/python/gallery.py) is cloned and the released pylibkriging installed.\n", "import importlib\n", "import os\n", "import subprocess\n", @@ -79,7 +79,7 @@ " \"(PyPI has wheels for CPython 3.10 to 3.12 only). On Colab: Runtime > Change runtime version > \"\n", " \"2026.07 (Python 3.12), then run again.\\n\" + r.stderr[-500:])\n", " importlib.invalidate_caches()\n", - "sys.path.insert(0, str(Path(\"tools\").resolve())) # lkgallery, when not installed (Colab, Binder)\n", + "sys.path.insert(0, str(Path(\"tools/python\").resolve())) # for \"import gallery\" (tools/python/gallery.py)\n", "print(\"working directory:\", os.getcwd())" ] }, @@ -97,10 +97,10 @@ "import matplotlib.pyplot as plt\n", "import pylibkriging as lk\n", "\n", - "import lkgallery\n", - "from lkgallery import TESTS, design, metrics, compat\n", + "import gallery\n", + "from gallery import TESTS, design, metrics, compat\n", "\n", - "suite = lkgallery.Suite(\"kernels-trends-interpolation_Kriging_python\")\n", + "suite = gallery.Suite(\"kernels-trends-interpolation_Kriging_python\")\n", "print(\"pylibkriging\", compat.version())\n", "\n", "KERNELS = [\"gauss\", \"exp\", \"matern3_2\", \"matern5_2\"]\n", @@ -146,35 +146,35 @@ " \"quadratic\": 1 + 2 * tf.d + tf.d * (tf.d - 1) // 2}[trend]\n", " case = f\"{fname}/{kernel}/{trend}\"\n", " if n_beta >= n - 1:\n", - " suite.skip(case, f\"{n_beta} trend coefficients for n={n}\")\n", + " suite.check_skip(case, f\"{n_beta} trend coefficients for n={n}\")\n", " continue\n", " t0 = time.time()\n", " try:\n", " k = lk.Kriging(y, X, kernel, regmodel=trend, optim=\"BFGS\")\n", " except Exception as exc: # noqa: BLE001\n", - " suite.true(f\"{case}: fit\", False, detail=str(exc)[:200])\n", + " suite.check_true(f\"{case}: fit\", False, detail=str(exc)[:200])\n", " continue\n", " fit_s = time.time() - t0\n", " m, s, _ = predict(k, Xt)\n", " res = metrics.all_metrics(yt, m, s)\n", " suite.metric(case, \"pylibkriging\", fit_s=fit_s, **res)\n", " theta, sigma2 = np.asarray(k.theta()).ravel(), float(k.sigma2())\n", - " suite.true(f\"{case}: theta, sigma2 finite and > 0\",\n", + " suite.check_true(f\"{case}: theta, sigma2 finite and > 0\",\n", " np.all(np.isfinite(theta)) and np.all(theta > 0) and np.isfinite(sigma2) and sigma2 > 0,\n", " detail=f\"theta={theta}, sigma2={sigma2:.3g}\")\n", " # interpolation at the design points (relative to the spread of y)\n", " md, sd, _ = predict(k, X)\n", " scale = np.std(y)\n", " sev = \"warn\" if kernel == \"gauss\" else \"error\"\n", - " suite.less(f\"{case}: interpolates y (max |m(X)-y| / sd(y))\", np.max(np.abs(md - y)) / scale, 1e-4,\n", + " suite.check_less(f\"{case}: interpolates y (max |m(X)-y| / sd(y))\", np.max(np.abs(md - y)) / scale, 1e-4,\n", " severity=sev)\n", - " suite.less(f\"{case}: stdev(X) ~ 0 (max sd / sd(y))\", np.max(sd) / scale, 1e-2, severity=sev)\n", + " suite.check_less(f\"{case}: stdev(X) ~ 0 (max sd / sd(y))\", np.max(sd) / scale, 1e-2, severity=sev)\n", " if kernel == \"matern5_2\" and trend == \"constant\":\n", - " suite.greater(f\"{case}: Q2 (single-start BFGS)\", res[\"q2\"], q2_floor, severity=\"warn\")\n", + " suite.check_greater(f\"{case}: Q2 (single-start BFGS)\", res[\"q2\"], q2_floor, severity=\"warn\")\n", " k10 = lk.Kriging(y, X, kernel, regmodel=trend, optim=\"BFGS10\")\n", " res10 = metrics.all_metrics(yt, *predict(k10, Xt)[:2])\n", " suite.metric(case, \"pylibkriging BFGS10\", **res10)\n", - " suite.greater(f\"{case}: Q2 (BFGS10)\", res10[\"q2\"], q2_floor)" + " suite.check_greater(f\"{case}: Q2 (BFGS10)\", res10[\"q2\"], q2_floor)" ] }, { @@ -210,10 +210,10 @@ "outputs": [], "source": [ "q = tab.set_index([\"function\", \"kernel\", \"trend\"])[\"q2\"]\n", - "suite.true(\"branin: exp is the worst kernel (constant trend)\",\n", + "suite.check_true(\"branin: exp is the worst kernel (constant trend)\",\n", " q[\"branin\", \"exp\", \"constant\"] <= min(q[\"branin\", k, \"constant\"] for k in KERNELS),\n", " severity=\"warn\")\n", - "suite.greater(\"lim_polynomial/matern5_2: Q2(quadratic) - Q2(constant)\",\n", + "suite.check_greater(\"lim_polynomial/matern5_2: Q2(quadratic) - Q2(constant)\",\n", " q[\"lim_polynomial\", \"matern5_2\", \"quadratic\"] - q[\"lim_polynomial\", \"matern5_2\", \"constant\"], -1e-3,\n", " severity=\"warn\")" ] @@ -238,16 +238,16 @@ "k = lk.Kriging(y, X, \"matern5_2\")\n", "m, s, cov = predict(k, Xt, return_cov=True)\n", "cov = np.asarray(cov)\n", - "suite.close(\"predict: diag(cov) == stdev^2\", np.diag(cov), s ** 2, rtol=1e-6, atol=1e-10)\n", - "suite.close(\"predict: cov symmetric\", cov, cov.T, rtol=1e-10, atol=1e-12)\n", - "suite.greater(\"predict: cov PSD (min eigenvalue / max)\", np.linalg.eigvalsh(cov).min() / np.abs(cov).max(), -1e-8)\n", + "suite.check_close(\"predict: diag(cov) == stdev^2\", np.diag(cov), s ** 2, rtol=1e-6, atol=1e-10)\n", + "suite.check_close(\"predict: cov symmetric\", cov, cov.T, rtol=1e-10, atol=1e-12)\n", + "suite.check_greater(\"predict: cov PSD (min eigenvalue / max)\", np.linalg.eigvalsh(cov).min() / np.abs(cov).max(), -1e-8)\n", "m1 = np.array([predict(k, Xt[i:i + 1])[0][0] for i in range(len(Xt))])\n", "s1 = np.array([predict(k, Xt[i:i + 1])[1][0] for i in range(len(Xt))])\n", - "suite.close(\"predict: batch == point by point (mean)\", m, m1, rtol=1e-8, atol=1e-10 * np.std(y))\n", - "suite.close(\"predict: batch == point by point (stdev)\", s, s1, rtol=1e-6, atol=1e-10 * np.std(y))\n", + "suite.check_close(\"predict: batch == point by point (mean)\", m, m1, rtol=1e-8, atol=1e-10 * np.std(y))\n", + "suite.check_close(\"predict: batch == point by point (stdev)\", s, s1, rtol=1e-6, atol=1e-10 * np.std(y))\n", "# the mean is a function of the point only: duplicating/permuting the batch changes nothing\n", "perm = np.random.default_rng(0).permutation(len(Xt))\n", - "suite.close(\"predict: invariant to the order of the batch\", predict(k, Xt[perm])[0], m[perm], rtol=1e-10, atol=1e-10)" + "suite.check_close(\"predict: invariant to the order of the batch\", predict(k, Xt[perm])[0], m[perm], rtol=1e-10, atol=1e-10)" ] }, { @@ -268,7 +268,7 @@ "metadata": {}, "outputs": [], "source": [ - "with suite.runs(\"predict(return_deriv=True)\"):\n", + "with suite.check_runs(\"predict(return_deriv=True)\"):\n", " out = k.predict(Xt[:5], True, False, True)\n", " dm, ds = np.asarray(out[3]), np.asarray(out[4])\n", " h = 1e-6\n", @@ -278,8 +278,8 @@ " mp, sp, _ = predict(k, Xt[:5] + e)\n", " mm, sm, _ = predict(k, Xt[:5] - e)\n", " fd_m[:, j], fd_s[:, j] = (mp - mm) / (2 * h), (sp - sm) / (2 * h)\n", - " suite.close(\"d mean / dx == finite differences\", dm, fd_m, rtol=1e-4, atol=1e-4 * np.abs(fd_m).max())\n", - " suite.close(\"d stdev / dx == finite differences\", ds, fd_s, rtol=1e-3, atol=1e-3 * np.abs(fd_s).max())" + " suite.check_close(\"d mean / dx == finite differences\", dm, fd_m, rtol=1e-4, atol=1e-4 * np.abs(fd_m).max())\n", + " suite.check_close(\"d stdev / dx == finite differences\", ds, fd_s, rtol=1e-3, atol=1e-3 * np.abs(fd_s).max())" ] }, { diff --git a/large-n_Kriging-vecchia-nystrom_python.ipynb b/large-n_Kriging-vecchia-nystrom_python.ipynb index cf72cc4..d83bfe0 100644 --- a/large-n_Kriging-vecchia-nystrom_python.ipynb +++ b/large-n_Kriging-vecchia-nystrom_python.ipynb @@ -50,7 +50,7 @@ "source": [ "# Setup for running this notebook outside the repository (Colab, Binder, a lone copy):\n", "# nothing to do in the repository itself (CI, local runs). Otherwise the repository\n", - "# (datasets of data/, helpers of tools/) is cloned and the released pylibkriging installed.\n", + "# (datasets of data/, helpers of tools/python/gallery.py) is cloned and the released pylibkriging installed.\n", "import importlib\n", "import os\n", "import subprocess\n", @@ -72,7 +72,7 @@ " \"(PyPI has wheels for CPython 3.10 to 3.12 only). On Colab: Runtime > Change runtime version > \"\n", " \"2026.07 (Python 3.12), then run again.\\n\" + r.stderr[-500:])\n", " importlib.invalidate_caches()\n", - "sys.path.insert(0, str(Path(\"tools\").resolve())) # lkgallery, when not installed (Colab, Binder)\n", + "sys.path.insert(0, str(Path(\"tools/python\").resolve())) # for \"import gallery\" (tools/python/gallery.py)\n", "print(\"working directory:\", os.getcwd())" ] }, @@ -90,14 +90,14 @@ "import pandas as pd\n", "import pylibkriging as lk\n", "\n", - "import lkgallery\n", - "from lkgallery import TESTS, design, metrics, compat\n", + "import gallery\n", + "from gallery import TESTS, design, metrics, compat\n", "\n", - "suite = lkgallery.Suite(\"large-n_Kriging-vecchia-nystrom_python\")\n", + "suite = gallery.Suite(\"large-n_Kriging-vecchia-nystrom_python\")\n", "print(\"pylibkriging\", compat.version())\n", "HAS_METHOD = compat.has(\"method\")\n", "if not HAS_METHOD:\n", - " suite.skip(\"method=vecchia/nystrom\", \"pylibkriging without the method= argument\")\n", + " suite.check_skip(\"method=vecchia/nystrom\", \"pylibkriging without the method= argument\")\n", "\n", "\n", "def pred(k, X, **kw):\n", @@ -132,17 +132,17 @@ " ke = lk.Kriging(y, X, \"matern5_2\")\n", " fixed = {\"theta\": np.asarray(ke.theta()).reshape(1, -1), \"is_theta_estim\": False}\n", " kv = lk.Kriging(y, X, \"matern5_2\", optim=\"none\", parameters=fixed, method=f\"vecchia({len(y) - 1})\")\n", - " suite.close(\"vecchia(n-1) LL == exact LL (same theta)\", kv.logLikelihood(method=\"vecchia\"),\n", + " suite.check_close(\"vecchia(n-1) LL == exact LL (same theta)\", kv.logLikelihood(method=\"vecchia\"),\n", " ke.logLikelihood(), rtol=1e-6)\n", - " suite.close(\"vecchia(n-1) predict == exact predict\", pred(kv, Xt, method=\"vecchia\")[0], pred(ke, Xt)[0],\n", + " suite.check_close(\"vecchia(n-1) predict == exact predict\", pred(kv, Xt, method=\"vecchia\")[0], pred(ke, Xt)[0],\n", " rtol=1e-5, atol=1e-6 * np.std(y), severity=\"warn\")\n", " kn = lk.Kriging(y, X, \"matern5_2\", optim=\"none\", parameters=fixed, method=f\"nystrom({len(y)})\")\n", - " suite.close(\"nystrom(n) LL == exact LL (same theta)\", kn.logLikelihood(method=\"nystrom\"), ke.logLikelihood(),\n", + " suite.check_close(\"nystrom(n) LL == exact LL (same theta)\", kn.logLikelihood(method=\"nystrom\"), ke.logLikelihood(),\n", " rtol=1e-3, severity=\"warn\")\n", " # the method must be explicit after an approximate fit\n", - " suite.raises(\"nystrom fit: predict(method='exact') raises\", lambda: kn.predict(Xt, return_stdev=True,\n", + " suite.check_raises(\"nystrom fit: predict(method='exact') raises\", lambda: kn.predict(Xt, return_stdev=True,\n", " method=\"exact\"))\n", - " suite.true(\"method() reports the fit method\", kv.method().startswith(\"vecchia\") and\n", + " suite.check_true(\"method() reports the fit method\", kv.method().startswith(\"vecchia\") and\n", " kn.method().startswith(\"nystrom\"), detail=f\"{kv.method()}, {kn.method()}\")" ] }, @@ -177,7 +177,7 @@ " rows = []\n", " for meth, sizes in [(\"vecchia\", [2, 5, 10, 20, 40, 80, 160, n - 1]), (\"nystrom\", [10, 25, 50, 100, 200, n])]:\n", " for size in sizes:\n", - " with suite.runs(f\"{meth}({size}) at fixed theta: fit + predict\"):\n", + " with suite.check_runs(f\"{meth}({size}) at fixed theta: fit + predict\"):\n", " ka = lk.Kriging(y, X, \"matern5_2\", optim=\"none\", parameters=fixed, method=f\"{meth}({size})\")\n", " rows.append(dict(method=meth, size=size,\n", " ll_gap=abs(ka.logLikelihood(method=meth) - ll_e) / abs(ll_e),\n", @@ -186,8 +186,8 @@ " for meth, g in conv.groupby(\"method\"):\n", " suite.metric(f\"hartmann3/n={n}/fixed theta\", meth, **{f\"ll_gap_{s}\": v for s, v in zip(g[\"size\"], g[\"ll_gap\"])})\n", " for col, lim in [(\"ll_gap\", 1e-6), (\"mean_gap\", 1e-2)]:\n", - " suite.less(f\"{meth}: {col} at the limit size ({g['size'].iloc[-1]})\", g[col].iloc[-1], lim)\n", - " suite.true(f\"{meth}: {col} decreases with the size\", bool(np.all(np.diff(g[col].to_numpy()) < 0)),\n", + " suite.check_less(f\"{meth}: {col} at the limit size ({g['size'].iloc[-1]})\", g[col].iloc[-1], lim)\n", + " suite.check_true(f\"{meth}: {col} decreases with the size\", bool(np.all(np.diff(g[col].to_numpy()) < 0)),\n", " detail=\", \".join(f\"{v:.2g}\" for v in g[col]), severity=\"warn\")\n", " fig, axes = plt.subplots(1, 2, figsize=(10, 3.8))\n", " for ax, col, label in [(axes[0], \"ll_gap\", \"|LL − LL(exact)| / |LL(exact)|\"),\n", @@ -223,7 +223,7 @@ " ke, t_exact = timed(lambda: lk.Kriging(ym, Xm, \"matern5_2\"))\n", " res_e = metrics.all_metrics(yt, *pred(ke, Xt))\n", " suite.metric(f\"{fname}/n={n_mid}\", \"exact\", fit_s=t_exact, **res_e)\n", - " suite.greater(f\"{fname}/n={n_mid}: Q2 exact\", res_e[\"q2\"], FLOORS[fname])\n", + " suite.check_greater(f\"{fname}/n={n_mid}: Q2 exact\", res_e[\"q2\"], FLOORS[fname])\n", " variants = {}\n", " if HAS_METHOD:\n", " variants.update({\n", @@ -238,18 +238,18 @@ " for label, make in variants.items():\n", " for n in (n_mid, n_big):\n", " case = f\"{fname}/n={n}\"\n", - " with suite.runs(f\"{case}/{label}: fit + predict\"):\n", + " with suite.check_runs(f\"{case}/{label}: fit + predict\"):\n", " (k, meth), t = timed(lambda: make(X[:n], y[:n]))\n", " m, s = pred(k, Xt, method=meth) if meth else (\n", " lambda o: (np.ravel(o[0]), np.ravel(o[1])))(k.predict(Xt, return_stdev=True))\n", - " suite.finite(f\"{case}/{label}: finite predictions\", m)\n", + " suite.check_finite(f\"{case}/{label}: finite predictions\", m)\n", " res = metrics.all_metrics(yt, m, s)\n", " suite.metric(case, label, fit_s=t, **res)\n", - " suite.greater(f\"{case}/{label}: Q2\", res[\"q2\"], FLOORS[fname] - 0.05)\n", + " suite.check_greater(f\"{case}/{label}: Q2\", res[\"q2\"], FLOORS[fname] - 0.05)\n", " if n == n_mid:\n", - " suite.greater(f\"{case}/{label}: Q2 - Q2(exact)\", res[\"q2\"] - res_e[\"q2\"], -0.02,\n", + " suite.check_greater(f\"{case}/{label}: Q2 - Q2(exact)\", res[\"q2\"] - res_e[\"q2\"], -0.02,\n", " severity=\"warn\")\n", - " suite.less(f\"{case}/{label}: fit time / exact fit time\", t / t_exact, 1.0, severity=\"warn\")\n", + " suite.check_less(f\"{case}/{label}: fit time / exact fit time\", t / t_exact, 1.0, severity=\"warn\")\n", "suite.table().round(4)" ] }, diff --git a/noisy-data_Kriging-nugget-noise_python.ipynb b/noisy-data_Kriging-nugget-noise_python.ipynb index bc98d4e..68ad63d 100644 --- a/noisy-data_Kriging-nugget-noise_python.ipynb +++ b/noisy-data_Kriging-nugget-noise_python.ipynb @@ -49,7 +49,7 @@ "source": [ "# Setup for running this notebook outside the repository (Colab, Binder, a lone copy):\n", "# nothing to do in the repository itself (CI, local runs). Otherwise the repository\n", - "# (datasets of data/, helpers of tools/) is cloned and the released pylibkriging installed.\n", + "# (datasets of data/, helpers of tools/python/gallery.py) is cloned and the released pylibkriging installed.\n", "import importlib\n", "import os\n", "import subprocess\n", @@ -71,7 +71,7 @@ " \"(PyPI has wheels for CPython 3.10 to 3.12 only). On Colab: Runtime > Change runtime version > \"\n", " \"2026.07 (Python 3.12), then run again.\\n\" + r.stderr[-500:])\n", " importlib.invalidate_caches()\n", - "sys.path.insert(0, str(Path(\"tools\").resolve())) # lkgallery, when not installed (Colab, Binder)\n", + "sys.path.insert(0, str(Path(\"tools/python\").resolve())) # for \"import gallery\" (tools/python/gallery.py)\n", "print(\"working directory:\", os.getcwd())" ] }, @@ -87,10 +87,10 @@ "import matplotlib.pyplot as plt\n", "import pylibkriging as lk\n", "\n", - "import lkgallery\n", - "from lkgallery import TESTS, design, metrics, compat\n", + "import gallery\n", + "from gallery import TESTS, design, metrics, compat\n", "\n", - "suite = lkgallery.Suite(\"noisy-data_Kriging-nugget-noise_python\")\n", + "suite = gallery.Suite(\"noisy-data_Kriging-nugget-noise_python\")\n", "print(\"pylibkriging\", compat.version())\n", "\n", "\n", @@ -122,26 +122,26 @@ " m, s = pred(k, Xt)\n", " res = metrics.all_metrics(yt, m, s)\n", " suite.metric(fname, \"nugget\", nugget=nugget, true_var=sd ** 2, **res)\n", - " suite.between(f\"{fname}: nugget / true noise variance\", nugget / sd ** 2, 1 / 3, 3)\n", + " suite.check_between(f\"{fname}: nugget / true noise variance\", nugget / sd ** 2, 1 / 3, 3)\n", " k1 = lk.Kriging(y, X, \"matern5_2\", noise=\"nugget\") # default single start\n", " suite.metric(fname, \"nugget, single start\", nugget=float(np.ravel(k1.nugget())[0]), ll=float(k1.logLikelihood()),\n", " **metrics.all_metrics(yt, *pred(k1, Xt)))\n", - " suite.greater(f\"{fname}: LL(single start) - LL(10 restarts)\", k1.logLikelihood() - k.logLikelihood(), -1,\n", + " suite.check_greater(f\"{fname}: LL(single start) - LL(10 restarts)\", k1.logLikelihood() - k.logLikelihood(), -1,\n", " severity=\"warn\")\n", - " suite.greater(f\"{fname}: Q2 vs noise-free truth\", res[\"q2\"], q2_floor)\n", + " suite.check_greater(f\"{fname}: Q2 vs noise-free truth\", res[\"q2\"], q2_floor)\n", " mX, _ = pred(k, X)\n", " mX_near, _ = pred(k, np.clip(X + 1e-7, 0, 1))\n", " suite.metric(fname, \"nugget\", rms_resid_at_X=metrics.rmse(y, mX) / sd,\n", " rms_resid_near_X=metrics.rmse(y, mX_near) / sd)\n", - " suite.greater(f\"{fname}: smooths the noisy data next to the design points (rms residual / sd)\",\n", + " suite.check_greater(f\"{fname}: smooths the noisy data next to the design points (rms residual / sd)\",\n", " metrics.rmse(y, mX_near) / sd, 0.3)\n", - " suite.between(f\"{fname}: coverage of the 95% intervals (noise-free truth)\", res[\"coverage95\"], 0.80, 1.0,\n", + " suite.check_between(f\"{fname}: coverage of the 95% intervals (noise-free truth)\", res[\"coverage95\"], 0.80, 1.0,\n", " severity=\"warn\")\n", " # the noise-free model on the same noisy data interpolates, and generalizes worse\n", " k0 = lk.Kriging(y, X, \"matern5_2\")\n", " m0, s0 = pred(k0, Xt)\n", " suite.metric(fname, \"noise-free fit\", **metrics.all_metrics(yt, m0, s0))\n", - " suite.less(f\"{fname}: RMSE(nugget) < RMSE(interpolating fit)\", res[\"rmse\"], metrics.rmse(yt, m0),\n", + " suite.check_less(f\"{fname}: RMSE(nugget) < RMSE(interpolating fit)\", res[\"rmse\"], metrics.rmse(yt, m0),\n", " severity=\"warn\")\n", "suite.table()" ] @@ -167,13 +167,13 @@ "theta = np.asarray(k0.theta()).reshape(1, -1)\n", "fixed = {\"theta\": theta, \"is_theta_estim\": False, \"sigma2\": float(k0.sigma2()), \"is_sigma2_estim\": False}\n", "kf = lk.Kriging(y, X, \"matern5_2\", optim=\"none\", parameters=fixed)\n", - "with suite.runs(\"noise=zeros, theta and sigma2 fixed: fit\"):\n", + "with suite.check_runs(\"noise=zeros, theta and sigma2 fixed: fit\"):\n", " kz = lk.Kriging(y, X, \"matern5_2\", noise=np.zeros(len(y)), optim=\"none\", parameters=fixed)\n", - " suite.close(\"noise=zeros: same mean as the noise-free model\", pred(kz, Xt)[0], pred(kf, Xt)[0], rtol=1e-6,\n", + " suite.check_close(\"noise=zeros: same mean as the noise-free model\", pred(kz, Xt)[0], pred(kf, Xt)[0], rtol=1e-6,\n", " atol=1e-8 * np.std(y))\n", - " suite.close(\"noise=zeros: same stdev as the noise-free model\", pred(kz, Xt)[1], pred(kf, Xt)[1], rtol=1e-4,\n", + " suite.check_close(\"noise=zeros: same stdev as the noise-free model\", pred(kz, Xt)[1], pred(kf, Xt)[1], rtol=1e-4,\n", " atol=1e-6 * np.std(y))\n", - "with suite.runs(\"noise=zeros, sigma2 estimated: fit\", severity=\"warn\"):\n", + "with suite.check_runs(\"noise=zeros, sigma2 estimated: fit\", severity=\"warn\"):\n", " lk.Kriging(y, X, \"matern5_2\", noise=np.zeros(len(y)), optim=\"none\",\n", " parameters={\"theta\": theta, \"is_theta_estim\": False})" ] @@ -204,10 +204,10 @@ "res_nug = metrics.all_metrics(yt, *pred(k_nug, Xt))\n", "suite.metric(\"gramacy_lee_2012/hetero\", \"known variances\", **res_known)\n", "suite.metric(\"gramacy_lee_2012/hetero\", \"nugget\", **res_nug)\n", - "suite.finite(\"hetero: predictions finite\", pred(k_known, Xt)[0])\n", - "suite.less(\"hetero: NLPD(known variances) <= NLPD(nugget)\", res_known[\"nlpd\"] - res_nug[\"nlpd\"], 0.05,\n", + "suite.check_finite(\"hetero: predictions finite\", pred(k_known, Xt)[0])\n", + "suite.check_less(\"hetero: NLPD(known variances) <= NLPD(nugget)\", res_known[\"nlpd\"] - res_nug[\"nlpd\"], 0.05,\n", " severity=\"warn\")\n", - "suite.close(\"noise(): returns the given variances\", np.ravel(k_known.noise()), sd_x ** 2, rtol=1e-12)\n", + "suite.check_close(\"noise(): returns the given variances\", np.ravel(k_known.noise()), sd_x ** 2, rtol=1e-12)\n", "\n", "u = np.linspace(0, 1, 400)[:, None]\n", "fig, ax = plt.subplots(figsize=(10, 4))\n", @@ -250,9 +250,9 @@ " \"sigma2\": float(k_rep.sigma2()), \"is_sigma2_estim\": False}\n", "k_rep = lk.Kriging(yr, Xr, \"matern5_2\", noise=np.full(len(yr), sd ** 2), optim=\"none\", parameters=fixed)\n", "k_avg = lk.Kriging(ybar, X1, \"matern5_2\", noise=np.full(len(ybar), sd ** 2 / r), optim=\"none\", parameters=fixed)\n", - "suite.close(\"replicates == averages with variance/r (mean)\", pred(k_rep, Xt)[0], pred(k_avg, Xt)[0],\n", + "suite.check_close(\"replicates == averages with variance/r (mean)\", pred(k_rep, Xt)[0], pred(k_avg, Xt)[0],\n", " rtol=1e-6, atol=1e-6 * np.std(yr))\n", - "suite.close(\"replicates == averages with variance/r (stdev)\", pred(k_rep, Xt)[1], pred(k_avg, Xt)[1],\n", + "suite.check_close(\"replicates == averages with variance/r (stdev)\", pred(k_rep, Xt)[1], pred(k_avg, Xt)[1],\n", " rtol=1e-4, atol=1e-6 * np.std(yr))" ] }, diff --git a/objectives-LL-LOO-LMP-gradients_Kriging_python.ipynb b/objectives-LL-LOO-LMP-gradients_Kriging_python.ipynb index 5705554..74a140c 100644 --- a/objectives-LL-LOO-LMP-gradients_Kriging_python.ipynb +++ b/objectives-LL-LOO-LMP-gradients_Kriging_python.ipynb @@ -47,7 +47,7 @@ "source": [ "# Setup for running this notebook outside the repository (Colab, Binder, a lone copy):\n", "# nothing to do in the repository itself (CI, local runs). Otherwise the repository\n", - "# (datasets of data/, helpers of tools/) is cloned and the released pylibkriging installed.\n", + "# (datasets of data/, helpers of tools/python/gallery.py) is cloned and the released pylibkriging installed.\n", "import importlib\n", "import os\n", "import subprocess\n", @@ -69,7 +69,7 @@ " \"(PyPI has wheels for CPython 3.10 to 3.12 only). On Colab: Runtime > Change runtime version > \"\n", " \"2026.07 (Python 3.12), then run again.\\n\" + r.stderr[-500:])\n", " importlib.invalidate_caches()\n", - "sys.path.insert(0, str(Path(\"tools\").resolve())) # lkgallery, when not installed (Colab, Binder)\n", + "sys.path.insert(0, str(Path(\"tools/python\").resolve())) # for \"import gallery\" (tools/python/gallery.py)\n", "print(\"working directory:\", os.getcwd())" ] }, @@ -85,10 +85,10 @@ "import pandas as pd\n", "import pylibkriging as lk\n", "\n", - "import lkgallery\n", - "from lkgallery import TESTS, design, metrics, compat\n", + "import gallery\n", + "from gallery import TESTS, design, metrics, compat\n", "\n", - "suite = lkgallery.Suite(\"objectives-LL-LOO-LMP-gradients_Kriging_python\")\n", + "suite = gallery.Suite(\"objectives-LL-LOO-LMP-gradients_Kriging_python\")\n", "print(\"pylibkriging\", compat.version())\n", "CASES = {\"branin\": 25, \"hartmann3\": 40, \"ishigami\": 60, \"friedman\": 50}\n", "KERNELS = [\"gauss\", \"exp\", \"matern3_2\", \"matern5_2\"]\n", @@ -136,7 +136,7 @@ " err = np.max(np.abs(grad - fd)) / max(np.max(np.abs(fd)), 1e-8)\n", " worst = max(worst, err)\n", " rows.append((fname, kernel, objective, worst))\n", - " suite.less(f\"{fname}/{kernel}/{objective}: gradient vs finite diff (rel. err)\", worst, 1e-3)\n", + " suite.check_less(f\"{fname}/{kernel}/{objective}: gradient vs finite diff (rel. err)\", worst, 1e-3)\n", "pd.DataFrame(rows, columns=[\"function\", \"kernel\", \"objective\", \"max rel. error\"]).pivot_table(\n", " index=[\"function\", \"kernel\"], columns=\"objective\", values=\"max rel. error\").style.format(\"{:.1e}\")" ] @@ -174,7 +174,7 @@ " tol = 1e-4 * abs(best) + (0.0 if objective == \"LOO\" else 1e-2)\n", " if SIGN[objective] * f(t, False)[0] > best + tol and not at_bound:\n", " improved.append(f\"theta{j + 1}*{fac}\")\n", - " suite.true(f\"{fname}/{objective}: theta_hat not improved by +/-10% moves\", not improved,\n", + " suite.check_true(f\"{fname}/{objective}: theta_hat not improved by +/-10% moves\", not improved,\n", " detail=f\"theta={np.round(theta, 4)}; improving moves: {improved}\")\n", " m, s, *_ = k.predict(Xt, return_stdev=True)\n", " suite.metric(f\"{fname}/{objective}\", \"pylibkriging\", **metrics.all_metrics(yt, m, s))" @@ -223,7 +223,7 @@ " ax.semilogx(ts, vals); ax.axvline(1, color=\"k\", ls=\"--\", lw=.8)\n", " ax.set_title(f\"{objective}, θ̂ = {np.round(theta, 3)}\"); ax.set_xlabel(\"t (θ = t · θ̂)\")\n", " i = int(np.argmax(vals) if SIGN[objective] > 0 else np.argmin(vals))\n", - " suite.between(f\"branin/{objective}: optimum of the profile along t * theta_hat (t)\", ts[i], 0.8, 1.25,\n", + " suite.check_between(f\"branin/{objective}: optimum of the profile along t * theta_hat (t)\", ts[i], 0.8, 1.25,\n", " severity=\"warn\")\n", "axes[0].set_ylabel(\"criterion\"); plt.tight_layout(); plt.show()" ] @@ -248,7 +248,7 @@ "for objective, getter in [(\"LL\", \"logLikelihood\"), (\"LOO\", \"leaveOneOut\"), (\"LMP\", \"logMargPost\")]:\n", " k = lk.Kriging(y, X, \"matern5_2\", objective=objective)\n", " v_fun = objective_fun(k, objective)(np.asarray(k.theta()).ravel(), False)[0]\n", - " suite.close(f\"{getter}() == {objective} function at theta()\", getattr(k, getter)(), v_fun, rtol=1e-8)" + " suite.check_close(f\"{getter}() == {objective} function at theta()\", getattr(k, getter)(), v_fun, rtol=1e-8)" ] }, { @@ -286,14 +286,14 @@ " \"sigma2\": sigma2, \"is_sigma2_estim\": False})\n", " mi, si, *_ = ki.predict(X[i:i + 1], return_stdev=True)\n", " m_ref[i], s_ref[i] = np.ravel(mi)[0], np.ravel(si)[0]\n", - " suite.close(f\"{fname}/{trend}: leaveOneOutVec mean == explicit refits\", m_loo, m_ref,\n", + " suite.check_close(f\"{fname}/{trend}: leaveOneOutVec mean == explicit refits\", m_loo, m_ref,\n", " rtol=1e-6, atol=1e-8 * np.std(y))\n", - " suite.close(f\"{fname}/{trend}: leaveOneOutVec stdev == explicit refits\", s_loo, s_ref,\n", + " suite.check_close(f\"{fname}/{trend}: leaveOneOutVec stdev == explicit refits\", s_loo, s_ref,\n", " rtol=1e-4, atol=1e-8 * np.std(y), severity=\"warn\")\n", " # leaveOneOut() is the mean squared LOO error at the fitted theta\n", " k_loo = lk.Kriging(y, X, \"matern5_2\", regmodel=trend, objective=\"LOO\")\n", " m2, _ = k_loo.leaveOneOutVec(np.asarray(k_loo.theta()).ravel())\n", - " suite.close(f\"{fname}/{trend}: leaveOneOut() == mean squared LOO error\", k_loo.leaveOneOut(),\n", + " suite.check_close(f\"{fname}/{trend}: leaveOneOut() == mean squared LOO error\", k_loo.leaveOneOut(),\n", " np.mean((np.ravel(m2) - y) ** 2), rtol=1e-6, severity=\"warn\")" ] }, @@ -324,8 +324,8 @@ " suite.metric(fname, \"BFGS\", ll=ll1)\n", " suite.metric(fname, \"BFGS10\", ll=ll10)\n", " suite.metric(fname, \"BFGS20\", ll=ll20)\n", - " suite.greater(f\"{fname}: LL(BFGS10) - LL(BFGS)\", ll10 - ll1, -1e-3 * abs(ll1), severity=\"warn\")\n", - " suite.less(f\"{fname}: LL(BFGS20) - LL(BFGS) (default fit far from the best optimum)\", ll20 - ll1, 1.0,\n", + " suite.check_greater(f\"{fname}: LL(BFGS10) - LL(BFGS)\", ll10 - ll1, -1e-3 * abs(ll1), severity=\"warn\")\n", + " suite.check_less(f\"{fname}: LL(BFGS20) - LL(BFGS) (default fit far from the best optimum)\", ll20 - ll1, 1.0,\n", " severity=\"warn\")\n", "suite.table().loc[list(CASES)].unstack(\"package\")" ] diff --git a/robustness-invalid-inputs_Kriging_python.ipynb b/robustness-invalid-inputs_Kriging_python.ipynb index da28053..214d97c 100644 --- a/robustness-invalid-inputs_Kriging_python.ipynb +++ b/robustness-invalid-inputs_Kriging_python.ipynb @@ -41,7 +41,7 @@ "source": [ "# Setup for running this notebook outside the repository (Colab, Binder, a lone copy):\n", "# nothing to do in the repository itself (CI, local runs). Otherwise the repository\n", - "# (datasets of data/, helpers of tools/) is cloned and the released pylibkriging installed.\n", + "# (datasets of data/, helpers of tools/python/gallery.py) is cloned and the released pylibkriging installed.\n", "import importlib\n", "import os\n", "import subprocess\n", @@ -63,7 +63,7 @@ " \"(PyPI has wheels for CPython 3.10 to 3.12 only). On Colab: Runtime > Change runtime version > \"\n", " \"2026.07 (Python 3.12), then run again.\\n\" + r.stderr[-500:])\n", " importlib.invalidate_caches()\n", - "sys.path.insert(0, str(Path(\"tools\").resolve())) # lkgallery, when not installed (Colab, Binder)\n", + "sys.path.insert(0, str(Path(\"tools/python\").resolve())) # for \"import gallery\" (tools/python/gallery.py)\n", "print(\"working directory:\", os.getcwd())" ] }, @@ -80,10 +80,10 @@ "import numpy as np\n", "import pylibkriging as lk\n", "\n", - "import lkgallery\n", - "from lkgallery import TESTS, design, compat\n", + "import gallery\n", + "from gallery import TESTS, design, compat\n", "\n", - "suite = lkgallery.Suite(\"robustness-invalid-inputs_Kriging_python\")\n", + "suite = gallery.Suite(\"robustness-invalid-inputs_Kriging_python\")\n", "print(\"pylibkriging\", compat.version())\n", "tf = TESTS[\"branin\"]\n", "X, y, Xt, yt = design.dataset(tf, 20, n_test=200, seed=71)\n", @@ -99,10 +99,10 @@ " try:\n", " k = make()\n", " except Exception as exc: # noqa: BLE001\n", - " suite.true(f\"{check}: rejected or finite\", True, detail=f\"rejected: {type(exc).__name__}\")\n", + " suite.check_true(f\"{check}: rejected or finite\", True, detail=f\"rejected: {type(exc).__name__}\")\n", " return None\n", " m, s = pred(k, Xn)\n", - " suite.true(f\"{check}: rejected or finite\", np.all(np.isfinite(m)) and np.all(np.isfinite(s)),\n", + " suite.check_true(f\"{check}: rejected or finite\", np.all(np.isfinite(m)) and np.all(np.isfinite(s)),\n", " detail=f\"non-finite: {np.sum(~np.isfinite(m))} mean, {np.sum(~np.isfinite(s))} stdev\",\n", " severity=severity)\n", " return k" @@ -126,22 +126,22 @@ "Xn = X.copy(); Xn[3, 1] = np.nan\n", "yn = y.copy(); yn[5] = np.nan\n", "yi = y.copy(); yi[5] = np.inf\n", - "suite.raises(\"NaN in X is rejected\", lambda: lk.Kriging(y, Xn, \"matern5_2\"))\n", - "suite.raises(\"NaN in y is rejected\", lambda: lk.Kriging(yn, X, \"matern5_2\"))\n", - "suite.raises(\"Inf in y is rejected\", lambda: lk.Kriging(yi, X, \"matern5_2\"))\n", - "suite.raises(\"len(y) != nrow(X) is rejected\", lambda: lk.Kriging(y[:-1], X, \"matern5_2\"))\n", - "suite.raises(\"unknown kernel is rejected\", lambda: lk.Kriging(y, X, \"matern7_2\"))\n", - "suite.raises(\"unknown trend is rejected\", lambda: lk.Kriging(y, X, \"matern5_2\", regmodel=\"cubic\"))\n", - "suite.raises(\"unknown objective is rejected\", lambda: lk.Kriging(y, X, \"matern5_2\", objective=\"XYZ\"))\n", - "suite.raises(\"noise vector of the wrong length is rejected\",\n", + "suite.check_raises(\"NaN in X is rejected\", lambda: lk.Kriging(y, Xn, \"matern5_2\"))\n", + "suite.check_raises(\"NaN in y is rejected\", lambda: lk.Kriging(yn, X, \"matern5_2\"))\n", + "suite.check_raises(\"Inf in y is rejected\", lambda: lk.Kriging(yi, X, \"matern5_2\"))\n", + "suite.check_raises(\"len(y) != nrow(X) is rejected\", lambda: lk.Kriging(y[:-1], X, \"matern5_2\"))\n", + "suite.check_raises(\"unknown kernel is rejected\", lambda: lk.Kriging(y, X, \"matern7_2\"))\n", + "suite.check_raises(\"unknown trend is rejected\", lambda: lk.Kriging(y, X, \"matern5_2\", regmodel=\"cubic\"))\n", + "suite.check_raises(\"unknown objective is rejected\", lambda: lk.Kriging(y, X, \"matern5_2\", objective=\"XYZ\"))\n", + "suite.check_raises(\"noise vector of the wrong length is rejected\",\n", " lambda: lk.Kriging(y, X, \"matern5_2\", noise=np.ones(len(y) - 1)))\n", - "suite.raises(\"negative noise variance is rejected\",\n", + "suite.check_raises(\"negative noise variance is rejected\",\n", " lambda: lk.Kriging(y, X, \"matern5_2\", noise=-np.ones(len(y))), severity=\"warn\")\n", "k = lk.Kriging(y, X, \"matern5_2\")\n", - "suite.raises(\"predict with the wrong number of columns is rejected\", lambda: k.predict(np.ones((3, 3)), True))\n", + "suite.check_raises(\"predict with the wrong number of columns is rejected\", lambda: k.predict(np.ones((3, 3)), True))\n", "if compat.has(\"WarpKriging\"):\n", - " suite.raises(\"WarpKriging with 1 spec for 2 columns is rejected\", lambda: lk.WarpKriging(y, X, [\"none\"], \"gauss\"))\n", - " suite.raises(\"WarpKriging with an unknown spec is rejected\",\n", + " suite.check_raises(\"WarpKriging with 1 spec for 2 columns is rejected\", lambda: lk.WarpKriging(y, X, [\"none\"], \"gauss\"))\n", + " suite.check_raises(\"WarpKriging with an unknown spec is rejected\",\n", " lambda: lk.WarpKriging(y, X, [\"none\", \"spline\"], \"gauss\"))" ] }, @@ -164,10 +164,10 @@ "finite_model(\"duplicated points, noise-free\", lambda: lk.Kriging(yd, Xd, \"matern5_2\"))\n", "if compat.has(\"noise\"):\n", " k = finite_model(\"duplicated points, nugget\", lambda: lk.Kriging(yd, Xd, \"matern5_2\", noise=\"nugget\"))\n", - " suite.true(\"duplicated points, nugget: a model is built\", k is not None)\n", + " suite.check_true(\"duplicated points, nugget: a model is built\", k is not None)\n", "kc = finite_model(\"constant y\", lambda: lk.Kriging(np.full(20, 3.7), X, \"matern5_2\"))\n", "if kc is not None:\n", - " suite.close(\"constant y: predicts the constant\", pred(kc, Xt)[0], np.full(len(Xt), 3.7), rtol=1e-6, atol=1e-6)\n", + " suite.check_close(\"constant y: predicts the constant\", pred(kc, Xt)[0], np.full(len(Xt), 3.7), rtol=1e-6, atol=1e-6)\n", "for n in (2, 3):\n", " finite_model(f\"n = {n}\", lambda: lk.Kriging(y[:n], X[:n], \"matern5_2\"))\n", "Xclose = np.vstack([X[:10], X[:10] + 1e-8])\n", @@ -197,21 +197,21 @@ "k0 = lk.Kriging(y, X, \"matern5_2\", normalize=True)\n", "m0, s0 = pred(k0, Xt)\n", "for scale in (1e-6, 1e6):\n", - " with suite.runs(f\"X * {scale:g} with normalize=True: fit\"):\n", + " with suite.check_runs(f\"X * {scale:g} with normalize=True: fit\"):\n", " ks = lk.Kriging(y, X * scale, \"matern5_2\", normalize=True)\n", - " suite.close(f\"X * {scale:g}, normalize=True: same mean as on [0,1]\", pred(ks, Xt * scale)[0], m0,\n", + " suite.check_close(f\"X * {scale:g}, normalize=True: same mean as on [0,1]\", pred(ks, Xt * scale)[0], m0,\n", " rtol=1e-3, atol=1e-3 * np.std(y))\n", "for a, b in [(1e8, 0.0), (1.0, 1e6), (1e-8, 0.0)]:\n", - " with suite.runs(f\"y * {a:g} + {b:g}: fit\"):\n", + " with suite.check_runs(f\"y * {a:g} + {b:g}: fit\"):\n", " ka = lk.Kriging(a * y + b, X, \"matern5_2\", normalize=True)\n", " ma, sa = pred(ka, Xt)\n", - " suite.close(f\"y * {a:g} + {b:g}: mean equivariant\", ma, a * m0 + b, rtol=1e-3, atol=1e-3 * a * np.std(y))\n", - " suite.close(f\"y * {a:g} + {b:g}: stdev scales by |a|\", sa, abs(a) * s0, rtol=1e-2,\n", + " suite.check_close(f\"y * {a:g} + {b:g}: mean equivariant\", ma, a * m0 + b, rtol=1e-3, atol=1e-3 * a * np.std(y))\n", + " suite.check_close(f\"y * {a:g} + {b:g}: stdev scales by |a|\", sa, abs(a) * s0, rtol=1e-2,\n", " atol=1e-3 * a * np.std(y))\n", "# the same without normalize: the optimizer starts from a scale-dependent point, warn-level only\n", "for a, b in [(1e6, 0.0), (1.0, 1e6)]:\n", " ka = lk.Kriging(a * y + b, X, \"matern5_2\")\n", - " suite.close(f\"y * {a:g} + {b:g} (normalize=False): mean equivariant\", pred(ka, Xt)[0],\n", + " suite.check_close(f\"y * {a:g} + {b:g} (normalize=False): mean equivariant\", pred(ka, Xt)[0],\n", " a * pred(lk.Kriging(y, X, \"matern5_2\"), Xt)[0] + b, rtol=1e-3, atol=1e-3 * a * np.std(y),\n", " severity=\"warn\")" ] @@ -234,11 +234,11 @@ "k = lk.Kriging(y, X, \"matern5_2\")\n", "far = np.array([[50.0, 50.0], [-30.0, 80.0]])\n", "m, s = pred(k, far)\n", - "suite.close(\"far away: mean -> beta (constant trend)\", m, np.full(2, np.ravel(k.beta())[0]), rtol=1e-6,\n", + "suite.check_close(\"far away: mean -> beta (constant trend)\", m, np.full(2, np.ravel(k.beta())[0]), rtol=1e-6,\n", " atol=1e-8)\n", "# universal kriging: sigma2 * (1 + 1 / (1' R^-1 1)) with the trend uncertainty, slightly above sigma2\n", - "suite.between(\"far away: stdev / sqrt(sigma2)\", np.max(s) / np.sqrt(k.sigma2()), 1.0 - 1e-6, 1.5)\n", - "suite.close(\"far away: same stdev at both points\", s[0], s[1], rtol=1e-6)" + "suite.check_between(\"far away: stdev / sqrt(sigma2)\", np.max(s) / np.sqrt(k.sigma2()), 1.0 - 1e-6, 1.5)\n", + "suite.check_close(\"far away: same stdev at both points\", s[0], s[1], rtol=1e-6)" ] }, { @@ -273,7 +273,7 @@ " times[q] = best\n", " suite.metric(f\"predict_q={q}\", \"pylibkriging\", seconds=best)\n", "expo = np.log2(times[8000] / times[4000])\n", - "suite.less(\"predict(stdev) time growth exponent in q (8000 vs 4000)\", expo, 1.5,\n", + "suite.check_less(\"predict(stdev) time growth exponent in q (8000 vs 4000)\", expo, 1.5,\n", " detail=\", \".join(f\"q={q}: {t:.3f}s\" for q, t in times.items()))\n", "qs = np.array(list(times)); ts = np.array(list(times.values()))\n", "plt.figure(figsize=(5, 3.6))\n", diff --git a/save-load-pickle-copy-accessors_Kriging_python.ipynb b/save-load-pickle-copy-accessors_Kriging_python.ipynb index f3b2d89..e1f51b3 100644 --- a/save-load-pickle-copy-accessors_Kriging_python.ipynb +++ b/save-load-pickle-copy-accessors_Kriging_python.ipynb @@ -41,7 +41,7 @@ "source": [ "# Setup for running this notebook outside the repository (Colab, Binder, a lone copy):\n", "# nothing to do in the repository itself (CI, local runs). Otherwise the repository\n", - "# (datasets of data/, helpers of tools/) is cloned and the released pylibkriging installed.\n", + "# (datasets of data/, helpers of tools/python/gallery.py) is cloned and the released pylibkriging installed.\n", "import importlib\n", "import os\n", "import subprocess\n", @@ -63,7 +63,7 @@ " \"(PyPI has wheels for CPython 3.10 to 3.12 only). On Colab: Runtime > Change runtime version > \"\n", " \"2026.07 (Python 3.12), then run again.\\n\" + r.stderr[-500:])\n", " importlib.invalidate_caches()\n", - "sys.path.insert(0, str(Path(\"tools\").resolve())) # lkgallery, when not installed (Colab, Binder)\n", + "sys.path.insert(0, str(Path(\"tools/python\").resolve())) # for \"import gallery\" (tools/python/gallery.py)\n", "print(\"working directory:\", os.getcwd())" ] }, @@ -79,10 +79,10 @@ "import numpy as np\n", "import pylibkriging as lk\n", "\n", - "import lkgallery\n", - "from lkgallery import TESTS, design, compat\n", + "import gallery\n", + "from gallery import TESTS, design, compat\n", "\n", - "suite = lkgallery.Suite(\"save-load-pickle-copy-accessors_Kriging_python\")\n", + "suite = gallery.Suite(\"save-load-pickle-copy-accessors_Kriging_python\")\n", "print(\"pylibkriging\", compat.version())\n", "\n", "tf = TESTS[\"hartmann3\"]\n", @@ -130,8 +130,8 @@ "outputs": [], "source": [ "for name, make in MODELS.items():\n", - " with suite.runs(f\"{name}: fit\"):\n", - " lkgallery.persistence.check(suite, name, make(), Xt, predict_any)" + " with suite.check_runs(f\"{name}: fit\"):\n", + " gallery.persistence.check(suite, name, make(), Xt, predict_any)" ] }, { @@ -153,8 +153,8 @@ "m_before = mean_sd(k, Xt)[0]\n", "kc = k.copy() if hasattr(k, \"copy\") else copy.deepcopy(k)\n", "kc.update(y[30:], X[30:], refit=False)\n", - "suite.close(\"update on a copy leaves the original unchanged\", mean_sd(k, Xt)[0], m_before, rtol=0, atol=0)\n", - "suite.true(\"the updated copy did change\", not np.allclose(mean_sd(kc, Xt)[0], m_before))" + "suite.check_close(\"update on a copy leaves the original unchanged\", mean_sd(k, Xt)[0], m_before, rtol=0, atol=0)\n", + "suite.check_true(\"the updated copy did change\", not np.allclose(mean_sd(kc, Xt)[0], m_before))" ] }, { @@ -175,19 +175,19 @@ "k = lk.Kriging(y, X, \"matern3_2\", regmodel=\"linear\", normalize=False, optim=\"BFGS\", objective=\"LMP\")\n", "for acc, expected in [(\"kernel\", \"matern3_2\"), (\"regmodel\", \"linear\"), (\"optim\", \"BFGS\"), (\"objective\", \"LMP\"),\n", " (\"normalize\", False)]:\n", - " suite.true(f\"Kriging.{acc}() == {expected!r}\", getattr(k, acc)() == expected, detail=str(getattr(k, acc)()))\n", - "suite.close(\"Kriging.X() is the design\", np.asarray(k.X()), X, rtol=0, atol=0)\n", - "suite.close(\"Kriging.y() is the response\", np.ravel(k.y()), y, rtol=0, atol=0)\n", + " suite.check_true(f\"Kriging.{acc}() == {expected!r}\", getattr(k, acc)() == expected, detail=str(getattr(k, acc)()))\n", + "suite.check_close(\"Kriging.X() is the design\", np.asarray(k.X()), X, rtol=0, atol=0)\n", + "suite.check_close(\"Kriging.y() is the response\", np.ravel(k.y()), y, rtol=0, atol=0)\n", "# with normalize=True: do the accessors return the data as given, or as rescaled internally?\n", "kn = lk.Kriging(y, X * 10, \"matern3_2\", normalize=True)\n", - "suite.true(\"Kriging.normalize() == True\", kn.normalize() is True or kn.normalize() == 1)\n", - "suite.close(\"normalize=True: X() is the design as given\", np.asarray(kn.X()), X * 10, rtol=1e-12, atol=0,\n", + "suite.check_true(\"Kriging.normalize() == True\", kn.normalize() is True or kn.normalize() == 1)\n", + "suite.check_close(\"normalize=True: X() is the design as given\", np.asarray(kn.X()), X * 10, rtol=1e-12, atol=0,\n", " severity=\"warn\")\n", - "suite.close(\"normalize=True: y() is the response as given\", np.ravel(kn.y()), y, rtol=1e-12, atol=0,\n", + "suite.check_close(\"normalize=True: y() is the response as given\", np.ravel(kn.y()), y, rtol=1e-12, atol=0,\n", " severity=\"warn\")\n", - "suite.true(\"Kriging.beta() has 1 + d coefficients (linear trend)\", np.ravel(k.beta()).size == 1 + X.shape[1])\n", - "suite.true(\"summary() is a non-empty string\", isinstance(k.summary(), str) and len(k.summary()) > 20)\n", - "suite.true(\"is_theta_estim() after an optimized fit\", bool(k.is_theta_estim()))" + "suite.check_true(\"Kriging.beta() has 1 + d coefficients (linear trend)\", np.ravel(k.beta()).size == 1 + X.shape[1])\n", + "suite.check_true(\"summary() is a non-empty string\", isinstance(k.summary(), str) and len(k.summary()) > 20)\n", + "suite.check_true(\"is_theta_estim() after an optimized fit\", bool(k.is_theta_estim()))" ] }, { diff --git a/sklearn-estimators_Kriging_python.ipynb b/sklearn-estimators_Kriging_python.ipynb index e9c055d..0c2752e 100644 --- a/sklearn-estimators_Kriging_python.ipynb +++ b/sklearn-estimators_Kriging_python.ipynb @@ -28,7 +28,7 @@ "source": [ "# Setup for running this notebook outside the repository (Colab, Binder, a lone copy):\n", "# nothing to do in the repository itself (CI, local runs). Otherwise the repository\n", - "# (datasets of data/, helpers of tools/) is cloned and the released pylibkriging installed.\n", + "# (datasets of data/, helpers of tools/python/gallery.py) is cloned and the released pylibkriging installed.\n", "import importlib\n", "import os\n", "import subprocess\n", @@ -50,7 +50,7 @@ " \"(PyPI has wheels for CPython 3.10 to 3.12 only). On Colab: Runtime > Change runtime version > \"\n", " \"2026.07 (Python 3.12), then run again.\\n\" + r.stderr[-500:])\n", " importlib.invalidate_caches()\n", - "sys.path.insert(0, str(Path(\"tools\").resolve())) # lkgallery, when not installed (Colab, Binder)\n", + "sys.path.insert(0, str(Path(\"tools/python\").resolve())) # for \"import gallery\" (tools/python/gallery.py)\n", "print(\"working directory:\", os.getcwd())" ] }, @@ -66,10 +66,10 @@ "import pandas as pd\n", "import pylibkriging as lk\n", "\n", - "import lkgallery\n", - "from lkgallery import TESTS, design, compat\n", + "import gallery\n", + "from gallery import TESTS, design, compat\n", "\n", - "suite = lkgallery.Suite(\"sklearn-estimators_Kriging_python\")\n", + "suite = gallery.Suite(\"sklearn-estimators_Kriging_python\")\n", "print(\"pylibkriging\", compat.version())\n", "\n", "tf = TESTS[\"hartmann3\"]\n", @@ -138,22 +138,22 @@ "\n", " est = KrigingRegressor(kernel=\"matern5_2\").fit(X, y)\n", " m_sk, s_sk = est.predict(Xt, return_std=True)\n", - " suite.true(\"KrigingRegressor.predict returns 1-D arrays (scikit-learn convention)\",\n", + " suite.check_true(\"KrigingRegressor.predict returns 1-D arrays (scikit-learn convention)\",\n", " np.ndim(m_sk) == 1 and np.ndim(s_sk) == 1, detail=f\"shapes {np.shape(m_sk)}, {np.shape(s_sk)}\",\n", " severity=\"warn\")\n", " m_sk, s_sk = np.ravel(m_sk), np.ravel(s_sk)\n", " m_nat, s_nat = mean_sd(lk.Kriging(y, X, \"matern5_2\"), Xt)\n", - " suite.close(\"KrigingRegressor == Kriging (mean)\", m_sk, m_nat, rtol=1e-8, atol=1e-10)\n", - " suite.close(\"KrigingRegressor == Kriging (stdev)\", s_sk, s_nat, rtol=1e-6, atol=1e-10)\n", + " suite.check_close(\"KrigingRegressor == Kriging (mean)\", m_sk, m_nat, rtol=1e-8, atol=1e-10)\n", + " suite.check_close(\"KrigingRegressor == Kriging (stdev)\", s_sk, s_nat, rtol=1e-6, atol=1e-10)\n", " est2 = clone(est).set_params(kernel=\"gauss\")\n", - " suite.true(\"clone + set_params\", est2.get_params()[\"kernel\"] == \"gauss\" and est.get_params()[\"kernel\"] ==\n", + " suite.check_true(\"clone + set_params\", est2.get_params()[\"kernel\"] == \"gauss\" and est.get_params()[\"kernel\"] ==\n", " \"matern5_2\")\n", - " with suite.runs(\"Pipeline(MinMaxScaler, KrigingRegressor)\"):\n", + " with suite.check_runs(\"Pipeline(MinMaxScaler, KrigingRegressor)\"):\n", " pipe = make_pipeline(MinMaxScaler(), KrigingRegressor(kernel=\"matern5_2\")).fit(tf.scale(X), y)\n", - " suite.greater(\"pipeline Q2 on native inputs\", pipe.score(tf.scale(Xt), yt), 0.8)\n", - " with suite.runs(\"GridSearchCV over kernels\"):\n", + " suite.check_greater(\"pipeline Q2 on native inputs\", pipe.score(tf.scale(Xt), yt), 0.8)\n", + " with suite.check_runs(\"GridSearchCV over kernels\"):\n", " gs = GridSearchCV(KrigingRegressor(), {\"kernel\": [\"gauss\", \"matern3_2\", \"matern5_2\"]}, cv=3).fit(X, y)\n", - " suite.true(\"GridSearchCV picked a kernel\", gs.best_params_[\"kernel\"] in (\"gauss\", \"matern3_2\", \"matern5_2\"),\n", + " suite.check_true(\"GridSearchCV picked a kernel\", gs.best_params_[\"kernel\"] in (\"gauss\", \"matern3_2\", \"matern5_2\"),\n", " detail=str(gs.best_params_))\n", " cv = pd.DataFrame(gs.cv_results_)[[\"param_kernel\", \"mean_test_score\", \"std_test_score\", \"mean_fit_time\"]]\n", " for _, r in cv.iterrows():\n", @@ -163,14 +163,14 @@ " plt.bar(cv[\"param_kernel\"].astype(str), cv[\"mean_test_score\"], yerr=cv[\"std_test_score\"], capsize=4)\n", " plt.ylabel(\"3-fold CV R²\"); plt.title(f\"GridSearchCV(KrigingRegressor): best = {gs.best_params_['kernel']}\")\n", " plt.ylim(min(0, cv[\"mean_test_score\"].min() - .05), 1.02); plt.show()\n", - " with suite.runs(\"cross_val_score\"):\n", + " with suite.check_runs(\"cross_val_score\"):\n", " sc = cross_val_score(KrigingRegressor(kernel=\"matern5_2\"), X, y, cv=4)\n", - " suite.finite(\"cross_val_score finite\", sc)\n", - " with suite.runs(\"sample_y\"):\n", + " suite.check_finite(\"cross_val_score finite\", sc)\n", + " with suite.check_runs(\"sample_y\"):\n", " S = est.sample_y(Xt[:10], n_samples=5, random_state=1)\n", - " suite.true(\"sample_y shape (q, n_samples)\", np.shape(S) == (10, 5), detail=str(np.shape(S)))\n", + " suite.check_true(\"sample_y shape (q, n_samples)\", np.shape(S) == (10, 5), detail=str(np.shape(S)))\n", "else:\n", - " suite.skip(\"sklearn estimators\", \"pylibkriging.sklearn or scikit-learn not available\")" + " suite.check_skip(\"sklearn estimators\", \"pylibkriging.sklearn or scikit-learn not available\")" ] }, { diff --git a/test-functions-reference-values_no-model_python.ipynb b/test-functions-reference-values_no-model_python.ipynb index 3d5d3d2..99997be 100644 --- a/test-functions-reference-values_no-model_python.ipynb +++ b/test-functions-reference-values_no-model_python.ipynb @@ -15,7 +15,7 @@ "Experiments, Surjanovic & Bingham).\n", "\n", "Every notebook of this gallery relies on the test functions of\n", - "`lkgallery.functions` (after the [Virtual Library of Simulation Experiments](https://www.sfu.ca/~ssurjano/)).\n", + "`gallery.functions` (after the [Virtual Library of Simulation Experiments](https://www.sfu.ca/~ssurjano/)).\n", "A typo in one of them would silently bias every result built on it, so they are\n", "checked first, against the reference values published on the library pages\n", "(global optima, nominal points) and against the known output ranges." @@ -30,7 +30,7 @@ "source": [ "# Setup for running this notebook outside the repository (Colab, Binder, a lone copy):\n", "# nothing to do in the repository itself (CI, local runs). Otherwise the repository\n", - "# (datasets of data/, helpers of tools/) is cloned and the released pylibkriging installed.\n", + "# (datasets of data/, helpers of tools/python/gallery.py) is cloned and the released pylibkriging installed.\n", "import importlib\n", "import os\n", "import subprocess\n", @@ -52,7 +52,7 @@ " \"(PyPI has wheels for CPython 3.10 to 3.12 only). On Colab: Runtime > Change runtime version > \"\n", " \"2026.07 (Python 3.12), then run again.\\n\" + r.stderr[-500:])\n", " importlib.invalidate_caches()\n", - "sys.path.insert(0, str(Path(\"tools\").resolve())) # lkgallery, when not installed (Colab, Binder)\n", + "sys.path.insert(0, str(Path(\"tools/python\").resolve())) # for \"import gallery\" (tools/python/gallery.py)\n", "print(\"working directory:\", os.getcwd())" ] }, @@ -67,10 +67,10 @@ "import pandas as pd\n", "import matplotlib.pyplot as plt\n", "\n", - "import lkgallery\n", - "from lkgallery import TESTS, design\n", + "import gallery\n", + "from gallery import TESTS, design\n", "\n", - "suite = lkgallery.Suite(\"test-functions-reference-values_no-model_python\")" + "suite = gallery.Suite(\"test-functions-reference-values_no-model_python\")" ] }, { @@ -92,7 +92,7 @@ "for name, tf in TESTS.items():\n", " for x, v in tf.known:\n", " fx = float(tf(x)[0])\n", - " suite.close(f\"{name}: f({np.round(x, 4).tolist()})\", fx, v, rtol=1e-4, atol=1e-4)\n", + " suite.check_close(f\"{name}: f({np.round(x, 4).tolist()})\", fx, v, rtol=1e-4, atol=1e-4)\n", " rows.append((name, tf.d, np.round(x, 4).tolist(), v, fx))\n", "pd.DataFrame(rows, columns=[\"function\", \"d\", \"x\", \"reference\", \"computed\"])" ] @@ -128,10 +128,10 @@ "for name, tf in TESTS.items():\n", " U = design.uniform(20_000, tf.d, seed=1)\n", " y = tf.unit(U)\n", - " suite.finite(f\"{name}: finite on the domain\", y)\n", + " suite.check_finite(f\"{name}: finite on the domain\", y)\n", " if name in EXPECTED_RANGES:\n", " lo, hi = EXPECTED_RANGES[name]\n", - " suite.true(f\"{name}: range within [{lo}, {hi}]\", lo <= y.min() and y.max() <= hi,\n", + " suite.check_true(f\"{name}: range within [{lo}, {hi}]\", lo <= y.min() and y.max() <= hi,\n", " detail=f\"sampled range [{y.min():.4g}, {y.max():.4g}]\")\n", " summary.append((name, tf.d, \",\".join(tf.tags), y.min(), y.max(), y.std()))\n", "pd.DataFrame(summary, columns=[\"function\", \"d\", \"tags\", \"min\", \"max\", \"std\"])" @@ -157,9 +157,9 @@ "outputs": [], "source": [ "A, B = design.lhs(20, 3, seed=7), design.lhs(20, 3, seed=7)\n", - "suite.close(\"lhs: reproducible with a seed\", A, B, rtol=0, atol=0)\n", + "suite.check_close(\"lhs: reproducible with a seed\", A, B, rtol=0, atol=0)\n", "strata = np.floor(A * 20).astype(int)\n", - "suite.true(\"lhs: one point per stratum\", all(len(set(strata[:, j])) == 20 for j in range(3)))" + "suite.check_true(\"lhs: one point per stratum\", all(len(set(strata[:, j])) == 20 for j in range(3)))" ] }, { diff --git a/tools/R/gallery.R b/tools/R/gallery.R new file mode 100644 index 0000000..1f429cd --- /dev/null +++ b/tools/R/gallery.R @@ -0,0 +1,138 @@ +# Helpers of the libKriging gallery notebooks (R), in one file: source("tools/R/gallery.R"). +# +# The same functions, under the same names, as tools/python/gallery.py (where they are +# methods of Suite: suite.check_close(...) is check_close(s, ...) here), and the same +# CSV outputs (results/.checks.csv, .metrics.csv), aggregated with the other +# languages by tools/report.py. +# +# suite(name) a suite of checks, named after the notebook +# check(s, check, ok, value, expected, ...) record a check (severity "error" or "warn") +# check_true, check_less, check_greater, check_between, check_close, check_finite, +# check_runs, check_skip the usual checks +# metric(s, case, package, name = value, ...) record metrics +# suite_summary(s) end-of-notebook summary +# suite_finalize(s) write the CSV files, stop if an error-level check failed +# read_data(name, stem), rmse, q2, nlpd, coverage +# +# Statuses: PASS, FAIL (error-level), WARN (warn-level), SKIP (feature absent from the +# installed version), KNOWN (failing, listed in tools/known_issues.csv with its issue). +# Paths are relative to the repository root ($LKGALLERY_ROOT, else the working directory, +# where the notebooks are executed). + +.gallery_root <- Sys.getenv("LKGALLERY_ROOT", ".") +.known_file <- file.path(.gallery_root, "tools", "known_issues.csv") +if (!file.exists(.known_file)) stop("run the notebooks from the repository root (", .known_file, " not found)") +.known <- read.csv(.known_file, stringsAsFactors = FALSE) +results_dir <- Sys.getenv("LKGALLERY_RESULTS", file.path(.gallery_root, "results")) +dir.create(results_dir, showWarnings = FALSE, recursive = TRUE) +.STATUSES <- c("PASS", "FAIL", "WARN", "KNOWN", "SKIP") + +# ---- suite and checks ----------------------------------------------------------------- + +suite <- function(name) { + s <- new.env() + s$name <- name + s$checks <- list() + s$metrics <- list() + s$t0 <- Sys.time() + s +} + +.fmt <- function(v) if (is.null(v) || length(v) == 0) "" else if (is.numeric(v)) format(signif(v[1], 6)) else as.character(v[1]) + +.record <- function(s, check, severity, status, value, expected, detail) { + s$checks[[length(s$checks) + 1]] <- data.frame(suite = s$name, lang = "R", check = check, severity = severity, + status = status, value = .fmt(value), expected = .fmt(expected), detail = detail, stringsAsFactors = FALSE) +} + +check <- function(s, check, ok, value = NULL, expected = NULL, detail = "", severity = "error") { + ok <- isTRUE(ok) + status <- if (ok) "PASS" else if (severity == "error") "FAIL" else "WARN" + k <- .known[.known$suite == s$name & .known$check == check, ] + if (!ok && nrow(k) > 0) { status <- "KNOWN"; detail <- paste(k$issue[1], detail, sep = " | ") } + if (!ok) message(sprintf("[%s] %s: value=%s expected %s %s", status, check, .fmt(value), .fmt(expected), detail)) + .record(s, check, severity, status, value, expected, detail) + invisible(ok) +} + +check_true <- function(s, check, cond, detail = "", severity = "error") + check(s, check, isTRUE(cond), cond, TRUE, detail, severity) + +check_less <- function(s, check, value, bound, detail = "", severity = "error") + check(s, check, is.finite(value) && value < bound, value, paste("<", bound), detail, severity) + +check_greater <- function(s, check, value, bound, detail = "", severity = "error") + check(s, check, is.finite(value) && value > bound, value, paste(">", bound), detail, severity) + +check_between <- function(s, check, value, lo, hi, detail = "", severity = "error") + check(s, check, is.finite(value) && value >= lo && value <= hi, value, sprintf("in [%g, %g]", lo, hi), detail, severity) + +check_close <- function(s, check, a, b, rtol = 1e-6, atol = 0, detail = "", severity = "error") { + a <- as.numeric(a); b <- as.numeric(b) + if (length(a) != length(b)) + return(check(s, check, FALSE, paste("length", length(a)), paste("length", length(b)), detail, severity)) + gap <- abs(a - b) + ok <- all(is.finite(a)) && all(gap <= atol + rtol * abs(b)) + rel <- max(gap / pmax(abs(b), 1e-300)) + check(s, check, ok, rel, sprintf("rtol %g, atol %g", rtol, atol), detail, severity) +} + +check_finite <- function(s, check, a, detail = "", severity = "error") + check(s, check, length(a) > 0 && all(is.finite(a)), paste(sum(!is.finite(a)), "non-finite"), "all finite", detail, severity) + +check_runs <- function(s, check, expr, severity = "error") { + # check_runs(s, "fit", { ... }): an error becomes a failed check instead of stopping the notebook + r <- tryCatch({ force(expr); NULL }, error = function(e) conditionMessage(e)) + check(s, check, is.null(r), if (is.null(r)) "ok" else "error", "no error", if (is.null(r)) "" else substr(r, 1, 200), severity) +} + +check_skip <- function(s, check, reason) { + .record(s, check, "info", "SKIP", "", "", reason) + message("[SKIP] ", check, ": ", reason) +} + +# ---- metrics and results -------------------------------------------------------------- + +metric <- function(s, case, package, ...) { + v <- list(...) + for (n in names(v)) s$metrics[[length(s$metrics) + 1]] <- data.frame(suite = s$name, lang = "R", case = case, + package = package, metric = n, value = .fmt(v[[n]]), stringsAsFactors = FALSE) +} + +suite_summary <- function(s) { + # end-of-notebook summary: metrics, count per status, every check that did not pass + if (length(s$metrics) > 0) { + me <- do.call(rbind, s$metrics) + me$value <- as.numeric(me$value) + print(reshape(me[, c("case", "package", "metric", "value")], idvar = c("case", "package"), + timevar = "metric", direction = "wide"), row.names = FALSE) + } + ch <- do.call(rbind, s$checks) + print(table(factor(ch$status, levels = .STATUSES))) + other <- ch[ch$status != "PASS", c("status", "severity", "check", "value", "expected", "detail")] + if (nrow(other) > 0) print(other, row.names = FALSE) + invisible(ch) +} + +suite_finalize <- function(s) { + metric(s, "_suite", "_", elapsed_s = as.numeric(difftime(Sys.time(), s$t0, units = "secs"))) + ch <- do.call(rbind, s$checks) + me <- do.call(rbind, s$metrics) + write.csv(ch, file.path(results_dir, paste0(s$name, ".checks.csv")), row.names = FALSE) + write.csv(me, file.path(results_dir, paste0(s$name, ".metrics.csv")), row.names = FALSE) + print(table(factor(ch$status, levels = .STATUSES))) + failed <- ch$check[ch$status == "FAIL"] + if (length(failed) > 0) + stop(sprintf("%d check(s) failed in %s: %s", length(failed), s$name, paste(head(failed, 20), collapse = "; "))) + invisible(ch) +} + +# ---- data and metrics ----------------------------------------------------------------- + +read_data <- function(name, stem) # data/_.csv (no header) as a matrix + as.matrix(read.csv(file.path(.gallery_root, "data", paste0(name, "_", stem, ".csv")), header = FALSE)) + +rmse <- function(y, m) sqrt(mean((y - m)^2)) +q2 <- function(y, m) 1 - sum((y - m)^2) / sum((y - mean(y))^2) +nlpd <- function(y, m, s) { s <- pmax(s, 1e-12); mean(0.5 * log(2 * pi * s^2) + 0.5 * ((y - m) / s)^2) } +coverage <- function(y, m, s, level = 0.95) mean(abs(y - m) <= qnorm(0.5 + level / 2) * s) diff --git a/tools/install/python.sh b/tools/install/python.sh index 1547e46..0c04e2e 100755 --- a/tools/install/python.sh +++ b/tools/install/python.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash -# Install pylibkriging, then the notebooks' helpers (tools/lkgallery) and runner. +# Install pylibkriging, then what the notebooks need besides it (tools/python/requirements.txt; +# their helpers, tools/python/gallery.py, are loaded from the repository, not installed). # tools/install/python.sh wheel LIBKRIGING_SRC OUT_DIR only build a wheel from a libKriging checkout # tools/install/python.sh master WHEEL_DIR|LIBKRIGING_SRC install the wheel of WHEEL_DIR, or build it first # tools/install/python.sh release the last release, from PyPI @@ -23,5 +24,5 @@ case "$1" in release) python -m pip install pylibkriging ;; *) echo "usage: $0 wheel SRC OUT | master WHEEL_DIR|SRC | release" >&2; exit 2 ;; esac -python -m pip install -e "${tools}[notebooks]" +python -m pip install -r "$tools/python/requirements.txt" python -c "import pylibkriging as lk; print('pylibkriging', lk.__version__, 'from', lk.__file__)" diff --git a/tools/julia/checks.jl b/tools/julia/checks.jl deleted file mode 100644 index 01b50b3..0000000 --- a/tools/julia/checks.jl +++ /dev/null @@ -1,84 +0,0 @@ -# Same CSV outputs as lkgallery/checks.py (see there), for the Julia notebooks. -# Paths are relative to the repository root, where the notebooks are executed. -using DelimitedFiles, Printf - -const RESULTS_DIR = get(ENV, "LKGALLERY_RESULTS", "results") -mkpath(RESULTS_DIR) - -function read_known() - f = joinpath("tools", "known_issues.csv") - isfile(f) || error("run the notebooks from the repository root ($f not found)") - rows = readdlm(f, ',', String; header=true, quotes=true)[1] # quoted fields may hold commas - Dict{Tuple{String,String},String}((rows[i, 1], rows[i, 2]) => rows[i, 3] for i in 1:size(rows, 1)) -end - -mutable struct Suite - name::String - checks::Vector{Vector{String}} - metrics::Vector{Vector{String}} - t0::Float64 - known::Dict{Tuple{String,String},String} -end -Suite(name) = Suite(name, [], [], time(), read_known()) - -fmt(v::Real) = @sprintf("%.6g", v) -fmt(v) = string(v) - -function add_check!(s::Suite, check, ok; value="", expected="", detail="", severity="error") - status = ok ? "PASS" : (severity == "error" ? "FAIL" : "WARN") - if !ok && haskey(s.known, (s.name, check)) - status = "KNOWN"; detail = s.known[(s.name, check)] * " | " * detail - end - ok || println("[$status] $check: value=$(fmt(value)) expected $expected $detail") - push!(s.checks, [s.name, "julia", check, severity, status, fmt(value), expected, detail]) - ok -end - -function check_close!(s::Suite, check, a, b; rtol=1e-6, atol=0.0, detail="", severity="error") - a = vec(Float64.(collect(a))); b = vec(Float64.(collect(b))) - length(a) == length(b) || return add_check!(s, check, false; value="length $(length(a))", - expected="length $(length(b))", detail, severity) - gap = abs.(a .- b) - ok = all(isfinite, a) && all(gap .<= atol .+ rtol .* abs.(b)) - rel = maximum(gap ./ max.(abs.(b), 1e-300)) - add_check!(s, check, ok; value=rel, expected="rtol $rtol, atol $atol", detail, severity) -end - -add_metric!(s::Suite, case, package; kw...) = - for (k, v) in kw; push!(s.metrics, [s.name, "julia", case, package, string(k), fmt(v)]); end - -csvq(x) = "\"" * replace(string(x), "\"" => "\"\"") * "\"" - -function write_rows(f, header, rows) - open(f, "w") do io - println(io, join(header, ",")) - for r in rows; println(io, join(csvq.(r), ",")); end - end -end - -function summary!(s::Suite) - # end-of-notebook summary: metrics, count per status, every check that did not pass - for r in s.metrics - println(rpad(r[3], 28), rpad(r[4], 28), rpad(r[5], 12), r[6]) - end - st = [c[5] for c in s.checks] - println(length(st), " checks: ", join(["$k $(count(==(k), st))" for k in ("PASS", "FAIL", "WARN", "KNOWN", "SKIP")], ", ")) - for c in s.checks - c[5] == "PASS" || println(rpad(c[5], 7), rpad(c[4], 7), c[3], ": value ", c[6], ", expected ", c[7], " ", c[8]) - end -end - -function finalize!(s::Suite) - add_metric!(s, "_suite", "_"; elapsed_s=time() - s.t0) - write_rows(joinpath(RESULTS_DIR, s.name * ".checks.csv"), - ["suite", "lang", "check", "severity", "status", "value", "expected", "detail"], s.checks) - write_rows(joinpath(RESULTS_DIR, s.name * ".metrics.csv"), - ["suite", "lang", "case", "package", "metric", "value"], s.metrics) - st = [c[5] for c in s.checks] - println(s.name, ": ", join(["$k $(count(==(k), st))" for k in ("PASS", "FAIL", "WARN", "KNOWN", "SKIP")], ", ")) - failed = [c[3] for c in s.checks if c[5] == "FAIL"] - isempty(failed) || error("$(length(failed)) check(s) failed in $(s.name): " * join(first(failed, 20), "; ")) -end - -read_data(name, stem) = readdlm(joinpath("data", "$(name)_$(stem).csv"), ',', Float64) -q2(y, m) = 1 - sum((y .- m) .^ 2) / sum((y .- sum(y) / length(y)) .^ 2) diff --git a/tools/julia/gallery.jl b/tools/julia/gallery.jl new file mode 100644 index 0000000..ecd66df --- /dev/null +++ b/tools/julia/gallery.jl @@ -0,0 +1,136 @@ +# Helpers of the libKriging gallery notebooks (Julia), in one file: include("tools/julia/gallery.jl"). +# +# The same functions, under the same names, as tools/python/gallery.py (where they are +# methods of Suite: suite.check_close(...) is check_close!(s, ...) here), and the same CSV +# outputs (results/.checks.csv, .metrics.csv), aggregated with the other +# languages by tools/report.py. +# +# s = Suite(name) a suite of checks, named after the notebook +# check!(s, check, ok; value, expected, detail, severity) +# check_true!, check_less!, check_greater!, check_between!, check_close!, check_finite!, +# check_runs!(f, s, check) (do-block), check_skip! the usual checks +# metric!(s, case, package; name=value, ...) record metrics +# suite_summary(s) end-of-notebook summary +# suite_finalize!(s) write the CSV files, error if an error-level check failed +# read_data(name, stem), rmse, q2 +# +# Paths are relative to the repository root ($LKGALLERY_ROOT, else the working +# directory, where the notebooks are executed). +using DelimitedFiles, Printf + +const GALLERY_ROOT = get(ENV, "LKGALLERY_ROOT", ".") +const RESULTS_DIR = get(ENV, "LKGALLERY_RESULTS", joinpath(GALLERY_ROOT, "results")) +mkpath(RESULTS_DIR) + +# ---- suite and checks ------------------------------------------------------------------ + +function read_known() + f = joinpath(GALLERY_ROOT, "tools", "known_issues.csv") + isfile(f) || error("run the notebooks from the repository root ($f not found)") + rows = readdlm(f, ',', String; header=true, quotes=true)[1] # quoted fields may hold commas + Dict{Tuple{String,String},String}((rows[i, 1], rows[i, 2]) => rows[i, 3] for i in 1:size(rows, 1)) +end + +mutable struct Suite + name::String + checks::Vector{Vector{String}} + metrics::Vector{Vector{String}} + t0::Float64 + known::Dict{Tuple{String,String},String} +end +Suite(name) = Suite(name, [], [], time(), read_known()) + +fmt(v::Real) = @sprintf("%.6g", v) +fmt(v) = string(v) + +function check!(s::Suite, check, ok; value="", expected="", detail="", severity="error") + status = ok ? "PASS" : (severity == "error" ? "FAIL" : "WARN") + if !ok && haskey(s.known, (s.name, check)) + status = "KNOWN"; detail = s.known[(s.name, check)] * " | " * detail + end + ok || println("[$status] $check: value=$(fmt(value)) expected $expected $detail") + push!(s.checks, [s.name, "julia", check, severity, status, fmt(value), expected, detail]) + ok +end + +check_true!(s::Suite, check, cond; detail="", severity="error") = + check!(s, check, cond === true; value=string(cond), expected="true", detail, severity) +check_less!(s::Suite, check, value, bound; detail="", severity="error") = + check!(s, check, isfinite(value) && value < bound; value, expected="< $bound", detail, severity) +check_greater!(s::Suite, check, value, bound; detail="", severity="error") = + check!(s, check, isfinite(value) && value > bound; value, expected="> $bound", detail, severity) +check_between!(s::Suite, check, value, lo, hi; detail="", severity="error") = + check!(s, check, isfinite(value) && lo <= value <= hi; value, expected="in [$lo, $hi]", detail, severity) + +function check_close!(s::Suite, check, a, b; rtol=1e-6, atol=0.0, detail="", severity="error") + a = vec(Float64.(collect(a))); b = vec(Float64.(collect(b))) + length(a) == length(b) || return check!(s, check, false; value="length $(length(a))", + expected="length $(length(b))", detail, severity) + gap = abs.(a .- b) + ok = all(isfinite, a) && all(gap .<= atol .+ rtol .* abs.(b)) + rel = maximum(gap ./ max.(abs.(b), 1e-300)) + check!(s, check, ok; value=rel, expected="rtol $rtol, atol $atol", detail, severity) +end + +check_finite!(s::Suite, check, a; detail="", severity="error") = + check!(s, check, !isempty(a) && all(isfinite, a); value="$(count(!isfinite, a)) non-finite", + expected="all finite", detail, severity) + +function check_runs!(f, s::Suite, check; severity="error") + # check_runs!(s, "fit") do ... end: an error becomes a failed check instead of stopping the notebook + try + f() + check!(s, check, true; value="ok", expected="no error", severity) + catch e + check!(s, check, false; value=string(typeof(e)), expected="no error", + detail=first(sprint(showerror, e), 200), severity) + end +end + +function check_skip!(s::Suite, check, reason) + push!(s.checks, [s.name, "julia", check, "info", "SKIP", "", "", reason]) + println("[SKIP] $check: $reason") +end + +# ---- metrics and results --------------------------------------------------------------- + +metric!(s::Suite, case, package; kw...) = + for (k, v) in kw; push!(s.metrics, [s.name, "julia", case, package, string(k), fmt(v)]); end + +function suite_summary(s::Suite) + for r in s.metrics + println(rpad(r[3], 28), rpad(r[4], 28), rpad(r[5], 12), r[6]) + end + st = [c[5] for c in s.checks] + println(length(st), " checks: ", join(["$k $(count(==(k), st))" for k in ("PASS", "FAIL", "WARN", "KNOWN", "SKIP")], ", ")) + for c in s.checks + c[5] == "PASS" || println(rpad(c[5], 7), rpad(c[4], 7), c[3], ": value ", c[6], ", expected ", c[7], " ", c[8]) + end +end + +csvq(x) = "\"" * replace(string(x), "\"" => "\"\"") * "\"" + +function write_rows(f, header, rows) + open(f, "w") do io + println(io, join(header, ",")) + for r in rows; println(io, join(csvq.(r), ",")); end + end +end + +function suite_finalize!(s::Suite) + metric!(s, "_suite", "_"; elapsed_s=time() - s.t0) + write_rows(joinpath(RESULTS_DIR, s.name * ".checks.csv"), + ["suite", "lang", "check", "severity", "status", "value", "expected", "detail"], s.checks) + write_rows(joinpath(RESULTS_DIR, s.name * ".metrics.csv"), + ["suite", "lang", "case", "package", "metric", "value"], s.metrics) + st = [c[5] for c in s.checks] + println(s.name, ": ", join(["$k $(count(==(k), st))" for k in ("PASS", "FAIL", "WARN", "KNOWN", "SKIP")], ", ")) + failed = [c[3] for c in s.checks if c[5] == "FAIL"] + isempty(failed) || error("$(length(failed)) check(s) failed in $(s.name): " * join(first(failed, 20), "; ")) +end + +# ---- data and metrics ------------------------------------------------------------------ + +read_data(name, stem) = readdlm(joinpath(GALLERY_ROOT, "data", "$(name)_$(stem).csv"), ',', Float64) +rmse(y, m) = sqrt(sum((y .- m) .^ 2) / length(y)) +q2(y, m) = 1 - sum((y .- m) .^ 2) / sum((y .- sum(y) / length(y)) .^ 2) diff --git a/tools/lkgallery/__init__.py b/tools/lkgallery/__init__.py deleted file mode 100644 index 9e4eb50..0000000 --- a/tools/lkgallery/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -"""Helpers shared by the notebooks of the libKriging gallery (test functions, designs, metrics, checks).""" -from . import checks, compat, design, functions, metrics, persistence # noqa: F401 -from .checks import Suite # noqa: F401 -from .functions import TESTS # noqa: F401 diff --git a/tools/lkgallery/checks.py b/tools/lkgallery/checks.py deleted file mode 100644 index d8aafd6..0000000 --- a/tools/lkgallery/checks.py +++ /dev/null @@ -1,199 +0,0 @@ -"""Record checks and metrics from a notebook, and fail it at the end if needed. - -A notebook creates one ``Suite``, records any number of checks (assertions that -do *not* stop the notebook, so every problem of a run is reported at once) and -metrics, then calls ``suite.finalize()`` in its last cell: it writes - - results/.checks.csv suite,lang,check,severity,status,value,expected,detail - results/.metrics.csv suite,lang,case,package,metric,value - -and raises ``AssertionError`` if any check of severity "error" failed, which makes -the notebook (hence the CI job) fail. Checks of severity "warn" are reported but -never fail the run: they flag performance drifts or comparisons with other -packages, where a threshold is a judgement call rather than a correctness bound. - -Statuses: PASS, FAIL (error-level), WARN (warn-level), SKIP (feature missing in -the installed version), KNOWN (a failing check, of either severity, listed in -``tools/known_issues.csv``: reported with its issue, but does not fail the run). A listed check that passes is -reported as FIXED by ``tools/report.py``, so that the list does not rot. - -The R, Octave and Julia notebooks write the same two CSV formats with a few lines -of their own code (tools/r/checks.R, tools/octave/lkb_*.m, tools/julia/checks.jl), so that ``tools/report.py`` can -aggregate every language in one report. -""" -import csv -import math -import os -import time -import traceback -from contextlib import contextmanager -from pathlib import Path - -import numpy as np - - -def _find_root(): - """The repository root: $LKGALLERY_ROOT, or the first directory, from the working - directory up, that holds tools/known_issues.csv (notebooks run from the root). - Not derived from this file's location, which is site-packages for a non-editable - install: the known issues would then be silently ignored.""" - if os.environ.get("LKGALLERY_ROOT"): - return Path(os.environ["LKGALLERY_ROOT"]).resolve() - for d in [Path.cwd(), *Path.cwd().parents]: - if (d / "tools" / "known_issues.csv").is_file(): - return d - raise RuntimeError("libKriging gallery repository not found from the working directory " - f"{Path.cwd()}: run the notebooks from the repository root, or set LKGALLERY_ROOT") - - -ROOT = _find_root() -RESULTS_DIR = Path(os.environ.get("LKGALLERY_RESULTS", ROOT / "results")) -KNOWN_ISSUES_FILE = ROOT / "tools" / "known_issues.csv" -DATA_DIR = ROOT / "data" - - -def known_issues(): - """{(suite, check): issue} from tools/known_issues.csv.""" - with open(KNOWN_ISSUES_FILE, newline="") as fh: - return {(r["suite"], r["check"]): r["issue"] for r in csv.DictReader(fh)} - - -CHECK_FIELDS = ["suite", "lang", "check", "severity", "status", "value", "expected", "detail"] -METRIC_FIELDS = ["suite", "lang", "case", "package", "metric", "value"] - - -def _fmt(v): - if v is None: - return "" - if isinstance(v, (bool, np.bool_)): - return str(bool(v)) - if isinstance(v, (int, float, np.integer, np.floating)): - return f"{float(v):.6g}" - return str(v) - - -class Suite: - def __init__(self, name, lang="python"): - self.name, self.lang = name, lang - self.checks, self.metrics = [], [] - self.t0 = time.time() - self.known = known_issues() - - # -- checks --------------------------------------------------------------- - - def _add(self, check, ok, value=None, expected=None, detail="", severity="error"): - status = "PASS" if ok else ("FAIL" if severity == "error" else "WARN") - if not ok and (self.name, check) in self.known: - status, detail = "KNOWN", f"{self.known[self.name, check]} | {detail}".rstrip(" |") - self.checks.append(dict(suite=self.name, lang=self.lang, check=check, severity=severity, - status=status, value=_fmt(value), expected=_fmt(expected), detail=detail)) - if not ok: - print(f"[{status}] {check}: value={_fmt(value)} expected {_fmt(expected)} {detail}") - return ok - - def true(self, check, cond, detail="", severity="error"): - return self._add(check, bool(cond), cond, True, detail, severity) - - def less(self, check, value, bound, detail="", severity="error"): - """value < bound (NaN fails).""" - value = float(value) - return self._add(check, math.isfinite(value) and value < bound, value, f"< {bound:g}", detail, severity) - - def greater(self, check, value, bound, detail="", severity="error"): - value = float(value) - return self._add(check, math.isfinite(value) and value > bound, value, f"> {bound:g}", detail, severity) - - def between(self, check, value, lo, hi, detail="", severity="error"): - value = float(value) - return self._add(check, math.isfinite(value) and lo <= value <= hi, value, f"in [{lo:g}, {hi:g}]", - detail, severity) - - def close(self, check, a, b, rtol=1e-6, atol=0.0, detail="", severity="error"): - """All |a - b| <= atol + rtol |b| (element-wise); reports the worst relative gap.""" - a, b = np.asarray(a, float), np.asarray(b, float) - if a.shape != b.shape: - return self._add(check, False, f"shape {a.shape}", f"shape {b.shape}", detail, severity) - gap = np.abs(a - b) - ok = bool(np.all(np.isfinite(a)) and np.all(gap <= atol + rtol * np.abs(b))) - rel = float(np.max(gap / np.maximum(np.abs(b), 1e-300))) if a.size else 0.0 - return self._add(check, ok, rel, f"rtol {rtol:g}, atol {atol:g}", detail, severity) - - def finite(self, check, a, detail="", severity="error"): - a = np.asarray(a, float) - return self._add(check, bool(a.size and np.all(np.isfinite(a))), f"{np.sum(~np.isfinite(a))} non-finite", - "all finite", detail, severity) - - def skip(self, check, reason): - self.checks.append(dict(suite=self.name, lang=self.lang, check=check, severity="info", - status="SKIP", value="", expected="", detail=reason)) - print(f"[SKIP] {check}: {reason}") - - @contextmanager - def runs(self, check, severity="error"): - """``with suite.runs("x"): ...`` records a FAIL (with the exception) instead of stopping.""" - try: - yield - except Exception as exc: # noqa: BLE001 - every failure is a finding here - self._add(check, False, type(exc).__name__, "no exception", - str(exc).splitlines()[0][:200] if str(exc) else traceback.format_exc(limit=1), severity) - else: - self._add(check, True, "ok", "no exception", "", severity) - - def raises(self, check, fn, *args, severity="error", **kwargs): - """``fn(*args)`` must raise (invalid input must be rejected, not silently accepted).""" - try: - fn(*args, **kwargs) - except Exception as exc: # noqa: BLE001 - return self._add(check, True, type(exc).__name__, "an exception", "", severity) - return self._add(check, False, "no exception", "an exception", "", severity) - - # -- metrics -------------------------------------------------------------- - - def metric(self, case, package, **values): - for k, v in values.items(): - self.metrics.append(dict(suite=self.name, lang=self.lang, case=case, package=package, - metric=k, value=_fmt(v))) - - def table(self): - """Metrics as a wide pandas DataFrame (one row per case x package).""" - import pandas as pd - if not self.metrics: - return pd.DataFrame() - df = pd.DataFrame(self.metrics) - df["value"] = pd.to_numeric(df["value"], errors="coerce") - return df.pivot_table(index=["case", "package"], columns="metric", values="value", sort=False) - - def summary(self): - """End-of-notebook summary: the metrics table, the count per status, and every check - that did not pass (with its value, threshold and, for KNOWN, its issue).""" - import pandas as pd - try: - from IPython.display import display - except ImportError: # outside Jupyter - display = print - if self.metrics: - display(self.table().round(4)) - df = pd.DataFrame(self.checks, columns=CHECK_FIELDS) - counts = {s: int((df["status"] == s).sum()) for s in ("PASS", "FAIL", "WARN", "KNOWN", "SKIP")} - print(f"{len(df)} checks: " + ", ".join(f"{k} {v}" for k, v in counts.items())) - others = df[df["status"] != "PASS"][["status", "severity", "check", "value", "expected", "detail"]] - if len(others): - display(others.reset_index(drop=True)) - - # -- output --------------------------------------------------------------- - - def finalize(self): - RESULTS_DIR.mkdir(parents=True, exist_ok=True) - self.metric("_suite", "_", elapsed_s=time.time() - self.t0) - for rows, fields, ext in ((self.checks, CHECK_FIELDS, "checks"), (self.metrics, METRIC_FIELDS, "metrics")): - with open(RESULTS_DIR / f"{self.name}.{ext}.csv", "w", newline="") as fh: - w = csv.DictWriter(fh, fieldnames=fields) - w.writeheader() - w.writerows(rows) - counts = {s: sum(c["status"] == s for c in self.checks) for s in ("PASS", "FAIL", "WARN", "KNOWN", "SKIP")} - print(f"{self.name}: " + ", ".join(f"{k} {v}" for k, v in counts.items())) - failed = [c for c in self.checks if c["status"] == "FAIL"] - if failed: - raise AssertionError(f"{len(failed)} check(s) failed in {self.name}: " - + "; ".join(c["check"] for c in failed[:20])) - return counts diff --git a/tools/lkgallery/compat.py b/tools/lkgallery/compat.py deleted file mode 100644 index 1fae0f4..0000000 --- a/tools/lkgallery/compat.py +++ /dev/null @@ -1,61 +0,0 @@ -"""What the installed pylibkriging offers, so that one notebook runs against both -the last release and the development head: a feature missing from the installed -version is recorded as SKIP (with the reason), never as a failure.""" -import functools -import importlib - -import numpy as np - - -def version(): - import pylibkriging as lk - v = getattr(lk, "__version__", None) - if v is None: - try: - from importlib.metadata import version as _v - v = _v("pylibkriging") - except Exception: # noqa: BLE001 - v = "unknown" - return v - - -@functools.lru_cache(None) -def has(feature): - """Probe a feature on a tiny problem (cached).""" - import pylibkriging as lk - X = np.linspace(0, 1, 6).reshape(-1, 1) - y = np.sin(3 * X[:, 0]) - probes = { - "method": lambda: lk.Kriging(y, X, "matern5_2", method="vecchia(3)"), - "noise": lambda: lk.Kriging(y, X, "matern5_2", noise="nugget"), - "WarpKriging": lambda: lk.WarpKriging(y, X, ["none"], "matern5_2"), - "MLPKriging": lambda: lk.MLPKriging(y, X, hidden_dims=[2], d_out=1, kernel="gauss"), - "NestedKriging": lambda: lk.NestedKriging(y, X, "matern5_2", nb_groups=2), - "subsetOfData": lambda: lk.Kriging.subsetOfData(X, n_max=3, method="kmeans", seed=1), - "save": lambda: lk.Kriging(y, X, "matern5_2").save, - "pickle": lambda: __import__("pickle").dumps(lk.Kriging(y, X, "matern5_2")), - "NestedKriging.save": lambda: lk.NestedKriging(y, X, "matern5_2", nb_groups=2).save, - "load(noise)": lambda: _roundtrip(lk, lk.Kriging(y, X, "matern5_2", noise="nugget")), - "sklearn": lambda: importlib.import_module("pylibkriging.sklearn").KrigingRegressor, - } - try: - probes[feature]() - return True - except Exception: # noqa: BLE001 - return False - - -def _roundtrip(lk, k): - import os - import tempfile - f = os.path.join(tempfile.mkdtemp(), "k.json") - k.save(f) - return lk.load(f) - - -def optional(module): - """Import an optional comparison package, or return None.""" - try: - return importlib.import_module(module) - except Exception: # noqa: BLE001 - broken installs count as absent - return None diff --git a/tools/lkgallery/design.py b/tools/lkgallery/design.py deleted file mode 100644 index e4452f3..0000000 --- a/tools/lkgallery/design.py +++ /dev/null @@ -1,31 +0,0 @@ -"""Seeded designs of experiments, all on [0,1]^d (scale with ``TestFunction.scale``).""" -import numpy as np -from scipy.stats import qmc - - -def lhs(n, d, seed=0, optimized=True): - """Latin hypercube; ``optimized`` applies scipy's random-CD improvement (space filling).""" - sampler = qmc.LatinHypercube(d=d, seed=seed, optimization="random-cd" if optimized and d > 1 else None) - return sampler.random(n) - - -def sobol(n, d, seed=0): - return qmc.Sobol(d=d, scramble=True, seed=seed).random(n) - - -def uniform(n, d, seed=0): - return np.random.default_rng(seed).random((n, d)) - - -def dataset(tf, n, n_test=1000, seed=0, noise_sd=0.0): - """Training/test sets for test function ``tf`` on the unit hypercube. - - Returns (X, y, Xt, yt) with X in [0,1]^d (the GP sees the unit cube; the - function is evaluated in its native domain). ``noise_sd`` adds Gaussian - noise to y only (yt stays noise-free, it is the target).""" - X = lhs(n, tf.d, seed=seed) - Xt = uniform(n_test, tf.d, seed=10_000 + seed) - y = tf.unit(X) - if noise_sd > 0: - y = y + np.random.default_rng(20_000 + seed).normal(0.0, noise_sd, size=n) - return X, y, Xt, tf.unit(Xt) diff --git a/tools/lkgallery/functions.py b/tools/lkgallery/functions.py deleted file mode 100644 index 3b606dd..0000000 --- a/tools/lkgallery/functions.py +++ /dev/null @@ -1,241 +0,0 @@ -"""Test functions for surrogate modelling, after the Virtual Library of Simulation -Experiments (Surjanovic & Bingham, https://www.sfu.ca/~ssurjano/). - -Every function takes ``X`` of shape (n, d) in its *native* domain and returns a -length-n vector. ``TESTS`` gathers the metadata (dimension, domain, a reference -point with its known value) used by ``test-functions-reference-values_no-model_python.ipynb`` -to check the implementations themselves before any model is fitted on them. -""" -from dataclasses import dataclass, field - -import numpy as np - - -# --- 1D ------------------------------------------------------------------------- - -def forrester(X): - x = X[:, 0] - return (6 * x - 2) ** 2 * np.sin(12 * x - 4) - - -def gramacy_lee_2012(X): - x = X[:, 0] - return np.sin(10 * np.pi * x) / (2 * x) + (x - 1) ** 4 - - -# --- 2D ------------------------------------------------------------------------- - -def branin(X): - x1, x2 = X[:, 0], X[:, 1] - a, b, c = 1.0, 5.1 / (4 * np.pi ** 2), 5 / np.pi - r, s, t = 6.0, 10.0, 1 / (8 * np.pi) - return a * (x2 - b * x1 ** 2 + c * x1 - r) ** 2 + s * (1 - t) * np.cos(x1) + s - - -def six_hump_camel(X): - x1, x2 = X[:, 0], X[:, 1] - return (4 - 2.1 * x1 ** 2 + x1 ** 4 / 3) * x1 ** 2 + x1 * x2 + (-4 + 4 * x2 ** 2) * x2 ** 2 - - -def goldstein_price(X): - x1, x2 = X[:, 0], X[:, 1] - a = 1 + (x1 + x2 + 1) ** 2 * (19 - 14 * x1 + 3 * x1 ** 2 - 14 * x2 + 6 * x1 * x2 + 3 * x2 ** 2) - b = 30 + (2 * x1 - 3 * x2) ** 2 * (18 - 32 * x1 + 12 * x1 ** 2 + 48 * x2 - 36 * x1 * x2 + 27 * x2 ** 2) - return a * b - - -def log_goldstein_price(X): - """Logarithmic Goldstein-Price (Picheny et al. 2013), on [0,1]^2.""" - Z = 4 * X - 2 - return (np.log(goldstein_price(Z)) - 8.693) / 2.427 - - -def currin_exp(X): - x1, x2 = X[:, 0], X[:, 1] - return (1 - np.exp(-1 / (2 * x2))) * (2300 * x1 ** 3 + 1900 * x1 ** 2 + 2092 * x1 + 60) / ( - 100 * x1 ** 3 + 500 * x1 ** 2 + 4 * x1 + 20) - - -def lim_polynomial(X): - x1, x2 = X[:, 0], X[:, 1] - return (9 + 5 / 2 * x1 - 35 / 2 * x2 + 5 / 2 * x1 * x2 + 19 * x2 ** 2 - 15 / 2 * x1 ** 3 - - 5 / 2 * x1 * x2 ** 2 - 11 / 2 * x2 ** 4 + x1 ** 3 * x2 ** 2) - - -def gramacy_lee_2008(X): - """Non-stationary 2D function x1 exp(-x1^2 - x2^2) on [-2, 6]^2.""" - x1, x2 = X[:, 0], X[:, 1] - return x1 * np.exp(-x1 ** 2 - x2 ** 2) - - -# --- 3D to 5D ------------------------------------------------------------------- - -def ishigami(X, a=7.0, b=0.1): - x1, x2, x3 = X[:, 0], X[:, 1], X[:, 2] - return np.sin(x1) + a * np.sin(x2) ** 2 + b * x3 ** 4 * np.sin(x1) - - -_H3_ALPHA = np.array([1.0, 1.2, 3.0, 3.2]) -_H3_A = np.array([[3, 10, 30], [0.1, 10, 35], [3, 10, 30], [0.1, 10, 35]], float) -_H3_P = 1e-4 * np.array([[3689, 1170, 2673], [4699, 4387, 7470], - [1091, 8732, 5547], [381, 5743, 8828]], float) - - -def hartmann3(X): - diff2 = (X[:, None, :] - _H3_P[None, :, :]) ** 2 - return -np.exp(-np.sum(_H3_A[None, :, :] * diff2, axis=2)) @ _H3_ALPHA - - -def friedman(X): - x1, x2, x3, x4, x5 = (X[:, i] for i in range(5)) - return 10 * np.sin(np.pi * x1 * x2) + 20 * (x3 - 0.5) ** 2 + 10 * x4 + 5 * x5 - - -# --- 6D to 8D ------------------------------------------------------------------- - -_H6_ALPHA = np.array([1.0, 1.2, 3.0, 3.2]) -_H6_A = np.array([[10, 3, 17, 3.5, 1.7, 8], - [0.05, 10, 17, 0.1, 8, 14], - [3, 3.5, 1.7, 10, 17, 8], - [17, 8, 0.05, 10, 0.1, 14]], float) -_H6_P = 1e-4 * np.array([[1312, 1696, 5569, 124, 8283, 5886], - [2329, 4135, 8307, 3736, 1004, 9991], - [2348, 1451, 3522, 2883, 3047, 6650], - [4047, 8828, 8732, 5743, 1091, 381]], float) - - -def hartmann6(X): - diff2 = (X[:, None, :] - _H6_P[None, :, :]) ** 2 - return -np.exp(-np.sum(_H6_A[None, :, :] * diff2, axis=2)) @ _H6_ALPHA - - -def otl_circuit(X): - Rb1, Rb2, Rf, Rc1, Rc2, beta = (X[:, i] for i in range(6)) - Vb1 = 12 * Rb2 / (Rb1 + Rb2) - den = beta * (Rc2 + 9) + Rf - return ((Vb1 + 0.74) * beta * (Rc2 + 9) / den + 11.35 * Rf / den - + 0.74 * Rf * beta * (Rc2 + 9) / (den * Rc1)) - - -def piston(X): - M, S, V0, k, P0, Ta, T0 = (X[:, i] for i in range(7)) - A = P0 * S + 19.62 * M - k * V0 / S - V = S / (2 * k) * (np.sqrt(A ** 2 + 4 * k * P0 * V0 * Ta / T0) - A) - return 2 * np.pi * np.sqrt(M / (k + S ** 2 * P0 * V0 * Ta / (T0 * V ** 2))) - - -def borehole(X): - rw, r, Tu, Hu, Tl, Hl, L, Kw = (X[:, i] for i in range(8)) - lnr = np.log(r / rw) - return 2 * np.pi * Tu * (Hu - Hl) / (lnr * (1 + 2 * L * Tu / (lnr * rw ** 2 * Kw) + Tu / Tl)) - - -def robot_arm(X): - theta, L = X[:, :4], X[:, 4:] - angles = np.cumsum(theta, axis=1) - u = np.sum(L * np.cos(angles), axis=1) - v = np.sum(L * np.sin(angles), axis=1) - return np.sqrt(u ** 2 + v ** 2) - - -def dette_pepelyshev(X): - x = X - s = 0.0 - for i in range(4, 9): # i = 4..8, inner sum over j = 3..i (1-based) - s = s + i * np.log(1 + np.sum(x[:, 2:i], axis=1)) - return (4 * (x[:, 0] - 2 + 8 * x[:, 1] - 8 * x[:, 1] ** 2) ** 2 + (3 - 4 * x[:, 1]) ** 2 - + 16 * np.sqrt(x[:, 2] + 1) * (2 * x[:, 2] - 1) ** 2 + s) - - -# --- 10D and more --------------------------------------------------------------- - -def wing_weight(X): - Sw, Wfw, A, Lam, q, lam, tc, Nz, Wdg, Wp = (X[:, i] for i in range(10)) - Lam = np.deg2rad(Lam) - return (0.036 * Sw ** 0.758 * Wfw ** 0.0035 * (A / np.cos(Lam) ** 2) ** 0.6 * q ** 0.006 - * lam ** 0.04 * (100 * tc / np.cos(Lam)) ** -0.3 * (Nz * Wdg) ** 0.49 + Sw * Wp) - - -def welch(X): - x = [None] + [X[:, i] for i in range(20)] # 1-based, as in the reference - return (5 * x[12] / (1 + x[1]) + 5 * (x[4] - x[20]) ** 2 + x[5] + 40 * x[19] ** 3 - 5 * x[19] - + 0.05 * x[2] + 0.08 * x[3] - 0.03 * x[6] + 0.03 * x[7] - 0.09 * x[9] - 0.01 * x[10] - - 0.07 * x[11] + 0.25 * x[13] ** 2 - 0.04 * x[14] + 0.06 * x[15] - 0.01 * x[17] - - 0.03 * x[18]) - - -def ackley(X, a=20.0, b=0.2, c=2 * np.pi): - d = X.shape[1] - return (-a * np.exp(-b * np.sqrt(np.sum(X ** 2, axis=1) / d)) - - np.exp(np.sum(np.cos(c * X), axis=1) / d) + a + np.e) - - -@dataclass -class TestFunction: - name: str - f: callable - lower: np.ndarray - upper: np.ndarray - # (x, f(x)) pairs taken from the reference pages (global optima, nominal values) - known: list = field(default_factory=list) - tags: tuple = () - - @property - def d(self): - return len(self.lower) - - def __call__(self, X): - return self.f(np.atleast_2d(X)) - - def scale(self, U): - """Map points of [0,1]^d to the native domain.""" - return self.lower + np.asarray(U) * (self.upper - self.lower) - - def unit(self, U): - """The function seen on [0,1]^d.""" - return self.f(self.scale(np.atleast_2d(U))) - - -def _box(*bounds): - b = np.array(bounds, float) - return b[:, 0], b[:, 1] - - -def _tf(name, f, bounds, known=(), tags=()): - lo, up = _box(*bounds) - return TestFunction(name, f, lo, up, [(np.array(x, float), v) for x, v in known], tuple(tags)) - - -TESTS = {t.name: t for t in [ - _tf("forrester", forrester, [(0, 1)], [([0.757249], -6.02074)], ["1d"]), - _tf("gramacy_lee_2012", gramacy_lee_2012, [(0.5, 2.5)], [([0.548563], -0.869011)], ["1d", "nonstationary"]), - _tf("branin", branin, [(-5, 10), (0, 15)], - [([-np.pi, 12.275], 0.397887), ([np.pi, 2.275], 0.397887), ([9.42478, 2.475], 0.397887)], ["2d"]), - _tf("six_hump_camel", six_hump_camel, [(-3, 3), (-2, 2)], - [([0.0898, -0.7126], -1.0316), ([-0.0898, 0.7126], -1.0316)], ["2d"]), - _tf("goldstein_price", goldstein_price, [(-2, 2), (-2, 2)], [([0, -1], 3.0)], ["2d", "large-range"]), - _tf("log_goldstein_price", log_goldstein_price, [(0, 1), (0, 1)], - [([0.5, 0.25], (np.log(3.0) - 8.693) / 2.427)], ["2d"]), - _tf("currin_exp", currin_exp, [(0, 1), (1e-6, 1)], [], ["2d"]), - _tf("lim_polynomial", lim_polynomial, [(0, 1), (0, 1)], [([0, 0], 9.0)], ["2d", "polynomial"]), - _tf("gramacy_lee_2008", gramacy_lee_2008, [(-2, 6), (-2, 6)], - [([-1 / np.sqrt(2), 0], -np.exp(-0.5) / np.sqrt(2))], ["2d", "nonstationary"]), - _tf("ishigami", ishigami, [(-np.pi, np.pi)] * 3, [([0, 0, 0], 0.0), ([0, np.pi / 2, 0], 7.0)], ["3d"]), - _tf("hartmann3", hartmann3, [(0, 1)] * 3, [([0.114614, 0.555649, 0.852547], -3.86278)], ["3d"]), - _tf("friedman", friedman, [(0, 1)] * 5, [([0, 0, 0.5, 0, 0], 0.0)], ["5d"]), - _tf("hartmann6", hartmann6, [(0, 1)] * 6, - [([0.20169, 0.150011, 0.476874, 0.275332, 0.311652, 0.6573], -3.32237)], ["6d"]), - _tf("otl_circuit", otl_circuit, [(50, 150), (25, 70), (0.5, 3), (1.2, 2.5), (0.25, 1.2), (50, 300)], - [], ["6d", "engineering"]), - _tf("piston", piston, [(30, 60), (0.005, 0.020), (0.002, 0.010), (1000, 5000), (90000, 110000), - (290, 296), (340, 360)], [], ["7d", "engineering"]), - _tf("borehole", borehole, [(0.05, 0.15), (100, 50000), (63070, 115600), (990, 1110), - (63.1, 116), (700, 820), (1120, 1680), (9855, 12045)], [], ["8d", "engineering"]), - _tf("robot_arm", robot_arm, [(0, 2 * np.pi)] * 4 + [(0, 1)] * 4, - [([0, 0, 0, 0, 1, 1, 1, 1], 4.0), ([0, np.pi, 0, 0, 1, 1, 1, 1], 2.0)], ["8d", "oscillating"]), - _tf("dette_pepelyshev", dette_pepelyshev, [(0, 1)] * 8, [], ["8d", "curved"]), - _tf("wing_weight", wing_weight, [(150, 200), (220, 300), (6, 10), (-10, 10), (16, 45), (0.5, 1), - (0.08, 0.18), (2.5, 6), (1700, 2500), (0.025, 0.08)], [], ["10d", "engineering"]), - _tf("welch", welch, [(-0.5, 0.5)] * 20, [([0] * 20, 0.0)], ["20d", "screening"]), - _tf("ackley4", ackley, [(-2, 2)] * 4, [([0, 0, 0, 0], 0.0)], ["4d", "multimodal"]), -]} diff --git a/tools/lkgallery/metrics.py b/tools/lkgallery/metrics.py deleted file mode 100644 index aa5aa62..0000000 --- a/tools/lkgallery/metrics.py +++ /dev/null @@ -1,50 +0,0 @@ -"""Accuracy and calibration metrics for probabilistic predictions.""" -import numpy as np -from scipy.stats import norm - - -def rmse(y, m): - return float(np.sqrt(np.mean((np.asarray(y) - np.asarray(m)) ** 2))) - - -def q2(y, m): - """Predictivity coefficient 1 - SSE/SST on a test set (1 is perfect).""" - y, m = np.asarray(y), np.asarray(m) - return float(1 - np.sum((y - m) ** 2) / np.sum((y - y.mean()) ** 2)) - - -def nlpd(y, m, s): - """Mean negative log predictive density under N(m, s^2).""" - s = np.maximum(np.asarray(s), 1e-12) - return float(np.mean(0.5 * np.log(2 * np.pi * s ** 2) + 0.5 * ((np.asarray(y) - m) / s) ** 2)) - - -def coverage(y, m, s, level=0.95): - """Fraction of test values inside the central ``level`` prediction interval.""" - z = norm.ppf(0.5 + level / 2) - return float(np.mean(np.abs(np.asarray(y) - m) <= z * np.asarray(s))) - - -def crps(y, m, s): - """Mean continuous ranked probability score of N(m, s^2) (lower is better).""" - s = np.maximum(np.asarray(s), 1e-12) - z = (np.asarray(y) - m) / s - return float(np.mean(s * (z * (2 * norm.cdf(z) - 1) + 2 * norm.pdf(z) - 1 / np.sqrt(np.pi)))) - - -def all_metrics(y, m, s=None): - out = {"rmse": rmse(y, m), "q2": q2(y, m)} - if s is not None: - out.update(nlpd=nlpd(y, m, s), coverage95=coverage(y, m, s), crps=crps(y, m, s)) - return out - - -def finite_diff_grad(f, x, h=1e-6): - """Central finite-difference gradient of a scalar function, relative step.""" - x = np.asarray(x, float) - g = np.zeros_like(x) - for i in range(x.size): - e = np.zeros_like(x) - e[i] = h * max(1.0, abs(x[i])) - g[i] = (f(x + e) - f(x - e)) / (2 * e[i]) - return g diff --git a/tools/lkgallery/persistence.py b/tools/lkgallery/persistence.py deleted file mode 100644 index 7c46785..0000000 --- a/tools/lkgallery/persistence.py +++ /dev/null @@ -1,50 +0,0 @@ -"""Persistence checks shared by the notebooks of every model class: a model saved and -loaded back (class-specific and generic ``pylibkriging.load``), pickled, or deep-copied -must predict exactly as the original.""" -import copy -import pickle -import tempfile -from pathlib import Path - -import numpy as np - -from . import compat - - -def check(suite, name, k, Xn, predict): - """Record the persistence checks of model ``k`` (label ``name``) on the points ``Xn``. - - ``predict(model, Xn)`` returns (mean, stdev) as 1-D arrays; it carries the - prediction method of an approximate fit (Vecchia / Nystrom).""" - import pylibkriging as lk - m0, s0 = predict(k, Xn) - cls = type(k) - path = Path(tempfile.mkdtemp()) / f"{name.replace(' ', '_')}.json" - if not hasattr(k, "save"): - suite.skip(f"{name}: save + load", "no save() in this pylibkriging") - else: - with suite.runs(f"{name}: save + load"): - k.save(str(path)) - k1 = cls.load(str(path)) - m1, s1 = predict(k1, Xn) - suite.close(f"{name}: load(save(k)) predicts the same mean", m1, m0, rtol=1e-12, atol=1e-12) - suite.close(f"{name}: load(save(k)) predicts the same stdev", s1, s0, rtol=1e-10, atol=1e-12) - if hasattr(k, "theta"): - suite.close(f"{name}: same theta after load", np.ravel(k1.theta()), np.ravel(k.theta()), - rtol=1e-14, atol=0) - if hasattr(lk, "load") and (compat.has("load(noise)") or "noise" not in name and "nugget" not in name): - with suite.runs(f"{name}: generic pylibkriging.load"): - k2 = lk.load(str(path)) - suite.true(f"{name}: generic load returns {cls.__name__}", type(k2).__name__ == cls.__name__, - detail=type(k2).__name__) - suite.close(f"{name}: generic load predicts the same", predict(k2, Xn)[0], m0, rtol=1e-12, - atol=1e-12) - if not compat.has("pickle"): - suite.skip(f"{name}: pickle / deepcopy", "models not picklable in this pylibkriging") - return - with suite.runs(f"{name}: pickle round trip"): - k3 = pickle.loads(pickle.dumps(k)) - suite.close(f"{name}: pickle predicts the same", predict(k3, Xn)[0], m0, rtol=1e-12, atol=1e-12) - with suite.runs(f"{name}: deepcopy"): - k4 = copy.deepcopy(k) - suite.close(f"{name}: deepcopy predicts the same", predict(k4, Xn)[0], m0, rtol=1e-12, atol=1e-12) diff --git a/tools/make_data.py b/tools/make_data.py index 6dd966b..a346669 100644 --- a/tools/make_data.py +++ b/tools/make_data.py @@ -5,11 +5,13 @@ Plain CSV, no header, full precision; X is n x d (rows = observations), y is a column. Regenerating them is only needed when a dataset is added: the files are the reference, not the script (a new scipy could draw other LHS points).""" +import sys from pathlib import Path import numpy as np -from lkgallery import TESTS, design +sys.path.insert(0, str(Path(__file__).resolve().parent / "python")) +from gallery import TESTS, design # noqa: E402 (tools/python/gallery.py) DATA = Path(__file__).resolve().parents[1] / "data" DATASETS = [("branin", 25), ("hartmann3", 40), ("borehole", 80)] diff --git a/tools/notebook_setup.py b/tools/notebook_setup.py index a71a002..a81e4f1 100644 --- a/tools/notebook_setup.py +++ b/tools/notebook_setup.py @@ -1,6 +1,6 @@ # Setup for running this notebook outside the repository (Colab, Binder, a lone copy): # nothing to do in the repository itself (CI, local runs). Otherwise the repository -# (datasets of data/, helpers of tools/) is cloned and the released pylibkriging installed. +# (datasets of data/, helpers of tools/python/gallery.py) is cloned and the released pylibkriging installed. import importlib import os import subprocess @@ -22,5 +22,5 @@ "(PyPI has wheels for CPython 3.10 to 3.12 only). On Colab: Runtime > Change runtime version > " "2026.07 (Python 3.12), then run again.\n" + r.stderr[-500:]) importlib.invalidate_caches() -sys.path.insert(0, str(Path("tools").resolve())) # lkgallery, when not installed (Colab, Binder) +sys.path.insert(0, str(Path("tools/python").resolve())) # for "import gallery" (tools/python/gallery.py) print("working directory:", os.getcwd()) diff --git a/tools/octave/gallery.m b/tools/octave/gallery.m new file mode 100644 index 0000000..337ab63 --- /dev/null +++ b/tools/octave/gallery.m @@ -0,0 +1,190 @@ +% Helpers of the libKriging gallery notebooks (Octave), in one script of functions: +% source("tools/octave/gallery.m") (the leading "1;" makes it a script, so that it can +% define several functions). +% +% The same functions, under the same names, as tools/python/gallery.py (where they are +% methods of Suite: suite.check_close(...) is s = check_close(s, ...) here; a suite is a +% struct passed by value), and the same CSV outputs (results/.checks.csv, +% .metrics.csv), aggregated with the other languages by tools/report.py. +% +% s = suite(name) a suite of checks, named after the notebook +% s = check(s, check, ok, value, expected, detail, severity) +% s = check_true / check_less / check_greater / check_between / check_close / +% check_finite / check_skip (s, ...) the usual checks +% s = metric(s, case, package, name, value) record a metric +% suite_summary(s) end-of-notebook summary +% suite_finalize(s) write the CSV files, error if an error-level check failed +% read_data(name, stem), rmse, q2 +% +% Paths are relative to the repository root ($LKGALLERY_ROOT, else the working +% directory, where the notebooks are executed). +1; + +function r = gallery_root() + r = getenv("LKGALLERY_ROOT"); if isempty(r), r = "."; end +end + +% ---- suite and checks ------------------------------------------------------------------ + +function s = suite(name) + s.name = name; + s.checks = cell(0, 8); + s.metrics = cell(0, 6); + s.t0 = tic(); + s.known = cell(0, 3); + f = fullfile(gallery_root(), "tools", "known_issues.csv"); + if ~exist(f, "file"), error("run the notebooks from the repository root (%s not found)", f); end + fid = fopen(f, "r"); fgetl(fid); + while true + l = fgetl(fid); + if ~ischar(l), break; end + p = gallery_csv_fields(l); + if numel(p) >= 3, s.known(end + 1, :) = p(1:3); end + end + fclose(fid); +end + +function s = check(s, name, ok, value, expected, detail, severity) + if nargin < 4, value = ""; end + if nargin < 5, expected = ""; end + if nargin < 6, detail = ""; end + if nargin < 7, severity = "error"; end + if ok, status = "PASS"; elseif strcmp(severity, "error"), status = "FAIL"; else, status = "WARN"; end + if ~ok && ~isempty(s.known) + hit = strcmp(s.known(:, 1), s.name) & strcmp(s.known(:, 2), name); + if any(hit), status = "KNOWN"; detail = [s.known{find(hit, 1), 3} " | " detail]; end + end + if isnumeric(value) || islogical(value), value = sprintf("%.6g", double(value)); end + if ~ok, printf("[%s] %s: value=%s expected %s %s\n", status, name, value, expected, detail); end + s.checks(end + 1, :) = {s.name, "octave", name, severity, status, value, expected, detail}; +end + +function s = check_true(s, name, cond, detail, severity) + if nargin < 4, detail = ""; end + if nargin < 5, severity = "error"; end + s = check(s, name, logical(cond), logical(cond), "1", detail, severity); +end + +function s = check_less(s, name, value, bound, detail, severity) + if nargin < 5, detail = ""; end + if nargin < 6, severity = "error"; end + s = check(s, name, isfinite(value) && value < bound, value, sprintf("< %g", bound), detail, severity); +end + +function s = check_greater(s, name, value, bound, detail, severity) + if nargin < 5, detail = ""; end + if nargin < 6, severity = "error"; end + s = check(s, name, isfinite(value) && value > bound, value, sprintf("> %g", bound), detail, severity); +end + +function s = check_between(s, name, value, lo, hi, detail, severity) + if nargin < 6, detail = ""; end + if nargin < 7, severity = "error"; end + s = check(s, name, isfinite(value) && value >= lo && value <= hi, value, sprintf("in [%g, %g]", lo, hi), detail, severity); +end + +function s = check_close(s, name, a, b, rtol, atol, detail, severity) + if nargin < 5, rtol = 1e-6; end + if nargin < 6, atol = 0; end + if nargin < 7, detail = ""; end + if nargin < 8, severity = "error"; end + a = double(a(:)); b = double(b(:)); + if numel(a) ~= numel(b) + s = check(s, name, false, sprintf("length %d", numel(a)), sprintf("length %d", numel(b)), detail, severity); + return; + end + gap = abs(a - b); + ok = all(isfinite(a)) && all(gap <= atol + rtol * abs(b)); + rel = max(gap ./ max(abs(b), 1e-300)); + s = check(s, name, ok, rel, sprintf("rtol %g, atol %g", rtol, atol), detail, severity); +end + +function s = check_finite(s, name, a, detail, severity) + if nargin < 4, detail = ""; end + if nargin < 5, severity = "error"; end + s = check(s, name, ~isempty(a) && all(isfinite(a(:))), sprintf("%d non-finite", sum(~isfinite(a(:)))), "all finite", detail, severity); +end + +function s = check_skip(s, name, reason) + s.checks(end + 1, :) = {s.name, "octave", name, "info", "SKIP", "", "", reason}; + printf("[SKIP] %s: %s\n", name, reason); +end + +% ---- metrics and results --------------------------------------------------------------- + +function s = metric(s, case_name, package, name, value) + s.metrics(end + 1, :) = {s.name, "octave", case_name, package, name, sprintf("%.6g", double(value))}; +end + +function suite_summary(s) + for i = 1:size(s.metrics, 1) + printf("%-28s %-28s %-12s %s\n", s.metrics{i, 3}, s.metrics{i, 4}, s.metrics{i, 5}, s.metrics{i, 6}); + end + st = s.checks(:, 5); + printf("%d checks: PASS %d, FAIL %d, WARN %d, KNOWN %d, SKIP %d\n", numel(st), sum(strcmp(st, "PASS")), ... + sum(strcmp(st, "FAIL")), sum(strcmp(st, "WARN")), sum(strcmp(st, "KNOWN")), sum(strcmp(st, "SKIP"))); + for i = find(~strcmp(st, "PASS"))' + printf("%-6s %-6s %s: value %s, expected %s %s\n", s.checks{i, 5}, s.checks{i, 4}, s.checks{i, 3}, ... + s.checks{i, 6}, s.checks{i, 7}, s.checks{i, 8}); + end +end + +function suite_finalize(s) + s = metric(s, "_suite", "_", "elapsed_s", toc(s.t0)); + d = getenv("LKGALLERY_RESULTS"); if isempty(d), d = fullfile(gallery_root(), "results"); end + if ~exist(d, "dir"), mkdir(d); end + gallery_write_csv(fullfile(d, [s.name ".checks.csv"]), {"suite", "lang", "check", "severity", "status", "value", "expected", "detail"}, s.checks); + gallery_write_csv(fullfile(d, [s.name ".metrics.csv"]), {"suite", "lang", "case", "package", "metric", "value"}, s.metrics); + st = s.checks(:, 5); + printf("%s: PASS %d, FAIL %d, WARN %d, KNOWN %d, SKIP %d\n", s.name, sum(strcmp(st, "PASS")), sum(strcmp(st, "FAIL")), ... + sum(strcmp(st, "WARN")), sum(strcmp(st, "KNOWN")), sum(strcmp(st, "SKIP"))); + failed = s.checks(strcmp(st, "FAIL"), 3); + if ~isempty(failed) + error("%d check(s) failed in %s: %s", numel(failed), s.name, strjoin(failed', "; ")); + end +end + +% ---- data and metrics ------------------------------------------------------------------ + +function A = read_data(name, stem) % data/_.csv (no header) + A = dlmread(fullfile(gallery_root(), "data", [name "_" stem ".csv"]), ","); +end + +function r = rmse(y, m) + r = sqrt(mean((y(:) - m(:)).^2)); +end + +function r = q2(y, m) + r = 1 - sum((y(:) - m(:)).^2) / sum((y(:) - mean(y(:))).^2); +end + +% ---- internals ------------------------------------------------------------------------- + +function gallery_write_csv(f, header, rows) + fid = fopen(f, "w"); + fprintf(fid, "%s\n", strjoin(header, ",")); + for i = 1:size(rows, 1) + q = cellfun(@(c) ["\"" strrep(char(c), "\"", "\"\"") "\""], rows(i, :), "UniformOutput", false); + fprintf(fid, "%s\n", strjoin(q, ",")); + end + fclose(fid); +end + +function f = gallery_csv_fields(l) + % one CSV line into fields (RFC 4180 quoting: "a,b" and "" inside quotes) + f = {}; cur = ""; q = false; i = 1; + while i <= numel(l) + c = l(i); + if q && c == '"' && i < numel(l) && l(i + 1) == '"' + cur(end + 1) = '"'; i = i + 1; + elseif c == '"' + q = ~q; + elseif c == "," && ~q + f{end + 1} = cur; cur = ""; + else + cur(end + 1) = c; + end + i = i + 1; + end + f{end + 1} = cur; +end diff --git a/tools/octave/lkb_check.m b/tools/octave/lkb_check.m deleted file mode 100644 index 33c30e5..0000000 --- a/tools/octave/lkb_check.m +++ /dev/null @@ -1,14 +0,0 @@ -function s = lkb_check(s, check, ok, value, expected, detail, severity) - if nargin < 4, value = ""; end - if nargin < 5, expected = ""; end - if nargin < 6, detail = ""; end - if nargin < 7, severity = "error"; end - if ok, status = "PASS"; elseif strcmp(severity, "error"), status = "FAIL"; else, status = "WARN"; end - if ~ok && ~isempty(s.known) - hit = strcmp(s.known(:, 1), s.name) & strcmp(s.known(:, 2), check); - if any(hit), status = "KNOWN"; detail = [s.known{find(hit, 1), 3} " | " detail]; end - end - if isnumeric(value) || islogical(value), value = sprintf("%.6g", double(value)); end - if ~ok, printf("[%s] %s: value=%s expected %s %s\n", status, check, value, expected, detail); end - s.checks(end + 1, :) = {s.name, "octave", check, severity, status, value, expected, detail}; -end diff --git a/tools/octave/lkb_close.m b/tools/octave/lkb_close.m deleted file mode 100644 index e7172a1..0000000 --- a/tools/octave/lkb_close.m +++ /dev/null @@ -1,15 +0,0 @@ -function s = lkb_close(s, check, a, b, rtol, atol, detail, severity) - if nargin < 5, rtol = 1e-6; end - if nargin < 6, atol = 0; end - if nargin < 7, detail = ""; end - if nargin < 8, severity = "error"; end - a = double(a(:)); b = double(b(:)); - if numel(a) ~= numel(b) - s = lkb_check(s, check, false, sprintf("length %d", numel(a)), sprintf("length %d", numel(b)), detail, severity); - return; - end - gap = abs(a - b); - ok = all(isfinite(a)) && all(gap <= atol + rtol * abs(b)); - rel = max(gap ./ max(abs(b), 1e-300)); - s = lkb_check(s, check, ok, rel, sprintf("rtol %g, atol %g", rtol, atol), detail, severity); -end diff --git a/tools/octave/lkb_finalize.m b/tools/octave/lkb_finalize.m deleted file mode 100644 index 110d8c6..0000000 --- a/tools/octave/lkb_finalize.m +++ /dev/null @@ -1,24 +0,0 @@ -function lkb_finalize(s) - s = lkb_metric(s, "_suite", "_", "elapsed_s", toc(s.t0)); - d = getenv("LKGALLERY_RESULTS"); if isempty(d), d = "results"; end - if ~exist(d, "dir"), mkdir(d); end - write_csv(fullfile(d, [s.name ".checks.csv"]), {"suite", "lang", "check", "severity", "status", "value", "expected", "detail"}, s.checks); - write_csv(fullfile(d, [s.name ".metrics.csv"]), {"suite", "lang", "case", "package", "metric", "value"}, s.metrics); - st = s.checks(:, 5); - printf("%s: PASS %d, FAIL %d, WARN %d, KNOWN %d, SKIP %d\n", s.name, sum(strcmp(st, "PASS")), sum(strcmp(st, "FAIL")), ... - sum(strcmp(st, "WARN")), sum(strcmp(st, "KNOWN")), sum(strcmp(st, "SKIP"))); - failed = s.checks(strcmp(st, "FAIL"), 3); - if ~isempty(failed) - error("%d check(s) failed in %s: %s", numel(failed), s.name, strjoin(failed', "; ")); - end -end - -function write_csv(f, header, rows) - fid = fopen(f, "w"); - fprintf(fid, "%s\n", strjoin(header, ",")); - for i = 1:size(rows, 1) - q = cellfun(@(c) ["\"" strrep(char(c), "\"", "\"\"") "\""], rows(i, :), "UniformOutput", false); - fprintf(fid, "%s\n", strjoin(q, ",")); - end - fclose(fid); -end diff --git a/tools/octave/lkb_metric.m b/tools/octave/lkb_metric.m deleted file mode 100644 index bd38ed1..0000000 --- a/tools/octave/lkb_metric.m +++ /dev/null @@ -1,3 +0,0 @@ -function s = lkb_metric(s, case_name, package, metric, value) - s.metrics(end + 1, :) = {s.name, "octave", case_name, package, metric, sprintf("%.6g", double(value))}; -end diff --git a/tools/octave/lkb_suite.m b/tools/octave/lkb_suite.m deleted file mode 100644 index 3820d7e..0000000 --- a/tools/octave/lkb_suite.m +++ /dev/null @@ -1,39 +0,0 @@ -function s = lkb_suite(name) - % Same CSV outputs as lkgallery/checks.py (see there), for the Octave notebooks. - % A suite is a struct passed around by value: s = lkb_close(s, ...). - % Paths are relative to the repository root, where the notebooks are executed. - s.name = name; - s.checks = {}; - s.metrics = {}; - s.t0 = tic(); - s.known = cell(0, 3); - f = fullfile("tools", "known_issues.csv"); - if ~exist(f, "file"), error("run the notebooks from the repository root (%s not found)", f); end - fid = fopen(f, "r"); fgetl(fid); - while true - l = fgetl(fid); - if ~ischar(l), break; end - p = csv_fields(l); - if numel(p) >= 3, s.known(end + 1, :) = p(1:3); end - end - fclose(fid); -end - -function f = csv_fields(l) - % one CSV line into fields (RFC 4180 quoting: "a,b" and "" inside quotes) - f = {}; cur = ""; q = false; i = 1; - while i <= numel(l) - c = l(i); - if q && c == '"' && i < numel(l) && l(i + 1) == '"' - cur(end + 1) = '"'; i = i + 1; - elseif c == '"' - q = ~q; - elseif c == "," && ~q - f{end + 1} = cur; cur = ""; - else - cur(end + 1) = c; - end - i = i + 1; - end - f{end + 1} = cur; -end diff --git a/tools/octave/lkb_summary.m b/tools/octave/lkb_summary.m deleted file mode 100644 index 275bb1c..0000000 --- a/tools/octave/lkb_summary.m +++ /dev/null @@ -1,13 +0,0 @@ -function lkb_summary(s) - % End-of-notebook summary: metrics, count per status, every check that did not pass. - for i = 1:size(s.metrics, 1) - printf("%-28s %-28s %-12s %s\n", s.metrics{i, 3}, s.metrics{i, 4}, s.metrics{i, 5}, s.metrics{i, 6}); - end - st = s.checks(:, 5); - printf("%d checks: PASS %d, FAIL %d, WARN %d, KNOWN %d, SKIP %d\n", numel(st), sum(strcmp(st, "PASS")), ... - sum(strcmp(st, "FAIL")), sum(strcmp(st, "WARN")), sum(strcmp(st, "KNOWN")), sum(strcmp(st, "SKIP"))); - for i = find(~strcmp(st, "PASS"))' - printf("%-6s %-6s %s: value %s, expected %s %s\n", s.checks{i, 5}, s.checks{i, 4}, s.checks{i, 3}, ... - s.checks{i, 6}, s.checks{i, 7}, s.checks{i, 8}); - end -end diff --git a/tools/octave/read_data.m b/tools/octave/read_data.m deleted file mode 100644 index ada51ee..0000000 --- a/tools/octave/read_data.m +++ /dev/null @@ -1,3 +0,0 @@ -function A = read_data(name, stem) - A = dlmread(fullfile("data", [name "_" stem ".csv"]), ","); -end diff --git a/tools/pyproject.toml b/tools/pyproject.toml deleted file mode 100644 index fb31f56..0000000 --- a/tools/pyproject.toml +++ /dev/null @@ -1,18 +0,0 @@ -[build-system] -requires = ["setuptools>=64"] -build-backend = "setuptools.build_meta" - -[project] -name = "lkgallery" -version = "0.1.0" -description = "Helpers (test functions, designs, metrics, checks) for the notebooks of the libKriging gallery" -requires-python = ">=3.9" -dependencies = ["numpy", "scipy", "pandas", "matplotlib", "jinja2"] # jinja2: DataFrame.style - -[project.optional-dependencies] -# the comparison packages (scikit-learn, SMT, OpenTURNS, GPy, ...) are installed by the -# notebooks that use them, not here -notebooks = ["nbclient", "nbformat", "ipykernel"] - -[tool.setuptools] -packages = ["lkgallery"] diff --git a/tools/python/gallery.py b/tools/python/gallery.py new file mode 100644 index 0000000..dde1423 --- /dev/null +++ b/tools/python/gallery.py @@ -0,0 +1,674 @@ +"""Helpers of the libKriging gallery notebooks (Python), in one file. + +Loaded by the notebooks with ``import gallery`` (their first cell puts ``tools/python`` +on ``sys.path``). The R, Octave and Julia notebooks load ``tools/R/gallery.R``, +``tools/octave/gallery.m`` and ``tools/julia/gallery.jl``, which provide the same +check / metric / summary / data functions under the same names (see each file), +and write the same CSV files, aggregated by ``tools/report.py``. + +Sections: + 1. test functions (``TESTS``, ``TestFunction``) + 2. designs of experiments (``design.lhs``, ``design.dataset``, ...) + 3. metrics (``metrics.rmse``, ``metrics.q2``, ``metrics.nlpd``, ...) + 4. what the installed pylibkriging offers (``compat.has``, ``compat.version``) + 5. checks and results (``Suite``: ``check``, ``check_close``, ``metric``, ``summary``, ...) + 6. persistence checks (``persistence.check``) + 7. shared datasets (``read_data``) +""" +import copy +import csv +import functools +import importlib +import math +import os +import pickle +import tempfile +import time +import traceback +from contextlib import contextmanager +from dataclasses import dataclass, field +from pathlib import Path +from types import SimpleNamespace + +import numpy as np +from scipy.stats import norm, qmc + + +# ====================================================================================== +# 1. Test functions +# ====================================================================================== + +# Test functions for surrogate modelling, after the Virtual Library of Simulation +# Experiments (Surjanovic & Bingham, https://www.sfu.ca/~ssurjano/). +# +# Every function takes ``X`` of shape (n, d) in its *native* domain and returns a +# length-n vector. ``TESTS`` gathers the metadata (dimension, domain, a reference +# point with its known value) used by ``test-functions-reference-values_no-model_python.ipynb`` +# to check the implementations themselves before any model is fitted on them. + +# --- 1D ------------------------------------------------------------------------- + +def forrester(X): + x = X[:, 0] + return (6 * x - 2) ** 2 * np.sin(12 * x - 4) + + +def gramacy_lee_2012(X): + x = X[:, 0] + return np.sin(10 * np.pi * x) / (2 * x) + (x - 1) ** 4 + + +# --- 2D ------------------------------------------------------------------------- + +def branin(X): + x1, x2 = X[:, 0], X[:, 1] + a, b, c = 1.0, 5.1 / (4 * np.pi ** 2), 5 / np.pi + r, s, t = 6.0, 10.0, 1 / (8 * np.pi) + return a * (x2 - b * x1 ** 2 + c * x1 - r) ** 2 + s * (1 - t) * np.cos(x1) + s + + +def six_hump_camel(X): + x1, x2 = X[:, 0], X[:, 1] + return (4 - 2.1 * x1 ** 2 + x1 ** 4 / 3) * x1 ** 2 + x1 * x2 + (-4 + 4 * x2 ** 2) * x2 ** 2 + + +def goldstein_price(X): + x1, x2 = X[:, 0], X[:, 1] + a = 1 + (x1 + x2 + 1) ** 2 * (19 - 14 * x1 + 3 * x1 ** 2 - 14 * x2 + 6 * x1 * x2 + 3 * x2 ** 2) + b = 30 + (2 * x1 - 3 * x2) ** 2 * (18 - 32 * x1 + 12 * x1 ** 2 + 48 * x2 - 36 * x1 * x2 + 27 * x2 ** 2) + return a * b + + +def log_goldstein_price(X): + """Logarithmic Goldstein-Price (Picheny et al. 2013), on [0,1]^2.""" + Z = 4 * X - 2 + return (np.log(goldstein_price(Z)) - 8.693) / 2.427 + + +def currin_exp(X): + x1, x2 = X[:, 0], X[:, 1] + return (1 - np.exp(-1 / (2 * x2))) * (2300 * x1 ** 3 + 1900 * x1 ** 2 + 2092 * x1 + 60) / ( + 100 * x1 ** 3 + 500 * x1 ** 2 + 4 * x1 + 20) + + +def lim_polynomial(X): + x1, x2 = X[:, 0], X[:, 1] + return (9 + 5 / 2 * x1 - 35 / 2 * x2 + 5 / 2 * x1 * x2 + 19 * x2 ** 2 - 15 / 2 * x1 ** 3 + - 5 / 2 * x1 * x2 ** 2 - 11 / 2 * x2 ** 4 + x1 ** 3 * x2 ** 2) + + +def gramacy_lee_2008(X): + """Non-stationary 2D function x1 exp(-x1^2 - x2^2) on [-2, 6]^2.""" + x1, x2 = X[:, 0], X[:, 1] + return x1 * np.exp(-x1 ** 2 - x2 ** 2) + + +# --- 3D to 5D ------------------------------------------------------------------- + +def ishigami(X, a=7.0, b=0.1): + x1, x2, x3 = X[:, 0], X[:, 1], X[:, 2] + return np.sin(x1) + a * np.sin(x2) ** 2 + b * x3 ** 4 * np.sin(x1) + + +_H3_ALPHA = np.array([1.0, 1.2, 3.0, 3.2]) +_H3_A = np.array([[3, 10, 30], [0.1, 10, 35], [3, 10, 30], [0.1, 10, 35]], float) +_H3_P = 1e-4 * np.array([[3689, 1170, 2673], [4699, 4387, 7470], + [1091, 8732, 5547], [381, 5743, 8828]], float) + + +def hartmann3(X): + diff2 = (X[:, None, :] - _H3_P[None, :, :]) ** 2 + return -np.exp(-np.sum(_H3_A[None, :, :] * diff2, axis=2)) @ _H3_ALPHA + + +def friedman(X): + x1, x2, x3, x4, x5 = (X[:, i] for i in range(5)) + return 10 * np.sin(np.pi * x1 * x2) + 20 * (x3 - 0.5) ** 2 + 10 * x4 + 5 * x5 + + +# --- 6D to 8D ------------------------------------------------------------------- + +_H6_ALPHA = np.array([1.0, 1.2, 3.0, 3.2]) +_H6_A = np.array([[10, 3, 17, 3.5, 1.7, 8], + [0.05, 10, 17, 0.1, 8, 14], + [3, 3.5, 1.7, 10, 17, 8], + [17, 8, 0.05, 10, 0.1, 14]], float) +_H6_P = 1e-4 * np.array([[1312, 1696, 5569, 124, 8283, 5886], + [2329, 4135, 8307, 3736, 1004, 9991], + [2348, 1451, 3522, 2883, 3047, 6650], + [4047, 8828, 8732, 5743, 1091, 381]], float) + + +def hartmann6(X): + diff2 = (X[:, None, :] - _H6_P[None, :, :]) ** 2 + return -np.exp(-np.sum(_H6_A[None, :, :] * diff2, axis=2)) @ _H6_ALPHA + + +def otl_circuit(X): + Rb1, Rb2, Rf, Rc1, Rc2, beta = (X[:, i] for i in range(6)) + Vb1 = 12 * Rb2 / (Rb1 + Rb2) + den = beta * (Rc2 + 9) + Rf + return ((Vb1 + 0.74) * beta * (Rc2 + 9) / den + 11.35 * Rf / den + + 0.74 * Rf * beta * (Rc2 + 9) / (den * Rc1)) + + +def piston(X): + M, S, V0, k, P0, Ta, T0 = (X[:, i] for i in range(7)) + A = P0 * S + 19.62 * M - k * V0 / S + V = S / (2 * k) * (np.sqrt(A ** 2 + 4 * k * P0 * V0 * Ta / T0) - A) + return 2 * np.pi * np.sqrt(M / (k + S ** 2 * P0 * V0 * Ta / (T0 * V ** 2))) + + +def borehole(X): + rw, r, Tu, Hu, Tl, Hl, L, Kw = (X[:, i] for i in range(8)) + lnr = np.log(r / rw) + return 2 * np.pi * Tu * (Hu - Hl) / (lnr * (1 + 2 * L * Tu / (lnr * rw ** 2 * Kw) + Tu / Tl)) + + +def robot_arm(X): + theta, L = X[:, :4], X[:, 4:] + angles = np.cumsum(theta, axis=1) + u = np.sum(L * np.cos(angles), axis=1) + v = np.sum(L * np.sin(angles), axis=1) + return np.sqrt(u ** 2 + v ** 2) + + +def dette_pepelyshev(X): + x = X + s = 0.0 + for i in range(4, 9): # i = 4..8, inner sum over j = 3..i (1-based) + s = s + i * np.log(1 + np.sum(x[:, 2:i], axis=1)) + return (4 * (x[:, 0] - 2 + 8 * x[:, 1] - 8 * x[:, 1] ** 2) ** 2 + (3 - 4 * x[:, 1]) ** 2 + + 16 * np.sqrt(x[:, 2] + 1) * (2 * x[:, 2] - 1) ** 2 + s) + + +# --- 10D and more --------------------------------------------------------------- + +def wing_weight(X): + Sw, Wfw, A, Lam, q, lam, tc, Nz, Wdg, Wp = (X[:, i] for i in range(10)) + Lam = np.deg2rad(Lam) + return (0.036 * Sw ** 0.758 * Wfw ** 0.0035 * (A / np.cos(Lam) ** 2) ** 0.6 * q ** 0.006 + * lam ** 0.04 * (100 * tc / np.cos(Lam)) ** -0.3 * (Nz * Wdg) ** 0.49 + Sw * Wp) + + +def welch(X): + x = [None] + [X[:, i] for i in range(20)] # 1-based, as in the reference + return (5 * x[12] / (1 + x[1]) + 5 * (x[4] - x[20]) ** 2 + x[5] + 40 * x[19] ** 3 - 5 * x[19] + + 0.05 * x[2] + 0.08 * x[3] - 0.03 * x[6] + 0.03 * x[7] - 0.09 * x[9] - 0.01 * x[10] + - 0.07 * x[11] + 0.25 * x[13] ** 2 - 0.04 * x[14] + 0.06 * x[15] - 0.01 * x[17] + - 0.03 * x[18]) + + +def ackley(X, a=20.0, b=0.2, c=2 * np.pi): + d = X.shape[1] + return (-a * np.exp(-b * np.sqrt(np.sum(X ** 2, axis=1) / d)) + - np.exp(np.sum(np.cos(c * X), axis=1) / d) + a + np.e) + + +@dataclass +class TestFunction: + name: str + f: callable + lower: np.ndarray + upper: np.ndarray + # (x, f(x)) pairs taken from the reference pages (global optima, nominal values) + known: list = field(default_factory=list) + tags: tuple = () + + @property + def d(self): + return len(self.lower) + + def __call__(self, X): + return self.f(np.atleast_2d(X)) + + def scale(self, U): + """Map points of [0,1]^d to the native domain.""" + return self.lower + np.asarray(U) * (self.upper - self.lower) + + def unit(self, U): + """The function seen on [0,1]^d.""" + return self.f(self.scale(np.atleast_2d(U))) + + +def _box(*bounds): + b = np.array(bounds, float) + return b[:, 0], b[:, 1] + + +def _tf(name, f, bounds, known=(), tags=()): + lo, up = _box(*bounds) + return TestFunction(name, f, lo, up, [(np.array(x, float), v) for x, v in known], tuple(tags)) + + +TESTS = {t.name: t for t in [ + _tf("forrester", forrester, [(0, 1)], [([0.757249], -6.02074)], ["1d"]), + _tf("gramacy_lee_2012", gramacy_lee_2012, [(0.5, 2.5)], [([0.548563], -0.869011)], ["1d", "nonstationary"]), + _tf("branin", branin, [(-5, 10), (0, 15)], + [([-np.pi, 12.275], 0.397887), ([np.pi, 2.275], 0.397887), ([9.42478, 2.475], 0.397887)], ["2d"]), + _tf("six_hump_camel", six_hump_camel, [(-3, 3), (-2, 2)], + [([0.0898, -0.7126], -1.0316), ([-0.0898, 0.7126], -1.0316)], ["2d"]), + _tf("goldstein_price", goldstein_price, [(-2, 2), (-2, 2)], [([0, -1], 3.0)], ["2d", "large-range"]), + _tf("log_goldstein_price", log_goldstein_price, [(0, 1), (0, 1)], + [([0.5, 0.25], (np.log(3.0) - 8.693) / 2.427)], ["2d"]), + _tf("currin_exp", currin_exp, [(0, 1), (1e-6, 1)], [], ["2d"]), + _tf("lim_polynomial", lim_polynomial, [(0, 1), (0, 1)], [([0, 0], 9.0)], ["2d", "polynomial"]), + _tf("gramacy_lee_2008", gramacy_lee_2008, [(-2, 6), (-2, 6)], + [([-1 / np.sqrt(2), 0], -np.exp(-0.5) / np.sqrt(2))], ["2d", "nonstationary"]), + _tf("ishigami", ishigami, [(-np.pi, np.pi)] * 3, [([0, 0, 0], 0.0), ([0, np.pi / 2, 0], 7.0)], ["3d"]), + _tf("hartmann3", hartmann3, [(0, 1)] * 3, [([0.114614, 0.555649, 0.852547], -3.86278)], ["3d"]), + _tf("friedman", friedman, [(0, 1)] * 5, [([0, 0, 0.5, 0, 0], 0.0)], ["5d"]), + _tf("hartmann6", hartmann6, [(0, 1)] * 6, + [([0.20169, 0.150011, 0.476874, 0.275332, 0.311652, 0.6573], -3.32237)], ["6d"]), + _tf("otl_circuit", otl_circuit, [(50, 150), (25, 70), (0.5, 3), (1.2, 2.5), (0.25, 1.2), (50, 300)], + [], ["6d", "engineering"]), + _tf("piston", piston, [(30, 60), (0.005, 0.020), (0.002, 0.010), (1000, 5000), (90000, 110000), + (290, 296), (340, 360)], [], ["7d", "engineering"]), + _tf("borehole", borehole, [(0.05, 0.15), (100, 50000), (63070, 115600), (990, 1110), + (63.1, 116), (700, 820), (1120, 1680), (9855, 12045)], [], ["8d", "engineering"]), + _tf("robot_arm", robot_arm, [(0, 2 * np.pi)] * 4 + [(0, 1)] * 4, + [([0, 0, 0, 0, 1, 1, 1, 1], 4.0), ([0, np.pi, 0, 0, 1, 1, 1, 1], 2.0)], ["8d", "oscillating"]), + _tf("dette_pepelyshev", dette_pepelyshev, [(0, 1)] * 8, [], ["8d", "curved"]), + _tf("wing_weight", wing_weight, [(150, 200), (220, 300), (6, 10), (-10, 10), (16, 45), (0.5, 1), + (0.08, 0.18), (2.5, 6), (1700, 2500), (0.025, 0.08)], [], ["10d", "engineering"]), + _tf("welch", welch, [(-0.5, 0.5)] * 20, [([0] * 20, 0.0)], ["20d", "screening"]), + _tf("ackley4", ackley, [(-2, 2)] * 4, [([0, 0, 0, 0], 0.0)], ["4d", "multimodal"]), +]} + +# ====================================================================================== +# 2. Designs of experiments +# ====================================================================================== + +# Seeded designs of experiments, all on [0,1]^d (scale with ``TestFunction.scale``). + +def lhs(n, d, seed=0, optimized=True): + """Latin hypercube; ``optimized`` applies scipy's random-CD improvement (space filling).""" + sampler = qmc.LatinHypercube(d=d, seed=seed, optimization="random-cd" if optimized and d > 1 else None) + return sampler.random(n) + + +def sobol(n, d, seed=0): + return qmc.Sobol(d=d, scramble=True, seed=seed).random(n) + + +def uniform(n, d, seed=0): + return np.random.default_rng(seed).random((n, d)) + + +def dataset(tf, n, n_test=1000, seed=0, noise_sd=0.0): + """Training/test sets for test function ``tf`` on the unit hypercube. + + Returns (X, y, Xt, yt) with X in [0,1]^d (the GP sees the unit cube; the + function is evaluated in its native domain). ``noise_sd`` adds Gaussian + noise to y only (yt stays noise-free, it is the target).""" + X = lhs(n, tf.d, seed=seed) + Xt = uniform(n_test, tf.d, seed=10_000 + seed) + y = tf.unit(X) + if noise_sd > 0: + y = y + np.random.default_rng(20_000 + seed).normal(0.0, noise_sd, size=n) + return X, y, Xt, tf.unit(Xt) + +# ====================================================================================== +# 3. Metrics +# ====================================================================================== + +# Accuracy and calibration metrics for probabilistic predictions. + +def rmse(y, m): + return float(np.sqrt(np.mean((np.asarray(y) - np.asarray(m)) ** 2))) + + +def q2(y, m): + """Predictivity coefficient 1 - SSE/SST on a test set (1 is perfect).""" + y, m = np.asarray(y), np.asarray(m) + return float(1 - np.sum((y - m) ** 2) / np.sum((y - y.mean()) ** 2)) + + +def nlpd(y, m, s): + """Mean negative log predictive density under N(m, s^2).""" + s = np.maximum(np.asarray(s), 1e-12) + return float(np.mean(0.5 * np.log(2 * np.pi * s ** 2) + 0.5 * ((np.asarray(y) - m) / s) ** 2)) + + +def coverage(y, m, s, level=0.95): + """Fraction of test values inside the central ``level`` prediction interval.""" + z = norm.ppf(0.5 + level / 2) + return float(np.mean(np.abs(np.asarray(y) - m) <= z * np.asarray(s))) + + +def crps(y, m, s): + """Mean continuous ranked probability score of N(m, s^2) (lower is better).""" + s = np.maximum(np.asarray(s), 1e-12) + z = (np.asarray(y) - m) / s + return float(np.mean(s * (z * (2 * norm.cdf(z) - 1) + 2 * norm.pdf(z) - 1 / np.sqrt(np.pi)))) + + +def all_metrics(y, m, s=None): + out = {"rmse": rmse(y, m), "q2": q2(y, m)} + if s is not None: + out.update(nlpd=nlpd(y, m, s), coverage95=coverage(y, m, s), crps=crps(y, m, s)) + return out + + +def finite_diff_grad(f, x, h=1e-6): + """Central finite-difference gradient of a scalar function, relative step.""" + x = np.asarray(x, float) + g = np.zeros_like(x) + for i in range(x.size): + e = np.zeros_like(x) + e[i] = h * max(1.0, abs(x[i])) + g[i] = (f(x + e) - f(x - e)) / (2 * e[i]) + return g + +# ====================================================================================== +# 4. What the installed pylibkriging offers +# ====================================================================================== + +# What the installed pylibkriging offers, so that one notebook runs against both +# the last release and the development head: a feature missing from the installed +# version is recorded as SKIP (with the reason), never as a failure. + +def version(): + import pylibkriging as lk + v = getattr(lk, "__version__", None) + if v is None: + try: + from importlib.metadata import version as _v + v = _v("pylibkriging") + except Exception: # noqa: BLE001 + v = "unknown" + return v + + +@functools.lru_cache(None) +def has(feature): + """Probe a feature on a tiny problem (cached).""" + import pylibkriging as lk + X = np.linspace(0, 1, 6).reshape(-1, 1) + y = np.sin(3 * X[:, 0]) + probes = { + "method": lambda: lk.Kriging(y, X, "matern5_2", method="vecchia(3)"), + "noise": lambda: lk.Kriging(y, X, "matern5_2", noise="nugget"), + "WarpKriging": lambda: lk.WarpKriging(y, X, ["none"], "matern5_2"), + "MLPKriging": lambda: lk.MLPKriging(y, X, hidden_dims=[2], d_out=1, kernel="gauss"), + "NestedKriging": lambda: lk.NestedKriging(y, X, "matern5_2", nb_groups=2), + "subsetOfData": lambda: lk.Kriging.subsetOfData(X, n_max=3, method="kmeans", seed=1), + "save": lambda: lk.Kriging(y, X, "matern5_2").save, + "pickle": lambda: __import__("pickle").dumps(lk.Kriging(y, X, "matern5_2")), + "NestedKriging.save": lambda: lk.NestedKriging(y, X, "matern5_2", nb_groups=2).save, + "load(noise)": lambda: _roundtrip(lk, lk.Kriging(y, X, "matern5_2", noise="nugget")), + "sklearn": lambda: importlib.import_module("pylibkriging.sklearn").KrigingRegressor, + } + try: + probes[feature]() + return True + except Exception: # noqa: BLE001 + return False + + +def _roundtrip(lk, k): + import os + import tempfile + f = os.path.join(tempfile.mkdtemp(), "k.json") + k.save(f) + return lk.load(f) + + +def optional(module): + """Import an optional comparison package, or return None.""" + try: + return importlib.import_module(module) + except Exception: # noqa: BLE001 - broken installs count as absent + return None + +# ====================================================================================== +# 5. Checks and results +# ====================================================================================== + +# Record checks and metrics from a notebook, and fail it at the end if needed. +# +# A notebook creates one ``Suite``, records any number of checks (assertions that +# do *not* stop the notebook, so every problem of a run is reported at once) and +# metrics, then calls ``suite.finalize()`` in its last cell: it writes +# +# results/.checks.csv suite,lang,check,severity,status,value,expected,detail +# results/.metrics.csv suite,lang,case,package,metric,value +# +# and raises ``AssertionError`` if any check of severity "error" failed, which makes +# the notebook (hence the CI job) fail. Checks of severity "warn" are reported but +# never fail the run: they flag performance drifts or comparisons with other +# packages, where a threshold is a judgement call rather than a correctness bound. +# +# Statuses: PASS, FAIL (error-level), WARN (warn-level), SKIP (feature missing in +# the installed version), KNOWN (a failing check, of either severity, listed in +# ``tools/known_issues.csv``: reported with its issue, but does not fail the run). A +# listed check that passes is reported as FIXED by ``tools/report.py``, so that the +# list does not rot. +# +# Same names in every language: ``suite.check_close(...)`` here is ``check_close(s, ...)`` +# in R and Octave and ``check_close!(s, ...)`` in Julia; ``suite.metric``, +# ``suite.summary()`` and ``suite.finalize()`` are ``metric``, ``suite_summary`` and +# ``suite_finalize`` there. + +def _find_root(): + """The repository root: $LKGALLERY_ROOT, or the one this file belongs to + (tools/python/gallery.py).""" + root = Path(os.environ.get("LKGALLERY_ROOT", Path(__file__).resolve().parents[2])) + if not (root / "tools" / "known_issues.csv").is_file(): + raise RuntimeError(f"{root} is not the libKriging gallery repository (no tools/known_issues.csv)") + return root + + +ROOT = _find_root() +RESULTS_DIR = Path(os.environ.get("LKGALLERY_RESULTS", ROOT / "results")) +KNOWN_ISSUES_FILE = ROOT / "tools" / "known_issues.csv" +DATA_DIR = ROOT / "data" + + +def known_issues(): + """{(suite, check): issue} from tools/known_issues.csv.""" + with open(KNOWN_ISSUES_FILE, newline="") as fh: + return {(r["suite"], r["check"]): r["issue"] for r in csv.DictReader(fh)} + + +CHECK_FIELDS = ["suite", "lang", "check", "severity", "status", "value", "expected", "detail"] +METRIC_FIELDS = ["suite", "lang", "case", "package", "metric", "value"] + + +def _fmt(v): + if v is None: + return "" + if isinstance(v, (bool, np.bool_)): + return str(bool(v)) + if isinstance(v, (int, float, np.integer, np.floating)): + return f"{float(v):.6g}" + return str(v) + + +class Suite: + def __init__(self, name, lang="python"): + self.name, self.lang = name, lang + self.checks, self.metrics = [], [] + self.t0 = time.time() + self.known = known_issues() + + # -- checks --------------------------------------------------------------- + + def check(self, check, ok, value=None, expected=None, detail="", severity="error"): + status = "PASS" if ok else ("FAIL" if severity == "error" else "WARN") + if not ok and (self.name, check) in self.known: + status, detail = "KNOWN", f"{self.known[self.name, check]} | {detail}".rstrip(" |") + self.checks.append(dict(suite=self.name, lang=self.lang, check=check, severity=severity, + status=status, value=_fmt(value), expected=_fmt(expected), detail=detail)) + if not ok: + print(f"[{status}] {check}: value={_fmt(value)} expected {_fmt(expected)} {detail}") + return ok + + def check_true(self, check, cond, detail="", severity="error"): + return self.check(check, bool(cond), cond, True, detail, severity) + + def check_less(self, check, value, bound, detail="", severity="error"): + """value < bound (NaN fails).""" + value = float(value) + return self.check(check, math.isfinite(value) and value < bound, value, f"< {bound:g}", detail, severity) + + def check_greater(self, check, value, bound, detail="", severity="error"): + value = float(value) + return self.check(check, math.isfinite(value) and value > bound, value, f"> {bound:g}", detail, severity) + + def check_between(self, check, value, lo, hi, detail="", severity="error"): + value = float(value) + return self.check(check, math.isfinite(value) and lo <= value <= hi, value, f"in [{lo:g}, {hi:g}]", + detail, severity) + + def check_close(self, check, a, b, rtol=1e-6, atol=0.0, detail="", severity="error"): + """All |a - b| <= atol + rtol |b| (element-wise); reports the worst relative gap.""" + a, b = np.asarray(a, float), np.asarray(b, float) + if a.shape != b.shape: + return self.check(check, False, f"shape {a.shape}", f"shape {b.shape}", detail, severity) + gap = np.abs(a - b) + ok = bool(np.all(np.isfinite(a)) and np.all(gap <= atol + rtol * np.abs(b))) + rel = float(np.max(gap / np.maximum(np.abs(b), 1e-300))) if a.size else 0.0 + return self.check(check, ok, rel, f"rtol {rtol:g}, atol {atol:g}", detail, severity) + + def check_finite(self, check, a, detail="", severity="error"): + a = np.asarray(a, float) + return self.check(check, bool(a.size and np.all(np.isfinite(a))), f"{np.sum(~np.isfinite(a))} non-finite", + "all finite", detail, severity) + + def check_skip(self, check, reason): + self.checks.append(dict(suite=self.name, lang=self.lang, check=check, severity="info", + status="SKIP", value="", expected="", detail=reason)) + print(f"[SKIP] {check}: {reason}") + + @contextmanager + def check_runs(self, check, severity="error"): + """``with suite.check_runs("x"): ...`` records a FAIL (with the exception) instead of stopping.""" + try: + yield + except Exception as exc: # noqa: BLE001 - every failure is a finding here + self.check(check, False, type(exc).__name__, "no exception", + str(exc).splitlines()[0][:200] if str(exc) else traceback.format_exc(limit=1), severity) + else: + self.check(check, True, "ok", "no exception", "", severity) + + def check_raises(self, check, fn, *args, severity="error", **kwargs): + """``fn(*args)`` must raise (invalid input must be rejected, not silently accepted).""" + try: + fn(*args, **kwargs) + except Exception as exc: # noqa: BLE001 + return self.check(check, True, type(exc).__name__, "an exception", "", severity) + return self.check(check, False, "no exception", "an exception", "", severity) + + # -- metrics -------------------------------------------------------------- + + def metric(self, case, package, **values): + for k, v in values.items(): + self.metrics.append(dict(suite=self.name, lang=self.lang, case=case, package=package, + metric=k, value=_fmt(v))) + + def table(self): + """Metrics as a wide pandas DataFrame (one row per case x package).""" + import pandas as pd + if not self.metrics: + return pd.DataFrame() + df = pd.DataFrame(self.metrics) + df["value"] = pd.to_numeric(df["value"], errors="coerce") + return df.pivot_table(index=["case", "package"], columns="metric", values="value", sort=False) + + def summary(self): + """End-of-notebook summary: the metrics table, the count per status, and every check + that did not pass (with its value, threshold and, for KNOWN, its issue).""" + import pandas as pd + try: + from IPython.display import display + except ImportError: # outside Jupyter + display = print + if self.metrics: + display(self.table().round(4)) + df = pd.DataFrame(self.checks, columns=CHECK_FIELDS) + counts = {s: int((df["status"] == s).sum()) for s in ("PASS", "FAIL", "WARN", "KNOWN", "SKIP")} + print(f"{len(df)} checks: " + ", ".join(f"{k} {v}" for k, v in counts.items())) + others = df[df["status"] != "PASS"][["status", "severity", "check", "value", "expected", "detail"]] + if len(others): + display(others.reset_index(drop=True)) + + # -- output --------------------------------------------------------------- + + def finalize(self): + RESULTS_DIR.mkdir(parents=True, exist_ok=True) + self.metric("_suite", "_", elapsed_s=time.time() - self.t0) + for rows, fields, ext in ((self.checks, CHECK_FIELDS, "checks"), (self.metrics, METRIC_FIELDS, "metrics")): + with open(RESULTS_DIR / f"{self.name}.{ext}.csv", "w", newline="") as fh: + w = csv.DictWriter(fh, fieldnames=fields) + w.writeheader() + w.writerows(rows) + counts = {s: sum(c["status"] == s for c in self.checks) for s in ("PASS", "FAIL", "WARN", "KNOWN", "SKIP")} + print(f"{self.name}: " + ", ".join(f"{k} {v}" for k, v in counts.items())) + failed = [c for c in self.checks if c["status"] == "FAIL"] + if failed: + raise AssertionError(f"{len(failed)} check(s) failed in {self.name}: " + + "; ".join(c["check"] for c in failed[:20])) + return counts + +# ====================================================================================== +# 6. Persistence checks +# ====================================================================================== + +# Persistence checks shared by the notebooks of every model class: a model saved and +# loaded back (class-specific and generic ``pylibkriging.load``), pickled, or deep-copied +# must predict exactly as the original. + +def check_persistence(suite, name, k, Xn, predict): + """Record the persistence checks of model ``k`` (label ``name``) on the points ``Xn``. + + ``predict(model, Xn)`` returns (mean, stdev) as 1-D arrays; it carries the + prediction method of an approximate fit (Vecchia / Nystrom).""" + import pylibkriging as lk + m0, s0 = predict(k, Xn) + cls = type(k) + path = Path(tempfile.mkdtemp()) / f"{name.replace(' ', '_')}.json" + if not hasattr(k, "save"): + suite.check_skip(f"{name}: save + load", "no save() in this pylibkriging") + else: + with suite.check_runs(f"{name}: save + load"): + k.save(str(path)) + k1 = cls.load(str(path)) + m1, s1 = predict(k1, Xn) + suite.check_close(f"{name}: load(save(k)) predicts the same mean", m1, m0, rtol=1e-12, atol=1e-12) + suite.check_close(f"{name}: load(save(k)) predicts the same stdev", s1, s0, rtol=1e-10, atol=1e-12) + if hasattr(k, "theta"): + suite.check_close(f"{name}: same theta after load", np.ravel(k1.theta()), np.ravel(k.theta()), + rtol=1e-14, atol=0) + if hasattr(lk, "load") and (has("load(noise)") or "noise" not in name and "nugget" not in name): + with suite.check_runs(f"{name}: generic pylibkriging.load"): + k2 = lk.load(str(path)) + suite.check_true(f"{name}: generic load returns {cls.__name__}", type(k2).__name__ == cls.__name__, + detail=type(k2).__name__) + suite.check_close(f"{name}: generic load predicts the same", predict(k2, Xn)[0], m0, rtol=1e-12, + atol=1e-12) + if not has("pickle"): + suite.check_skip(f"{name}: pickle / deepcopy", "models not picklable in this pylibkriging") + return + with suite.check_runs(f"{name}: pickle round trip"): + k3 = pickle.loads(pickle.dumps(k)) + suite.check_close(f"{name}: pickle predicts the same", predict(k3, Xn)[0], m0, rtol=1e-12, atol=1e-12) + with suite.check_runs(f"{name}: deepcopy"): + k4 = copy.deepcopy(k) + suite.check_close(f"{name}: deepcopy predicts the same", predict(k4, Xn)[0], m0, rtol=1e-12, atol=1e-12) + +# ====================================================================================== +# 7. Shared datasets +# ====================================================================================== + +def read_data(name, stem): + """data/_.csv (no header) as a 2-D array, e.g. read_data("branin", "X").""" + return np.loadtxt(DATA_DIR / f"{name}_{stem}.csv", delimiter=",", ndmin=2) + + +# the notebooks use these names: design.dataset(...), metrics.q2(...), compat.has(...), ... +design = SimpleNamespace(lhs=lhs, sobol=sobol, uniform=uniform, dataset=dataset) +metrics = SimpleNamespace(rmse=rmse, q2=q2, nlpd=nlpd, coverage=coverage, crps=crps, all_metrics=all_metrics, + finite_diff_grad=finite_diff_grad) +compat = SimpleNamespace(version=version, has=has, optional=optional) +persistence = SimpleNamespace(check=check_persistence) diff --git a/tools/python/requirements.txt b/tools/python/requirements.txt new file mode 100644 index 0000000..3cbebe4 --- /dev/null +++ b/tools/python/requirements.txt @@ -0,0 +1,11 @@ +# What the Python notebooks need besides pylibkriging (tools/install/python.sh installs it): +# the dependencies of gallery.py, and the notebook runner. The comparison packages +# (scikit-learn, SMT, OpenTURNS, GPy, ...) are installed by the notebooks that use them. +numpy +scipy +pandas +matplotlib +jinja2 # DataFrame.style +nbclient +nbformat +ipykernel diff --git a/tools/r/checks.R b/tools/r/checks.R deleted file mode 100644 index a7f0994..0000000 --- a/tools/r/checks.R +++ /dev/null @@ -1,87 +0,0 @@ -# Same CSV outputs as lkgallery/checks.py (see there), for the R notebooks. -# Statuses: PASS, FAIL (error-level), WARN (warn-level), SKIP, KNOWN (listed in tools/known_issues.csv). -# Paths are relative to the repository root, where the notebooks are executed. -results_dir <- Sys.getenv("LKGALLERY_RESULTS", "results") -dir.create(results_dir, showWarnings = FALSE, recursive = TRUE) -known_file <- file.path("tools", "known_issues.csv") -if (!file.exists(known_file)) stop("run the notebooks from the repository root (", known_file, " not found)") -.known <- read.csv(known_file, stringsAsFactors = FALSE) - -new_suite <- function(name) { - s <- new.env() - s$name <- name - s$checks <- list() - s$metrics <- list() - s$t0 <- Sys.time() - s -} - -.fmt <- function(v) if (is.null(v) || length(v) == 0) "" else if (is.numeric(v)) format(signif(v[1], 6)) else as.character(v[1]) - -add_check <- function(s, check, ok, value = NULL, expected = NULL, detail = "", severity = "error") { - ok <- isTRUE(ok) - status <- if (ok) "PASS" else if (severity == "error") "FAIL" else "WARN" - k <- .known[.known$suite == s$name & .known$check == check, ] - if (!ok && nrow(k) > 0) { status <- "KNOWN"; detail <- paste(k$issue[1], detail, sep = " | ") } - if (!ok) message(sprintf("[%s] %s: value=%s expected %s %s", status, check, .fmt(value), .fmt(expected), detail)) - s$checks[[length(s$checks) + 1]] <- data.frame(suite = s$name, lang = "R", check = check, severity = severity, - status = status, value = .fmt(value), expected = .fmt(expected), detail = detail, stringsAsFactors = FALSE) - invisible(ok) -} - -check_close <- function(s, check, a, b, rtol = 1e-6, atol = 0, detail = "", severity = "error") { - a <- as.numeric(a); b <- as.numeric(b) - if (length(a) != length(b)) return(add_check(s, check, FALSE, paste("length", length(a)), paste("length", length(b)), detail, severity)) - gap <- abs(a - b) - ok <- all(is.finite(a)) && all(gap <= atol + rtol * abs(b)) - rel <- max(gap / pmax(abs(b), 1e-300)) - add_check(s, check, ok, rel, sprintf("rtol %g, atol %g", rtol, atol), detail, severity) -} - -check_less <- function(s, check, value, bound, detail = "", severity = "error") - add_check(s, check, is.finite(value) && value < bound, value, paste("<", bound), detail, severity) - -check_greater <- function(s, check, value, bound, detail = "", severity = "error") - add_check(s, check, is.finite(value) && value > bound, value, paste(">", bound), detail, severity) - -check_runs <- function(s, check, expr, severity = "error") { - r <- tryCatch({ force(expr); NULL }, error = function(e) conditionMessage(e)) - add_check(s, check, is.null(r), if (is.null(r)) "ok" else "error", "no error", if (is.null(r)) "" else substr(r, 1, 200), severity) -} - -add_metric <- function(s, case, package, ...) { - v <- list(...) - for (n in names(v)) s$metrics[[length(s$metrics) + 1]] <- data.frame(suite = s$name, lang = "R", case = case, - package = package, metric = n, value = .fmt(v[[n]]), stringsAsFactors = FALSE) -} - -summary_suite <- function(s) { - # end-of-notebook summary: metrics, count per status, every check that did not pass - if (length(s$metrics) > 0) { - me <- do.call(rbind, s$metrics) - me$value <- as.numeric(me$value) - print(reshape(me[, c("case", "package", "metric", "value")], idvar = c("case", "package"), - timevar = "metric", direction = "wide"), row.names = FALSE) - } - ch <- do.call(rbind, s$checks) - print(table(factor(ch$status, levels = c("PASS", "FAIL", "WARN", "KNOWN", "SKIP")))) - other <- ch[ch$status != "PASS", c("status", "severity", "check", "value", "expected", "detail")] - if (nrow(other) > 0) print(other, row.names = FALSE) - invisible(ch) -} - -finalize <- function(s) { - add_metric(s, "_suite", "_", elapsed_s = as.numeric(difftime(Sys.time(), s$t0, units = "secs"))) - ch <- do.call(rbind, s$checks) - me <- do.call(rbind, s$metrics) - write.csv(ch, file.path(results_dir, paste0(s$name, ".checks.csv")), row.names = FALSE) - write.csv(me, file.path(results_dir, paste0(s$name, ".metrics.csv")), row.names = FALSE) - print(table(factor(ch$status, levels = c("PASS", "FAIL", "WARN", "KNOWN", "SKIP")))) - failed <- ch$check[ch$status == "FAIL"] - if (length(failed) > 0) stop(sprintf("%d check(s) failed in %s: %s", length(failed), s$name, paste(head(failed, 20), collapse = "; "))) - invisible(ch) -} - -rmse <- function(y, m) sqrt(mean((y - m)^2)) -q2 <- function(y, m) 1 - sum((y - m)^2) / sum((y - mean(y))^2) -read_data <- function(name, stem) as.matrix(read.csv(file.path("data", paste0(name, "_", stem, ".csv")), header = FALSE)) diff --git a/tools/src/aggregations-large-n_NestedKriging_python.py b/tools/src/aggregations-large-n_NestedKriging_python.py index 22827f1..fad25de 100644 --- a/tools/src/aggregations-large-n_NestedKriging_python.py +++ b/tools/src/aggregations-large-n_NestedKriging_python.py @@ -36,13 +36,13 @@ import pandas as pd import pylibkriging as lk -import lkgallery -from lkgallery import TESTS, design, metrics, compat +import gallery +from gallery import TESTS, design, metrics, compat -suite = lkgallery.Suite("aggregations-large-n_NestedKriging_python") +suite = gallery.Suite("aggregations-large-n_NestedKriging_python") print("pylibkriging", compat.version()) if not compat.has("NestedKriging"): - suite.skip("NestedKriging", "not available in this pylibkriging") + suite.check_skip("NestedKriging", "not available in this pylibkriging") def pred(k, X, **kw): @@ -72,7 +72,7 @@ def timed(f): rows = [] for agg in ["NK", "rBCM"]: for g in [1, 2, 4, 8, 16, 32]: - with suite.runs(f"nested {agg}, {g} groups, fixed theta: fit + predict"): + with suite.check_runs(f"nested {agg}, {g} groups, fixed theta: fit + predict"): k, t = timed(lambda: lk.NestedKriging(y, X, "matern5_2", nb_groups=g, aggregation=agg, seed=1, optim="none", parameters=fixed)) m = pred(k, Xt)[0] @@ -81,11 +81,11 @@ def timed(f): q2=metrics.q2(yt, m))) conv = pd.DataFrame(rows) nk = conv[conv["aggregation"] == "NK"] - suite.less("nested NK, 1 group == exact predictor (RMS gap / sd)", nk["mean_gap"].iloc[0], 1e-6) - suite.less("nested NK, 32 groups: RMS gap to the exact predictor / sd", nk["mean_gap"].iloc[-1], 0.1, + suite.check_less("nested NK, 1 group == exact predictor (RMS gap / sd)", nk["mean_gap"].iloc[0], 1e-6) + suite.check_less("nested NK, 32 groups: RMS gap to the exact predictor / sd", nk["mean_gap"].iloc[-1], 0.1, severity="warn") rb = conv[conv["aggregation"] == "rBCM"] - suite.true("nested NK closer to the exact predictor than rBCM (2 to 32 groups)", + suite.check_true("nested NK closer to the exact predictor than rBCM (2 to 32 groups)", bool(np.all(nk["mean_gap"].to_numpy()[1:] <= rb["mean_gap"].to_numpy()[1:])), severity="warn") fig, axes = plt.subplots(1, 2, figsize=(10, 3.8)) for agg, g in conv.groupby("aggregation"): @@ -114,18 +114,18 @@ def timed(f): label = f"nested {agg} (20 groups)" for n in (n_mid, n_big): case = f"{fname}/n={n}" - with suite.runs(f"{case}/{label}: fit + predict"): + with suite.check_runs(f"{case}/{label}: fit + predict"): k, t = timed(lambda: lk.NestedKriging(y[:n], X[:n], "matern5_2", nb_groups=20, aggregation=agg, seed=1)) m, s = pred(k, Xt) - suite.finite(f"{case}/{label}: finite predictions", m) + suite.check_finite(f"{case}/{label}: finite predictions", m) res = metrics.all_metrics(yt, m, s) suite.metric(case, label, fit_s=t, **res) - suite.greater(f"{case}/{label}: Q2", res["q2"], FLOORS[fname] - 0.05) + suite.check_greater(f"{case}/{label}: Q2", res["q2"], FLOORS[fname] - 0.05) if n == n_mid: - suite.greater(f"{case}/{label}: Q2 - Q2(exact)", res["q2"] - res_e["q2"], -0.02, + suite.check_greater(f"{case}/{label}: Q2 - Q2(exact)", res["q2"] - res_e["q2"], -0.02, severity="warn") - suite.less(f"{case}/{label}: fit time / exact fit time", t / t_exact, 1.0, severity="warn") + suite.check_less(f"{case}/{label}: fit time / exact fit time", t / t_exact, 1.0, severity="warn") display(suite.table().round(4)) # %% [markdown] @@ -136,17 +136,17 @@ def timed(f): tf = TESTS["branin"] X, y, Xt, yt = design.dataset(tf, 200, n_test=500, seed=53) for agg in ["NK", "PoE", "gPoE", "BCM", "rBCM"]: - with suite.runs(f"nested {agg}: fit"): + with suite.check_runs(f"nested {agg}: fit"): k = lk.NestedKriging(y, X, "matern5_2", nb_groups=8, aggregation=agg, seed=2) m, s = (np.ravel(a) for a in k.predict(Xt, return_stdev=True)[:2]) - suite.true(f"nested {agg}: stdev >= 0", np.all(s >= 0)) + suite.check_true(f"nested {agg}: stdev >= 0", np.all(s >= 0)) suite.metric("branin/nested", agg, **metrics.all_metrics(yt, m, s)) if agg == "NK": mX = np.ravel(k.predict(X, return_stdev=True)[0]) - suite.less("nested NK: interpolates (max |m(X)-y| / sd(y))", np.max(np.abs(mX - y)) / np.std(y), + suite.check_less("nested NK: interpolates (max |m(X)-y| / sd(y))", np.max(np.abs(mX - y)) / np.std(y), 1e-4) display(suite.table().loc["branin/nested"].round(4)) - suite.raises("nested NK with a linear trend is rejected", + suite.check_raises("nested NK with a linear trend is rejected", lambda: lk.NestedKriging(y, X, "matern5_2", nb_groups=4, aggregation="NK", regmodel="linear"), severity="warn") @@ -160,8 +160,8 @@ def timed(f): # %% if compat.has("NestedKriging"): X, y, Xt, yt = design.dataset(TESTS["hartmann3"], 40, n_test=100, seed=61) - with suite.runs("NestedKriging: fit (persistence)"): - lkgallery.persistence.check(suite, "NestedKriging", lk.NestedKriging(y, X, "matern5_2", nb_groups=4, seed=1), Xt, pred) + with suite.check_runs("NestedKriging: fit (persistence)"): + gallery.persistence.check(suite, "NestedKriging", lk.NestedKriging(y, X, "matern5_2", nb_groups=4, seed=1), Xt, pred) # %% [markdown] # ## Summary diff --git a/tools/src/cross-binding-reference_all-classes_python.py b/tools/src/cross-binding-reference_all-classes_python.py index c83a08e..bc8187d 100644 --- a/tools/src/cross-binding-reference_all-classes_python.py +++ b/tools/src/cross-binding-reference_all-classes_python.py @@ -49,11 +49,11 @@ import pandas as pd import pylibkriging as lk -import lkgallery -from lkgallery import compat -from lkgallery.checks import DATA_DIR as DATA, RESULTS_DIR +import gallery +from gallery import compat +from gallery import DATA_DIR as DATA, RESULTS_DIR -suite = lkgallery.Suite("cross-binding-reference_all-classes_python") +suite = gallery.Suite("cross-binding-reference_all-classes_python") print("pylibkriging", compat.version()) DATASETS = ["branin", "hartmann3", "borehole"] rows = [] @@ -84,7 +84,7 @@ def load(name, stem): ll, grad, _ = kf.logLikelihoodFun(theta0.ravel(), True) put(name, "llfun", ll) put(name, "llfun_grad", grad) - suite.close(f"{name}: logLikelihood() == logLikelihoodFun(theta0)", kf.logLikelihood(), ll, rtol=1e-10) + suite.check_close(f"{name}: logLikelihood() == logLikelihoodFun(theta0)", kf.logLikelihood(), ll, rtol=1e-10) k = lk.Kriging(y, X, "matern5_2") put(name, "fit_theta", k.theta()) put(name, "fit_ll", k.logLikelihood()) @@ -115,7 +115,7 @@ def load(name, stem): put(name, f"{meth}_fixed_ll", ka.logLikelihood(method=meth)) put(name, f"{meth}_fixed_mean", ka.predict(Xt, return_stdev=False, method=meth)[0]) if not compat.has("method"): - suite.skip("vecchia / nystrom quantities", "pylibkriging without the method= argument") + suite.check_skip("vecchia / nystrom quantities", "pylibkriging without the method= argument") kw = lk.WarpKriging(y, X, ["kumaraswamy", "kumaraswamy"], "matern5_2") put(name, "warp_fit_mean", kw.predict(Xt, return_stdev=False)[0]) km = lk.MLPKriging(y, X, hidden_dims=[8], d_out=2, activation="tanh", kernel="matern5_2", optim="BFGS10") diff --git a/tools/src/cross-binding_all-classes_R.R b/tools/src/cross-binding_all-classes_R.R index 26a00ed..15b7532 100644 --- a/tools/src/cross-binding_all-classes_R.R +++ b/tools/src/cross-binding_all-classes_R.R @@ -13,8 +13,8 @@ # %% suppressPackageStartupMessages(library(rlibkriging)) -source("tools/r/checks.R") -s <- new_suite("cross-binding_all-classes_R") +source("tools/R/gallery.R") +s <- suite("cross-binding_all-classes_R") cat("rlibkriging", as.character(packageVersion("rlibkriging")), "\n") rows <- list() put <- function(dataset, quantity, values) { @@ -96,7 +96,7 @@ op <- par(mfrow = c(1, 3), mar = c(4, 4, 2, 1)) for (name in c("branin", "hartmann3", "borehole")) { yt <- as.numeric(read_data(name, "yt"))[1:50] m <- as.numeric(out$value[out$dataset == name & out$quantity == "fit_mean"]) - q <- q2(yt, m); add_metric(s, name, "rlibkriging default fit", q2 = q) + q <- q2(yt, m); metric(s, name, "rlibkriging default fit", q2 = q) plot(yt, m, xlab = "true", ylab = "predicted", main = sprintf("%s: Q2 = %.3f", name, q), pch = 16, cex = .7) abline(0, 1, lty = 2) } @@ -111,7 +111,7 @@ par(op) # All of them are also written to `results/cross-binding_all-classes_R.checks.csv` and `.metrics.csv`. # %% -summary_suite(s) +suite_summary(s) # %% -finalize(s) +suite_finalize(s) diff --git a/tools/src/cross-binding_all-classes_julia.jl b/tools/src/cross-binding_all-classes_julia.jl index 780b889..94a8d41 100644 --- a/tools/src/cross-binding_all-classes_julia.jl +++ b/tools/src/cross-binding_all-classes_julia.jl @@ -17,7 +17,7 @@ # %% import JLibKriging as LK -include("tools/julia/checks.jl") +include("tools/julia/gallery.jl") s = Suite("cross-binding_all-classes_julia") rows = Vector{Vector{String}}() record!(dataset, quantity, values) = @@ -108,7 +108,7 @@ end # All of them are also written to `results/cross-binding_all-classes_julia.checks.csv` and `.metrics.csv`. # %% -summary!(s) +suite_summary(s) # %% -finalize!(s) +suite_finalize!(s) diff --git a/tools/src/cross-binding_all-classes_octave.m b/tools/src/cross-binding_all-classes_octave.m index 4f5b96d..cc57e2d 100644 --- a/tools/src/cross-binding_all-classes_octave.m +++ b/tools/src/cross-binding_all-classes_octave.m @@ -13,8 +13,8 @@ % (`OCTAVE_PATH` or `addpath`). % %% -addpath(fullfile(pwd, "tools", "octave")); -s = lkb_suite("cross-binding_all-classes_octave"); +source("tools/octave/gallery.m"); +s = suite("cross-binding_all-classes_octave"); out = {}; function out = put(out, dataset, quantity, values) v = double(values(:)); @@ -42,7 +42,7 @@ [llf, grad] = kf.logLikelihoodFun(theta0', true, false); out = put(out, name, "llfun", llf); out = put(out, name, "llfun_grad", grad); - s = lkb_close(s, [name ": logLikelihood() == logLikelihoodFun(theta0)"], ll, llf, 1e-10); + s = check_close(s, [name ": logLikelihood() == logLikelihoodFun(theta0)"], ll, llf, 1e-10); k = Kriging(y, X, "matern5_2"); v = k.theta(); out = put(out, name, "fit_theta", v); llk = k.logLikelihood(); @@ -126,7 +126,7 @@ % All of them are also written to `results/cross-binding_all-classes_octave.checks.csv` and `.metrics.csv`. % %% -lkb_summary(s); +suite_summary(s); % %% -lkb_finalize(s); +suite_finalize(s); diff --git a/tools/src/deep-kernel-projected-branin_MLPKriging_python.py b/tools/src/deep-kernel-projected-branin_MLPKriging_python.py index d5f210b..79e7093 100644 --- a/tools/src/deep-kernel-projected-branin_MLPKriging_python.py +++ b/tools/src/deep-kernel-projected-branin_MLPKriging_python.py @@ -24,10 +24,10 @@ import numpy as np import pylibkriging as lk -import lkgallery -from lkgallery import TESTS, design, metrics, compat +import gallery +from gallery import TESTS, design, metrics, compat -suite = lkgallery.Suite("deep-kernel-projected-branin_MLPKriging_python") +suite = gallery.Suite("deep-kernel-projected-branin_MLPKriging_python") print("pylibkriging", compat.version()) @@ -55,26 +55,26 @@ def projected(X): for act in ["tanh", "selu"]: for optim in ["BFGS", "BFGS10"]: case = f"MLPKriging({act}, {optim})" - with suite.runs(f"{case}: fit"): + with suite.check_runs(f"{case}: fit"): t0 = time.time() km = lk.MLPKriging(y, X, hidden_dims=[16, 8], d_out=2, activation=act, kernel="matern5_2", optim=optim) fit_s = time.time() - t0 m, s = pred(km, Xt) - suite.finite(f"{case}: predictions finite", m) + suite.check_finite(f"{case}: predictions finite", m) res = metrics.all_metrics(yt, m, s) suite.metric("projected_branin_10d", f"MLPKriging {act} {optim}", fit_s=fit_s, ll=float(km.logLikelihood()), **res) if optim == "BFGS10": - suite.greater(f"{case}: Q2", res["q2"], 0.8) + suite.check_greater(f"{case}: Q2", res["q2"], 0.8) else: - suite.greater(f"{case}: Q2 - Q2(Kriging)", res["q2"] - res_k["q2"], -0.1, severity="warn") + suite.check_greater(f"{case}: Q2 - Q2(Kriging)", res["q2"] - res_k["q2"], -0.1, severity="warn") tab = suite.table().loc["projected_branin_10d"] tab["q2"].plot.barh(figsize=(6, 3)); plt.xlabel("Q² (test set)"); plt.xlim(min(0, tab["q2"].min()), 1) plt.title("Branin(Wx), x in [0,1]^10"); plt.show() display(tab.round(4)) else: - suite.skip("MLPKriging", "not available in this pylibkriging") + suite.check_skip("MLPKriging", "not available in this pylibkriging") # %% [markdown] # ## save / load, pickle, copy @@ -86,8 +86,8 @@ def projected(X): # %% if compat.has("MLPKriging"): X, y, Xt, yt = design.dataset(TESTS["hartmann3"], 40, n_test=100, seed=61) - with suite.runs("MLPKriging: fit (persistence)"): - lkgallery.persistence.check(suite, "MLPKriging", lk.MLPKriging(y, X, hidden_dims=[8], d_out=2, activation="tanh", + with suite.check_runs("MLPKriging: fit (persistence)"): + gallery.persistence.check(suite, "MLPKriging", lk.MLPKriging(y, X, hidden_dims=[8], d_out=2, activation="tanh", kernel="matern5_2"), Xt, pred) # %% [markdown] diff --git a/tools/src/high-dimension-normalize-screening_Kriging_python.py b/tools/src/high-dimension-normalize-screening_Kriging_python.py index 1720b54..22713f2 100644 --- a/tools/src/high-dimension-normalize-screening_Kriging_python.py +++ b/tools/src/high-dimension-normalize-screening_Kriging_python.py @@ -37,10 +37,10 @@ import pandas as pd import pylibkriging as lk -import lkgallery -from lkgallery import TESTS, design, metrics, compat +import gallery +from gallery import TESTS, design, metrics, compat -suite = lkgallery.Suite("high-dimension-normalize-screening_Kriging_python") +suite = gallery.Suite("high-dimension-normalize-screening_Kriging_python") print("pylibkriging", compat.version()) @@ -69,14 +69,14 @@ def pred(k, X): fit_s = time.time() - t0 res = metrics.all_metrics(yt, *pred(k, Ut)) suite.metric(fname, "unit cube", fit_s=fit_s, **res) - suite.greater(f"{fname}: Q2 (unit cube, matern5_2)", res["q2"], floor) + suite.check_greater(f"{fname}: Q2 (unit cube, matern5_2)", res["q2"], floor) # same problem in native units, rescaled internally Xn, Xtn = tf.scale(U), tf.scale(Ut) - with suite.runs(f"{fname}: fit on native inputs with normalize=True"): + with suite.check_runs(f"{fname}: fit on native inputs with normalize=True"): kn = lk.Kriging(y, Xn, "matern5_2", normalize=True) res_n = metrics.all_metrics(yt, *pred(kn, Xtn)) suite.metric(fname, "native + normalize", **res_n) - suite.greater(f"{fname}: Q2(normalize) - Q2(unit cube)", res_n["q2"] - res["q2"], -0.02) + suite.check_greater(f"{fname}: Q2(normalize) - Q2(unit cube)", res_n["q2"] - res["q2"], -0.02) tab = suite.table() tab.round(4) @@ -94,7 +94,7 @@ def pred(k, X): theta = np.ravel(k.theta()) inactive = [7, 15] # 0-based indices of x8, x16 active = [0, 3, 4, 11, 18, 19] # x1, x4, x5, x12, x19, x20 -suite.greater("welch: min theta(inactive) / max theta(active)", theta[inactive].min() / theta[active].max(), 1.0, +suite.check_greater("welch: min theta(inactive) / max theta(active)", theta[inactive].min() / theta[active].max(), 1.0, severity="warn") colors = ["tab:red" if i in active else "tab:gray" if i in inactive else "tab:blue" for i in range(20)] plt.figure(figsize=(9, 3)) diff --git a/tools/src/kernels-trends-interpolation_Kriging_python.py b/tools/src/kernels-trends-interpolation_Kriging_python.py index d0eee20..94a55c7 100644 --- a/tools/src/kernels-trends-interpolation_Kriging_python.py +++ b/tools/src/kernels-trends-interpolation_Kriging_python.py @@ -49,10 +49,10 @@ import matplotlib.pyplot as plt import pylibkriging as lk -import lkgallery -from lkgallery import TESTS, design, metrics, compat +import gallery +from gallery import TESTS, design, metrics, compat -suite = lkgallery.Suite("kernels-trends-interpolation_Kriging_python") +suite = gallery.Suite("kernels-trends-interpolation_Kriging_python") print("pylibkriging", compat.version()) KERNELS = ["gauss", "exp", "matern3_2", "matern5_2"] @@ -86,35 +86,35 @@ def predict(model, X, **kw): "quadratic": 1 + 2 * tf.d + tf.d * (tf.d - 1) // 2}[trend] case = f"{fname}/{kernel}/{trend}" if n_beta >= n - 1: - suite.skip(case, f"{n_beta} trend coefficients for n={n}") + suite.check_skip(case, f"{n_beta} trend coefficients for n={n}") continue t0 = time.time() try: k = lk.Kriging(y, X, kernel, regmodel=trend, optim="BFGS") except Exception as exc: # noqa: BLE001 - suite.true(f"{case}: fit", False, detail=str(exc)[:200]) + suite.check_true(f"{case}: fit", False, detail=str(exc)[:200]) continue fit_s = time.time() - t0 m, s, _ = predict(k, Xt) res = metrics.all_metrics(yt, m, s) suite.metric(case, "pylibkriging", fit_s=fit_s, **res) theta, sigma2 = np.asarray(k.theta()).ravel(), float(k.sigma2()) - suite.true(f"{case}: theta, sigma2 finite and > 0", + suite.check_true(f"{case}: theta, sigma2 finite and > 0", np.all(np.isfinite(theta)) and np.all(theta > 0) and np.isfinite(sigma2) and sigma2 > 0, detail=f"theta={theta}, sigma2={sigma2:.3g}") # interpolation at the design points (relative to the spread of y) md, sd, _ = predict(k, X) scale = np.std(y) sev = "warn" if kernel == "gauss" else "error" - suite.less(f"{case}: interpolates y (max |m(X)-y| / sd(y))", np.max(np.abs(md - y)) / scale, 1e-4, + suite.check_less(f"{case}: interpolates y (max |m(X)-y| / sd(y))", np.max(np.abs(md - y)) / scale, 1e-4, severity=sev) - suite.less(f"{case}: stdev(X) ~ 0 (max sd / sd(y))", np.max(sd) / scale, 1e-2, severity=sev) + suite.check_less(f"{case}: stdev(X) ~ 0 (max sd / sd(y))", np.max(sd) / scale, 1e-2, severity=sev) if kernel == "matern5_2" and trend == "constant": - suite.greater(f"{case}: Q2 (single-start BFGS)", res["q2"], q2_floor, severity="warn") + suite.check_greater(f"{case}: Q2 (single-start BFGS)", res["q2"], q2_floor, severity="warn") k10 = lk.Kriging(y, X, kernel, regmodel=trend, optim="BFGS10") res10 = metrics.all_metrics(yt, *predict(k10, Xt)[:2]) suite.metric(case, "pylibkriging BFGS10", **res10) - suite.greater(f"{case}: Q2 (BFGS10)", res10["q2"], q2_floor) + suite.check_greater(f"{case}: Q2 (BFGS10)", res10["q2"], q2_floor) # %% tab = suite.table().reset_index() @@ -131,10 +131,10 @@ def predict(model, X, **kw): # %% q = tab.set_index(["function", "kernel", "trend"])["q2"] -suite.true("branin: exp is the worst kernel (constant trend)", +suite.check_true("branin: exp is the worst kernel (constant trend)", q["branin", "exp", "constant"] <= min(q["branin", k, "constant"] for k in KERNELS), severity="warn") -suite.greater("lim_polynomial/matern5_2: Q2(quadratic) - Q2(constant)", +suite.check_greater("lim_polynomial/matern5_2: Q2(quadratic) - Q2(constant)", q["lim_polynomial", "matern5_2", "quadratic"] - q["lim_polynomial", "matern5_2", "constant"], -1e-3, severity="warn") @@ -147,16 +147,16 @@ def predict(model, X, **kw): k = lk.Kriging(y, X, "matern5_2") m, s, cov = predict(k, Xt, return_cov=True) cov = np.asarray(cov) -suite.close("predict: diag(cov) == stdev^2", np.diag(cov), s ** 2, rtol=1e-6, atol=1e-10) -suite.close("predict: cov symmetric", cov, cov.T, rtol=1e-10, atol=1e-12) -suite.greater("predict: cov PSD (min eigenvalue / max)", np.linalg.eigvalsh(cov).min() / np.abs(cov).max(), -1e-8) +suite.check_close("predict: diag(cov) == stdev^2", np.diag(cov), s ** 2, rtol=1e-6, atol=1e-10) +suite.check_close("predict: cov symmetric", cov, cov.T, rtol=1e-10, atol=1e-12) +suite.check_greater("predict: cov PSD (min eigenvalue / max)", np.linalg.eigvalsh(cov).min() / np.abs(cov).max(), -1e-8) m1 = np.array([predict(k, Xt[i:i + 1])[0][0] for i in range(len(Xt))]) s1 = np.array([predict(k, Xt[i:i + 1])[1][0] for i in range(len(Xt))]) -suite.close("predict: batch == point by point (mean)", m, m1, rtol=1e-8, atol=1e-10 * np.std(y)) -suite.close("predict: batch == point by point (stdev)", s, s1, rtol=1e-6, atol=1e-10 * np.std(y)) +suite.check_close("predict: batch == point by point (mean)", m, m1, rtol=1e-8, atol=1e-10 * np.std(y)) +suite.check_close("predict: batch == point by point (stdev)", s, s1, rtol=1e-6, atol=1e-10 * np.std(y)) # the mean is a function of the point only: duplicating/permuting the batch changes nothing perm = np.random.default_rng(0).permutation(len(Xt)) -suite.close("predict: invariant to the order of the batch", predict(k, Xt[perm])[0], m[perm], rtol=1e-10, atol=1e-10) +suite.check_close("predict: invariant to the order of the batch", predict(k, Xt[perm])[0], m[perm], rtol=1e-10, atol=1e-10) # %% [markdown] # ### Derivatives of the predictor @@ -165,7 +165,7 @@ def predict(model, X, **kw): # compared with central finite differences. # %% -with suite.runs("predict(return_deriv=True)"): +with suite.check_runs("predict(return_deriv=True)"): out = k.predict(Xt[:5], True, False, True) dm, ds = np.asarray(out[3]), np.asarray(out[4]) h = 1e-6 @@ -175,8 +175,8 @@ def predict(model, X, **kw): mp, sp, _ = predict(k, Xt[:5] + e) mm, sm, _ = predict(k, Xt[:5] - e) fd_m[:, j], fd_s[:, j] = (mp - mm) / (2 * h), (sp - sm) / (2 * h) - suite.close("d mean / dx == finite differences", dm, fd_m, rtol=1e-4, atol=1e-4 * np.abs(fd_m).max()) - suite.close("d stdev / dx == finite differences", ds, fd_s, rtol=1e-3, atol=1e-3 * np.abs(fd_s).max()) + suite.check_close("d mean / dx == finite differences", dm, fd_m, rtol=1e-4, atol=1e-4 * np.abs(fd_m).max()) + suite.check_close("d stdev / dx == finite differences", ds, fd_s, rtol=1e-3, atol=1e-3 * np.abs(fd_s).max()) # %% [markdown] # ## A look at the 1D fits diff --git a/tools/src/large-n_Kriging-vecchia-nystrom_python.py b/tools/src/large-n_Kriging-vecchia-nystrom_python.py index dc0fab0..d60b9e9 100644 --- a/tools/src/large-n_Kriging-vecchia-nystrom_python.py +++ b/tools/src/large-n_Kriging-vecchia-nystrom_python.py @@ -42,14 +42,14 @@ import pandas as pd import pylibkriging as lk -import lkgallery -from lkgallery import TESTS, design, metrics, compat +import gallery +from gallery import TESTS, design, metrics, compat -suite = lkgallery.Suite("large-n_Kriging-vecchia-nystrom_python") +suite = gallery.Suite("large-n_Kriging-vecchia-nystrom_python") print("pylibkriging", compat.version()) HAS_METHOD = compat.has("method") if not HAS_METHOD: - suite.skip("method=vecchia/nystrom", "pylibkriging without the method= argument") + suite.check_skip("method=vecchia/nystrom", "pylibkriging without the method= argument") def pred(k, X, **kw): @@ -72,17 +72,17 @@ def timed(f): ke = lk.Kriging(y, X, "matern5_2") fixed = {"theta": np.asarray(ke.theta()).reshape(1, -1), "is_theta_estim": False} kv = lk.Kriging(y, X, "matern5_2", optim="none", parameters=fixed, method=f"vecchia({len(y) - 1})") - suite.close("vecchia(n-1) LL == exact LL (same theta)", kv.logLikelihood(method="vecchia"), + suite.check_close("vecchia(n-1) LL == exact LL (same theta)", kv.logLikelihood(method="vecchia"), ke.logLikelihood(), rtol=1e-6) - suite.close("vecchia(n-1) predict == exact predict", pred(kv, Xt, method="vecchia")[0], pred(ke, Xt)[0], + suite.check_close("vecchia(n-1) predict == exact predict", pred(kv, Xt, method="vecchia")[0], pred(ke, Xt)[0], rtol=1e-5, atol=1e-6 * np.std(y), severity="warn") kn = lk.Kriging(y, X, "matern5_2", optim="none", parameters=fixed, method=f"nystrom({len(y)})") - suite.close("nystrom(n) LL == exact LL (same theta)", kn.logLikelihood(method="nystrom"), ke.logLikelihood(), + suite.check_close("nystrom(n) LL == exact LL (same theta)", kn.logLikelihood(method="nystrom"), ke.logLikelihood(), rtol=1e-3, severity="warn") # the method must be explicit after an approximate fit - suite.raises("nystrom fit: predict(method='exact') raises", lambda: kn.predict(Xt, return_stdev=True, + suite.check_raises("nystrom fit: predict(method='exact') raises", lambda: kn.predict(Xt, return_stdev=True, method="exact")) - suite.true("method() reports the fit method", kv.method().startswith("vecchia") and + suite.check_true("method() reports the fit method", kv.method().startswith("vecchia") and kn.method().startswith("nystrom"), detail=f"{kv.method()}, {kn.method()}") # %% [markdown] @@ -105,7 +105,7 @@ def timed(f): rows = [] for meth, sizes in [("vecchia", [2, 5, 10, 20, 40, 80, 160, n - 1]), ("nystrom", [10, 25, 50, 100, 200, n])]: for size in sizes: - with suite.runs(f"{meth}({size}) at fixed theta: fit + predict"): + with suite.check_runs(f"{meth}({size}) at fixed theta: fit + predict"): ka = lk.Kriging(y, X, "matern5_2", optim="none", parameters=fixed, method=f"{meth}({size})") rows.append(dict(method=meth, size=size, ll_gap=abs(ka.logLikelihood(method=meth) - ll_e) / abs(ll_e), @@ -114,8 +114,8 @@ def timed(f): for meth, g in conv.groupby("method"): suite.metric(f"hartmann3/n={n}/fixed theta", meth, **{f"ll_gap_{s}": v for s, v in zip(g["size"], g["ll_gap"])}) for col, lim in [("ll_gap", 1e-6), ("mean_gap", 1e-2)]: - suite.less(f"{meth}: {col} at the limit size ({g['size'].iloc[-1]})", g[col].iloc[-1], lim) - suite.true(f"{meth}: {col} decreases with the size", bool(np.all(np.diff(g[col].to_numpy()) < 0)), + suite.check_less(f"{meth}: {col} at the limit size ({g['size'].iloc[-1]})", g[col].iloc[-1], lim) + suite.check_true(f"{meth}: {col} decreases with the size", bool(np.all(np.diff(g[col].to_numpy()) < 0)), detail=", ".join(f"{v:.2g}" for v in g[col]), severity="warn") fig, axes = plt.subplots(1, 2, figsize=(10, 3.8)) for ax, col, label in [(axes[0], "ll_gap", "|LL − LL(exact)| / |LL(exact)|"), @@ -139,7 +139,7 @@ def timed(f): ke, t_exact = timed(lambda: lk.Kriging(ym, Xm, "matern5_2")) res_e = metrics.all_metrics(yt, *pred(ke, Xt)) suite.metric(f"{fname}/n={n_mid}", "exact", fit_s=t_exact, **res_e) - suite.greater(f"{fname}/n={n_mid}: Q2 exact", res_e["q2"], FLOORS[fname]) + suite.check_greater(f"{fname}/n={n_mid}: Q2 exact", res_e["q2"], FLOORS[fname]) variants = {} if HAS_METHOD: variants.update({ @@ -154,18 +154,18 @@ def sod(Xa, ya): for label, make in variants.items(): for n in (n_mid, n_big): case = f"{fname}/n={n}" - with suite.runs(f"{case}/{label}: fit + predict"): + with suite.check_runs(f"{case}/{label}: fit + predict"): (k, meth), t = timed(lambda: make(X[:n], y[:n])) m, s = pred(k, Xt, method=meth) if meth else ( lambda o: (np.ravel(o[0]), np.ravel(o[1])))(k.predict(Xt, return_stdev=True)) - suite.finite(f"{case}/{label}: finite predictions", m) + suite.check_finite(f"{case}/{label}: finite predictions", m) res = metrics.all_metrics(yt, m, s) suite.metric(case, label, fit_s=t, **res) - suite.greater(f"{case}/{label}: Q2", res["q2"], FLOORS[fname] - 0.05) + suite.check_greater(f"{case}/{label}: Q2", res["q2"], FLOORS[fname] - 0.05) if n == n_mid: - suite.greater(f"{case}/{label}: Q2 - Q2(exact)", res["q2"] - res_e["q2"], -0.02, + suite.check_greater(f"{case}/{label}: Q2 - Q2(exact)", res["q2"] - res_e["q2"], -0.02, severity="warn") - suite.less(f"{case}/{label}: fit time / exact fit time", t / t_exact, 1.0, severity="warn") + suite.check_less(f"{case}/{label}: fit time / exact fit time", t / t_exact, 1.0, severity="warn") suite.table().round(4) # %% [markdown] diff --git a/tools/src/noisy-data_Kriging-nugget-noise_python.py b/tools/src/noisy-data_Kriging-nugget-noise_python.py index 93978c0..54419f3 100644 --- a/tools/src/noisy-data_Kriging-nugget-noise_python.py +++ b/tools/src/noisy-data_Kriging-nugget-noise_python.py @@ -39,10 +39,10 @@ import matplotlib.pyplot as plt import pylibkriging as lk -import lkgallery -from lkgallery import TESTS, design, metrics, compat +import gallery +from gallery import TESTS, design, metrics, compat -suite = lkgallery.Suite("noisy-data_Kriging-nugget-noise_python") +suite = gallery.Suite("noisy-data_Kriging-nugget-noise_python") print("pylibkriging", compat.version()) @@ -62,26 +62,26 @@ def pred(k, X): m, s = pred(k, Xt) res = metrics.all_metrics(yt, m, s) suite.metric(fname, "nugget", nugget=nugget, true_var=sd ** 2, **res) - suite.between(f"{fname}: nugget / true noise variance", nugget / sd ** 2, 1 / 3, 3) + suite.check_between(f"{fname}: nugget / true noise variance", nugget / sd ** 2, 1 / 3, 3) k1 = lk.Kriging(y, X, "matern5_2", noise="nugget") # default single start suite.metric(fname, "nugget, single start", nugget=float(np.ravel(k1.nugget())[0]), ll=float(k1.logLikelihood()), **metrics.all_metrics(yt, *pred(k1, Xt))) - suite.greater(f"{fname}: LL(single start) - LL(10 restarts)", k1.logLikelihood() - k.logLikelihood(), -1, + suite.check_greater(f"{fname}: LL(single start) - LL(10 restarts)", k1.logLikelihood() - k.logLikelihood(), -1, severity="warn") - suite.greater(f"{fname}: Q2 vs noise-free truth", res["q2"], q2_floor) + suite.check_greater(f"{fname}: Q2 vs noise-free truth", res["q2"], q2_floor) mX, _ = pred(k, X) mX_near, _ = pred(k, np.clip(X + 1e-7, 0, 1)) suite.metric(fname, "nugget", rms_resid_at_X=metrics.rmse(y, mX) / sd, rms_resid_near_X=metrics.rmse(y, mX_near) / sd) - suite.greater(f"{fname}: smooths the noisy data next to the design points (rms residual / sd)", + suite.check_greater(f"{fname}: smooths the noisy data next to the design points (rms residual / sd)", metrics.rmse(y, mX_near) / sd, 0.3) - suite.between(f"{fname}: coverage of the 95% intervals (noise-free truth)", res["coverage95"], 0.80, 1.0, + suite.check_between(f"{fname}: coverage of the 95% intervals (noise-free truth)", res["coverage95"], 0.80, 1.0, severity="warn") # the noise-free model on the same noisy data interpolates, and generalizes worse k0 = lk.Kriging(y, X, "matern5_2") m0, s0 = pred(k0, Xt) suite.metric(fname, "noise-free fit", **metrics.all_metrics(yt, m0, s0)) - suite.less(f"{fname}: RMSE(nugget) < RMSE(interpolating fit)", res["rmse"], metrics.rmse(yt, m0), + suite.check_less(f"{fname}: RMSE(nugget) < RMSE(interpolating fit)", res["rmse"], metrics.rmse(yt, m0), severity="warn") suite.table() @@ -95,13 +95,13 @@ def pred(k, X): theta = np.asarray(k0.theta()).reshape(1, -1) fixed = {"theta": theta, "is_theta_estim": False, "sigma2": float(k0.sigma2()), "is_sigma2_estim": False} kf = lk.Kriging(y, X, "matern5_2", optim="none", parameters=fixed) -with suite.runs("noise=zeros, theta and sigma2 fixed: fit"): +with suite.check_runs("noise=zeros, theta and sigma2 fixed: fit"): kz = lk.Kriging(y, X, "matern5_2", noise=np.zeros(len(y)), optim="none", parameters=fixed) - suite.close("noise=zeros: same mean as the noise-free model", pred(kz, Xt)[0], pred(kf, Xt)[0], rtol=1e-6, + suite.check_close("noise=zeros: same mean as the noise-free model", pred(kz, Xt)[0], pred(kf, Xt)[0], rtol=1e-6, atol=1e-8 * np.std(y)) - suite.close("noise=zeros: same stdev as the noise-free model", pred(kz, Xt)[1], pred(kf, Xt)[1], rtol=1e-4, + suite.check_close("noise=zeros: same stdev as the noise-free model", pred(kz, Xt)[1], pred(kf, Xt)[1], rtol=1e-4, atol=1e-6 * np.std(y)) -with suite.runs("noise=zeros, sigma2 estimated: fit", severity="warn"): +with suite.check_runs("noise=zeros, sigma2 estimated: fit", severity="warn"): lk.Kriging(y, X, "matern5_2", noise=np.zeros(len(y)), optim="none", parameters={"theta": theta, "is_theta_estim": False}) @@ -120,10 +120,10 @@ def pred(k, X): res_nug = metrics.all_metrics(yt, *pred(k_nug, Xt)) suite.metric("gramacy_lee_2012/hetero", "known variances", **res_known) suite.metric("gramacy_lee_2012/hetero", "nugget", **res_nug) -suite.finite("hetero: predictions finite", pred(k_known, Xt)[0]) -suite.less("hetero: NLPD(known variances) <= NLPD(nugget)", res_known["nlpd"] - res_nug["nlpd"], 0.05, +suite.check_finite("hetero: predictions finite", pred(k_known, Xt)[0]) +suite.check_less("hetero: NLPD(known variances) <= NLPD(nugget)", res_known["nlpd"] - res_nug["nlpd"], 0.05, severity="warn") -suite.close("noise(): returns the given variances", np.ravel(k_known.noise()), sd_x ** 2, rtol=1e-12) +suite.check_close("noise(): returns the given variances", np.ravel(k_known.noise()), sd_x ** 2, rtol=1e-12) u = np.linspace(0, 1, 400)[:, None] fig, ax = plt.subplots(figsize=(10, 4)) @@ -154,9 +154,9 @@ def pred(k, X): "sigma2": float(k_rep.sigma2()), "is_sigma2_estim": False} k_rep = lk.Kriging(yr, Xr, "matern5_2", noise=np.full(len(yr), sd ** 2), optim="none", parameters=fixed) k_avg = lk.Kriging(ybar, X1, "matern5_2", noise=np.full(len(ybar), sd ** 2 / r), optim="none", parameters=fixed) -suite.close("replicates == averages with variance/r (mean)", pred(k_rep, Xt)[0], pred(k_avg, Xt)[0], +suite.check_close("replicates == averages with variance/r (mean)", pred(k_rep, Xt)[0], pred(k_avg, Xt)[0], rtol=1e-6, atol=1e-6 * np.std(yr)) -suite.close("replicates == averages with variance/r (stdev)", pred(k_rep, Xt)[1], pred(k_avg, Xt)[1], +suite.check_close("replicates == averages with variance/r (stdev)", pred(k_rep, Xt)[1], pred(k_avg, Xt)[1], rtol=1e-4, atol=1e-6 * np.std(yr)) # %% [markdown] diff --git a/tools/src/objectives-LL-LOO-LMP-gradients_Kriging_python.py b/tools/src/objectives-LL-LOO-LMP-gradients_Kriging_python.py index 6c3760b..50cf1e8 100644 --- a/tools/src/objectives-LL-LOO-LMP-gradients_Kriging_python.py +++ b/tools/src/objectives-LL-LOO-LMP-gradients_Kriging_python.py @@ -37,10 +37,10 @@ import pandas as pd import pylibkriging as lk -import lkgallery -from lkgallery import TESTS, design, metrics, compat +import gallery +from gallery import TESTS, design, metrics, compat -suite = lkgallery.Suite("objectives-LL-LOO-LMP-gradients_Kriging_python") +suite = gallery.Suite("objectives-LL-LOO-LMP-gradients_Kriging_python") print("pylibkriging", compat.version()) CASES = {"branin": 25, "hartmann3": 40, "ishigami": 60, "friedman": 50} KERNELS = ["gauss", "exp", "matern3_2", "matern5_2"] @@ -76,7 +76,7 @@ def objective_fun(k, objective): err = np.max(np.abs(grad - fd)) / max(np.max(np.abs(fd)), 1e-8) worst = max(worst, err) rows.append((fname, kernel, objective, worst)) - suite.less(f"{fname}/{kernel}/{objective}: gradient vs finite diff (rel. err)", worst, 1e-3) + suite.check_less(f"{fname}/{kernel}/{objective}: gradient vs finite diff (rel. err)", worst, 1e-3) pd.DataFrame(rows, columns=["function", "kernel", "objective", "max rel. error"]).pivot_table( index=["function", "kernel"], columns="objective", values="max rel. error").style.format("{:.1e}") @@ -102,7 +102,7 @@ def objective_fun(k, objective): tol = 1e-4 * abs(best) + (0.0 if objective == "LOO" else 1e-2) if SIGN[objective] * f(t, False)[0] > best + tol and not at_bound: improved.append(f"theta{j + 1}*{fac}") - suite.true(f"{fname}/{objective}: theta_hat not improved by +/-10% moves", not improved, + suite.check_true(f"{fname}/{objective}: theta_hat not improved by +/-10% moves", not improved, detail=f"theta={np.round(theta, 4)}; improving moves: {improved}") m, s, *_ = k.predict(Xt, return_stdev=True) suite.metric(f"{fname}/{objective}", "pylibkriging", **metrics.all_metrics(yt, m, s)) @@ -132,7 +132,7 @@ def objective_fun(k, objective): ax.semilogx(ts, vals); ax.axvline(1, color="k", ls="--", lw=.8) ax.set_title(f"{objective}, θ̂ = {np.round(theta, 3)}"); ax.set_xlabel("t (θ = t · θ̂)") i = int(np.argmax(vals) if SIGN[objective] > 0 else np.argmin(vals)) - suite.between(f"branin/{objective}: optimum of the profile along t * theta_hat (t)", ts[i], 0.8, 1.25, + suite.check_between(f"branin/{objective}: optimum of the profile along t * theta_hat (t)", ts[i], 0.8, 1.25, severity="warn") axes[0].set_ylabel("criterion"); plt.tight_layout(); plt.show() @@ -145,7 +145,7 @@ def objective_fun(k, objective): for objective, getter in [("LL", "logLikelihood"), ("LOO", "leaveOneOut"), ("LMP", "logMargPost")]: k = lk.Kriging(y, X, "matern5_2", objective=objective) v_fun = objective_fun(k, objective)(np.asarray(k.theta()).ravel(), False)[0] - suite.close(f"{getter}() == {objective} function at theta()", getattr(k, getter)(), v_fun, rtol=1e-8) + suite.check_close(f"{getter}() == {objective} function at theta()", getattr(k, getter)(), v_fun, rtol=1e-8) # %% [markdown] # ## Closed-form leave-one-out vs explicit refits @@ -171,14 +171,14 @@ def objective_fun(k, objective): "sigma2": sigma2, "is_sigma2_estim": False}) mi, si, *_ = ki.predict(X[i:i + 1], return_stdev=True) m_ref[i], s_ref[i] = np.ravel(mi)[0], np.ravel(si)[0] - suite.close(f"{fname}/{trend}: leaveOneOutVec mean == explicit refits", m_loo, m_ref, + suite.check_close(f"{fname}/{trend}: leaveOneOutVec mean == explicit refits", m_loo, m_ref, rtol=1e-6, atol=1e-8 * np.std(y)) - suite.close(f"{fname}/{trend}: leaveOneOutVec stdev == explicit refits", s_loo, s_ref, + suite.check_close(f"{fname}/{trend}: leaveOneOutVec stdev == explicit refits", s_loo, s_ref, rtol=1e-4, atol=1e-8 * np.std(y), severity="warn") # leaveOneOut() is the mean squared LOO error at the fitted theta k_loo = lk.Kriging(y, X, "matern5_2", regmodel=trend, objective="LOO") m2, _ = k_loo.leaveOneOutVec(np.asarray(k_loo.theta()).ravel()) - suite.close(f"{fname}/{trend}: leaveOneOut() == mean squared LOO error", k_loo.leaveOneOut(), + suite.check_close(f"{fname}/{trend}: leaveOneOut() == mean squared LOO error", k_loo.leaveOneOut(), np.mean((np.ravel(m2) - y) ** 2), rtol=1e-6, severity="warn") # %% [markdown] @@ -197,8 +197,8 @@ def objective_fun(k, objective): suite.metric(fname, "BFGS", ll=ll1) suite.metric(fname, "BFGS10", ll=ll10) suite.metric(fname, "BFGS20", ll=ll20) - suite.greater(f"{fname}: LL(BFGS10) - LL(BFGS)", ll10 - ll1, -1e-3 * abs(ll1), severity="warn") - suite.less(f"{fname}: LL(BFGS20) - LL(BFGS) (default fit far from the best optimum)", ll20 - ll1, 1.0, + suite.check_greater(f"{fname}: LL(BFGS10) - LL(BFGS)", ll10 - ll1, -1e-3 * abs(ll1), severity="warn") + suite.check_less(f"{fname}: LL(BFGS20) - LL(BFGS) (default fit far from the best optimum)", ll20 - ll1, 1.0, severity="warn") suite.table().loc[list(CASES)].unstack("package") diff --git a/tools/src/robustness-invalid-inputs_Kriging_python.py b/tools/src/robustness-invalid-inputs_Kriging_python.py index e4d4355..50102ff 100644 --- a/tools/src/robustness-invalid-inputs_Kriging_python.py +++ b/tools/src/robustness-invalid-inputs_Kriging_python.py @@ -32,10 +32,10 @@ import numpy as np import pylibkriging as lk -import lkgallery -from lkgallery import TESTS, design, compat +import gallery +from gallery import TESTS, design, compat -suite = lkgallery.Suite("robustness-invalid-inputs_Kriging_python") +suite = gallery.Suite("robustness-invalid-inputs_Kriging_python") print("pylibkriging", compat.version()) tf = TESTS["branin"] X, y, Xt, yt = design.dataset(tf, 20, n_test=200, seed=71) @@ -51,10 +51,10 @@ def finite_model(check, make, Xn=Xt, severity="error"): try: k = make() except Exception as exc: # noqa: BLE001 - suite.true(f"{check}: rejected or finite", True, detail=f"rejected: {type(exc).__name__}") + suite.check_true(f"{check}: rejected or finite", True, detail=f"rejected: {type(exc).__name__}") return None m, s = pred(k, Xn) - suite.true(f"{check}: rejected or finite", np.all(np.isfinite(m)) and np.all(np.isfinite(s)), + suite.check_true(f"{check}: rejected or finite", np.all(np.isfinite(m)) and np.all(np.isfinite(s)), detail=f"non-finite: {np.sum(~np.isfinite(m))} mean, {np.sum(~np.isfinite(s))} stdev", severity=severity) return k @@ -66,22 +66,22 @@ def finite_model(check, make, Xn=Xt, severity="error"): Xn = X.copy(); Xn[3, 1] = np.nan yn = y.copy(); yn[5] = np.nan yi = y.copy(); yi[5] = np.inf -suite.raises("NaN in X is rejected", lambda: lk.Kriging(y, Xn, "matern5_2")) -suite.raises("NaN in y is rejected", lambda: lk.Kriging(yn, X, "matern5_2")) -suite.raises("Inf in y is rejected", lambda: lk.Kriging(yi, X, "matern5_2")) -suite.raises("len(y) != nrow(X) is rejected", lambda: lk.Kriging(y[:-1], X, "matern5_2")) -suite.raises("unknown kernel is rejected", lambda: lk.Kriging(y, X, "matern7_2")) -suite.raises("unknown trend is rejected", lambda: lk.Kriging(y, X, "matern5_2", regmodel="cubic")) -suite.raises("unknown objective is rejected", lambda: lk.Kriging(y, X, "matern5_2", objective="XYZ")) -suite.raises("noise vector of the wrong length is rejected", +suite.check_raises("NaN in X is rejected", lambda: lk.Kriging(y, Xn, "matern5_2")) +suite.check_raises("NaN in y is rejected", lambda: lk.Kriging(yn, X, "matern5_2")) +suite.check_raises("Inf in y is rejected", lambda: lk.Kriging(yi, X, "matern5_2")) +suite.check_raises("len(y) != nrow(X) is rejected", lambda: lk.Kriging(y[:-1], X, "matern5_2")) +suite.check_raises("unknown kernel is rejected", lambda: lk.Kriging(y, X, "matern7_2")) +suite.check_raises("unknown trend is rejected", lambda: lk.Kriging(y, X, "matern5_2", regmodel="cubic")) +suite.check_raises("unknown objective is rejected", lambda: lk.Kriging(y, X, "matern5_2", objective="XYZ")) +suite.check_raises("noise vector of the wrong length is rejected", lambda: lk.Kriging(y, X, "matern5_2", noise=np.ones(len(y) - 1))) -suite.raises("negative noise variance is rejected", +suite.check_raises("negative noise variance is rejected", lambda: lk.Kriging(y, X, "matern5_2", noise=-np.ones(len(y))), severity="warn") k = lk.Kriging(y, X, "matern5_2") -suite.raises("predict with the wrong number of columns is rejected", lambda: k.predict(np.ones((3, 3)), True)) +suite.check_raises("predict with the wrong number of columns is rejected", lambda: k.predict(np.ones((3, 3)), True)) if compat.has("WarpKriging"): - suite.raises("WarpKriging with 1 spec for 2 columns is rejected", lambda: lk.WarpKriging(y, X, ["none"], "gauss")) - suite.raises("WarpKriging with an unknown spec is rejected", + suite.check_raises("WarpKriging with 1 spec for 2 columns is rejected", lambda: lk.WarpKriging(y, X, ["none"], "gauss")) + suite.check_raises("WarpKriging with an unknown spec is rejected", lambda: lk.WarpKriging(y, X, ["none", "spline"], "gauss")) # %% [markdown] @@ -92,10 +92,10 @@ def finite_model(check, make, Xn=Xt, severity="error"): finite_model("duplicated points, noise-free", lambda: lk.Kriging(yd, Xd, "matern5_2")) if compat.has("noise"): k = finite_model("duplicated points, nugget", lambda: lk.Kriging(yd, Xd, "matern5_2", noise="nugget")) - suite.true("duplicated points, nugget: a model is built", k is not None) + suite.check_true("duplicated points, nugget: a model is built", k is not None) kc = finite_model("constant y", lambda: lk.Kriging(np.full(20, 3.7), X, "matern5_2")) if kc is not None: - suite.close("constant y: predicts the constant", pred(kc, Xt)[0], np.full(len(Xt), 3.7), rtol=1e-6, atol=1e-6) + suite.check_close("constant y: predicts the constant", pred(kc, Xt)[0], np.full(len(Xt), 3.7), rtol=1e-6, atol=1e-6) for n in (2, 3): finite_model(f"n = {n}", lambda: lk.Kriging(y[:n], X[:n], "matern5_2")) Xclose = np.vstack([X[:10], X[:10] + 1e-8]) @@ -113,21 +113,21 @@ def finite_model(check, make, Xn=Xt, severity="error"): k0 = lk.Kriging(y, X, "matern5_2", normalize=True) m0, s0 = pred(k0, Xt) for scale in (1e-6, 1e6): - with suite.runs(f"X * {scale:g} with normalize=True: fit"): + with suite.check_runs(f"X * {scale:g} with normalize=True: fit"): ks = lk.Kriging(y, X * scale, "matern5_2", normalize=True) - suite.close(f"X * {scale:g}, normalize=True: same mean as on [0,1]", pred(ks, Xt * scale)[0], m0, + suite.check_close(f"X * {scale:g}, normalize=True: same mean as on [0,1]", pred(ks, Xt * scale)[0], m0, rtol=1e-3, atol=1e-3 * np.std(y)) for a, b in [(1e8, 0.0), (1.0, 1e6), (1e-8, 0.0)]: - with suite.runs(f"y * {a:g} + {b:g}: fit"): + with suite.check_runs(f"y * {a:g} + {b:g}: fit"): ka = lk.Kriging(a * y + b, X, "matern5_2", normalize=True) ma, sa = pred(ka, Xt) - suite.close(f"y * {a:g} + {b:g}: mean equivariant", ma, a * m0 + b, rtol=1e-3, atol=1e-3 * a * np.std(y)) - suite.close(f"y * {a:g} + {b:g}: stdev scales by |a|", sa, abs(a) * s0, rtol=1e-2, + suite.check_close(f"y * {a:g} + {b:g}: mean equivariant", ma, a * m0 + b, rtol=1e-3, atol=1e-3 * a * np.std(y)) + suite.check_close(f"y * {a:g} + {b:g}: stdev scales by |a|", sa, abs(a) * s0, rtol=1e-2, atol=1e-3 * a * np.std(y)) # the same without normalize: the optimizer starts from a scale-dependent point, warn-level only for a, b in [(1e6, 0.0), (1.0, 1e6)]: ka = lk.Kriging(a * y + b, X, "matern5_2") - suite.close(f"y * {a:g} + {b:g} (normalize=False): mean equivariant", pred(ka, Xt)[0], + suite.check_close(f"y * {a:g} + {b:g} (normalize=False): mean equivariant", pred(ka, Xt)[0], a * pred(lk.Kriging(y, X, "matern5_2"), Xt)[0] + b, rtol=1e-3, atol=1e-3 * a * np.std(y), severity="warn") @@ -138,11 +138,11 @@ def finite_model(check, make, Xn=Xt, severity="error"): k = lk.Kriging(y, X, "matern5_2") far = np.array([[50.0, 50.0], [-30.0, 80.0]]) m, s = pred(k, far) -suite.close("far away: mean -> beta (constant trend)", m, np.full(2, np.ravel(k.beta())[0]), rtol=1e-6, +suite.check_close("far away: mean -> beta (constant trend)", m, np.full(2, np.ravel(k.beta())[0]), rtol=1e-6, atol=1e-8) # universal kriging: sigma2 * (1 + 1 / (1' R^-1 1)) with the trend uncertainty, slightly above sigma2 -suite.between("far away: stdev / sqrt(sigma2)", np.max(s) / np.sqrt(k.sigma2()), 1.0 - 1e-6, 1.5) -suite.close("far away: same stdev at both points", s[0], s[1], rtol=1e-6) +suite.check_between("far away: stdev / sqrt(sigma2)", np.max(s) / np.sqrt(k.sigma2()), 1.0 - 1e-6, 1.5) +suite.check_close("far away: same stdev at both points", s[0], s[1], rtol=1e-6) # %% [markdown] # ## Many prediction points @@ -165,7 +165,7 @@ def finite_model(check, make, Xn=Xt, severity="error"): times[q] = best suite.metric(f"predict_q={q}", "pylibkriging", seconds=best) expo = np.log2(times[8000] / times[4000]) -suite.less("predict(stdev) time growth exponent in q (8000 vs 4000)", expo, 1.5, +suite.check_less("predict(stdev) time growth exponent in q (8000 vs 4000)", expo, 1.5, detail=", ".join(f"q={q}: {t:.3f}s" for q, t in times.items())) qs = np.array(list(times)); ts = np.array(list(times.values())) plt.figure(figsize=(5, 3.6)) diff --git a/tools/src/save-load-pickle-copy-accessors_Kriging_python.py b/tools/src/save-load-pickle-copy-accessors_Kriging_python.py index 7895da3..b1c1a22 100644 --- a/tools/src/save-load-pickle-copy-accessors_Kriging_python.py +++ b/tools/src/save-load-pickle-copy-accessors_Kriging_python.py @@ -31,10 +31,10 @@ import numpy as np import pylibkriging as lk -import lkgallery -from lkgallery import TESTS, design, compat +import gallery +from gallery import TESTS, design, compat -suite = lkgallery.Suite("save-load-pickle-copy-accessors_Kriging_python") +suite = gallery.Suite("save-load-pickle-copy-accessors_Kriging_python") print("pylibkriging", compat.version()) tf = TESTS["hartmann3"] @@ -70,8 +70,8 @@ def predict_any(k, Xn): # %% for name, make in MODELS.items(): - with suite.runs(f"{name}: fit"): - lkgallery.persistence.check(suite, name, make(), Xt, predict_any) + with suite.check_runs(f"{name}: fit"): + gallery.persistence.check(suite, name, make(), Xt, predict_any) # %% [markdown] # A copy is independent: updating it leaves the original model unchanged. @@ -81,8 +81,8 @@ def predict_any(k, Xn): m_before = mean_sd(k, Xt)[0] kc = k.copy() if hasattr(k, "copy") else copy.deepcopy(k) kc.update(y[30:], X[30:], refit=False) -suite.close("update on a copy leaves the original unchanged", mean_sd(k, Xt)[0], m_before, rtol=0, atol=0) -suite.true("the updated copy did change", not np.allclose(mean_sd(kc, Xt)[0], m_before)) +suite.check_close("update on a copy leaves the original unchanged", mean_sd(k, Xt)[0], m_before, rtol=0, atol=0) +suite.check_true("the updated copy did change", not np.allclose(mean_sd(kc, Xt)[0], m_before)) # %% [markdown] # ## Accessors mirror the constructor @@ -91,19 +91,19 @@ def predict_any(k, Xn): k = lk.Kriging(y, X, "matern3_2", regmodel="linear", normalize=False, optim="BFGS", objective="LMP") for acc, expected in [("kernel", "matern3_2"), ("regmodel", "linear"), ("optim", "BFGS"), ("objective", "LMP"), ("normalize", False)]: - suite.true(f"Kriging.{acc}() == {expected!r}", getattr(k, acc)() == expected, detail=str(getattr(k, acc)())) -suite.close("Kriging.X() is the design", np.asarray(k.X()), X, rtol=0, atol=0) -suite.close("Kriging.y() is the response", np.ravel(k.y()), y, rtol=0, atol=0) + suite.check_true(f"Kriging.{acc}() == {expected!r}", getattr(k, acc)() == expected, detail=str(getattr(k, acc)())) +suite.check_close("Kriging.X() is the design", np.asarray(k.X()), X, rtol=0, atol=0) +suite.check_close("Kriging.y() is the response", np.ravel(k.y()), y, rtol=0, atol=0) # with normalize=True: do the accessors return the data as given, or as rescaled internally? kn = lk.Kriging(y, X * 10, "matern3_2", normalize=True) -suite.true("Kriging.normalize() == True", kn.normalize() is True or kn.normalize() == 1) -suite.close("normalize=True: X() is the design as given", np.asarray(kn.X()), X * 10, rtol=1e-12, atol=0, +suite.check_true("Kriging.normalize() == True", kn.normalize() is True or kn.normalize() == 1) +suite.check_close("normalize=True: X() is the design as given", np.asarray(kn.X()), X * 10, rtol=1e-12, atol=0, severity="warn") -suite.close("normalize=True: y() is the response as given", np.ravel(kn.y()), y, rtol=1e-12, atol=0, +suite.check_close("normalize=True: y() is the response as given", np.ravel(kn.y()), y, rtol=1e-12, atol=0, severity="warn") -suite.true("Kriging.beta() has 1 + d coefficients (linear trend)", np.ravel(k.beta()).size == 1 + X.shape[1]) -suite.true("summary() is a non-empty string", isinstance(k.summary(), str) and len(k.summary()) > 20) -suite.true("is_theta_estim() after an optimized fit", bool(k.is_theta_estim())) +suite.check_true("Kriging.beta() has 1 + d coefficients (linear trend)", np.ravel(k.beta()).size == 1 + X.shape[1]) +suite.check_true("summary() is a non-empty string", isinstance(k.summary(), str) and len(k.summary()) > 20) +suite.check_true("is_theta_estim() after an optimized fit", bool(k.is_theta_estim())) # %% [markdown] # ## Summary diff --git a/tools/src/sklearn-estimators_Kriging_python.py b/tools/src/sklearn-estimators_Kriging_python.py index 56c3aff..98978c5 100644 --- a/tools/src/sklearn-estimators_Kriging_python.py +++ b/tools/src/sklearn-estimators_Kriging_python.py @@ -18,10 +18,10 @@ import pandas as pd import pylibkriging as lk -import lkgallery -from lkgallery import TESTS, design, compat +import gallery +from gallery import TESTS, design, compat -suite = lkgallery.Suite("sklearn-estimators_Kriging_python") +suite = gallery.Suite("sklearn-estimators_Kriging_python") print("pylibkriging", compat.version()) tf = TESTS["hartmann3"] @@ -66,22 +66,22 @@ def mean_sd(k, Xn): est = KrigingRegressor(kernel="matern5_2").fit(X, y) m_sk, s_sk = est.predict(Xt, return_std=True) - suite.true("KrigingRegressor.predict returns 1-D arrays (scikit-learn convention)", + suite.check_true("KrigingRegressor.predict returns 1-D arrays (scikit-learn convention)", np.ndim(m_sk) == 1 and np.ndim(s_sk) == 1, detail=f"shapes {np.shape(m_sk)}, {np.shape(s_sk)}", severity="warn") m_sk, s_sk = np.ravel(m_sk), np.ravel(s_sk) m_nat, s_nat = mean_sd(lk.Kriging(y, X, "matern5_2"), Xt) - suite.close("KrigingRegressor == Kriging (mean)", m_sk, m_nat, rtol=1e-8, atol=1e-10) - suite.close("KrigingRegressor == Kriging (stdev)", s_sk, s_nat, rtol=1e-6, atol=1e-10) + suite.check_close("KrigingRegressor == Kriging (mean)", m_sk, m_nat, rtol=1e-8, atol=1e-10) + suite.check_close("KrigingRegressor == Kriging (stdev)", s_sk, s_nat, rtol=1e-6, atol=1e-10) est2 = clone(est).set_params(kernel="gauss") - suite.true("clone + set_params", est2.get_params()["kernel"] == "gauss" and est.get_params()["kernel"] == + suite.check_true("clone + set_params", est2.get_params()["kernel"] == "gauss" and est.get_params()["kernel"] == "matern5_2") - with suite.runs("Pipeline(MinMaxScaler, KrigingRegressor)"): + with suite.check_runs("Pipeline(MinMaxScaler, KrigingRegressor)"): pipe = make_pipeline(MinMaxScaler(), KrigingRegressor(kernel="matern5_2")).fit(tf.scale(X), y) - suite.greater("pipeline Q2 on native inputs", pipe.score(tf.scale(Xt), yt), 0.8) - with suite.runs("GridSearchCV over kernels"): + suite.check_greater("pipeline Q2 on native inputs", pipe.score(tf.scale(Xt), yt), 0.8) + with suite.check_runs("GridSearchCV over kernels"): gs = GridSearchCV(KrigingRegressor(), {"kernel": ["gauss", "matern3_2", "matern5_2"]}, cv=3).fit(X, y) - suite.true("GridSearchCV picked a kernel", gs.best_params_["kernel"] in ("gauss", "matern3_2", "matern5_2"), + suite.check_true("GridSearchCV picked a kernel", gs.best_params_["kernel"] in ("gauss", "matern3_2", "matern5_2"), detail=str(gs.best_params_)) cv = pd.DataFrame(gs.cv_results_)[["param_kernel", "mean_test_score", "std_test_score", "mean_fit_time"]] for _, r in cv.iterrows(): @@ -91,14 +91,14 @@ def mean_sd(k, Xn): plt.bar(cv["param_kernel"].astype(str), cv["mean_test_score"], yerr=cv["std_test_score"], capsize=4) plt.ylabel("3-fold CV R²"); plt.title(f"GridSearchCV(KrigingRegressor): best = {gs.best_params_['kernel']}") plt.ylim(min(0, cv["mean_test_score"].min() - .05), 1.02); plt.show() - with suite.runs("cross_val_score"): + with suite.check_runs("cross_val_score"): sc = cross_val_score(KrigingRegressor(kernel="matern5_2"), X, y, cv=4) - suite.finite("cross_val_score finite", sc) - with suite.runs("sample_y"): + suite.check_finite("cross_val_score finite", sc) + with suite.check_runs("sample_y"): S = est.sample_y(Xt[:10], n_samples=5, random_state=1) - suite.true("sample_y shape (q, n_samples)", np.shape(S) == (10, 5), detail=str(np.shape(S))) + suite.check_true("sample_y shape (q, n_samples)", np.shape(S) == (10, 5), detail=str(np.shape(S))) else: - suite.skip("sklearn estimators", "pylibkriging.sklearn or scikit-learn not available") + suite.check_skip("sklearn estimators", "pylibkriging.sklearn or scikit-learn not available") # %% [markdown] # ## Summary diff --git a/tools/src/test-functions-reference-values_no-model_python.py b/tools/src/test-functions-reference-values_no-model_python.py index 36e8eda..5651a88 100644 --- a/tools/src/test-functions-reference-values_no-model_python.py +++ b/tools/src/test-functions-reference-values_no-model_python.py @@ -9,7 +9,7 @@ # Experiments, Surjanovic & Bingham). # # Every notebook of this gallery relies on the test functions of -# `lkgallery.functions` (after the [Virtual Library of Simulation Experiments](https://www.sfu.ca/~ssurjano/)). +# `gallery.functions` (after the [Virtual Library of Simulation Experiments](https://www.sfu.ca/~ssurjano/)). # A typo in one of them would silently bias every result built on it, so they are # checked first, against the reference values published on the library pages # (global optima, nominal points) and against the known output ranges. @@ -19,10 +19,10 @@ import pandas as pd import matplotlib.pyplot as plt -import lkgallery -from lkgallery import TESTS, design +import gallery +from gallery import TESTS, design -suite = lkgallery.Suite("test-functions-reference-values_no-model_python") +suite = gallery.Suite("test-functions-reference-values_no-model_python") # %% [markdown] # ## Reference values @@ -32,7 +32,7 @@ for name, tf in TESTS.items(): for x, v in tf.known: fx = float(tf(x)[0]) - suite.close(f"{name}: f({np.round(x, 4).tolist()})", fx, v, rtol=1e-4, atol=1e-4) + suite.check_close(f"{name}: f({np.round(x, 4).tolist()})", fx, v, rtol=1e-4, atol=1e-4) rows.append((name, tf.d, np.round(x, 4).tolist(), v, fx)) pd.DataFrame(rows, columns=["function", "d", "x", "reference", "computed"]) @@ -56,10 +56,10 @@ for name, tf in TESTS.items(): U = design.uniform(20_000, tf.d, seed=1) y = tf.unit(U) - suite.finite(f"{name}: finite on the domain", y) + suite.check_finite(f"{name}: finite on the domain", y) if name in EXPECTED_RANGES: lo, hi = EXPECTED_RANGES[name] - suite.true(f"{name}: range within [{lo}, {hi}]", lo <= y.min() and y.max() <= hi, + suite.check_true(f"{name}: range within [{lo}, {hi}]", lo <= y.min() and y.max() <= hi, detail=f"sampled range [{y.min():.4g}, {y.max():.4g}]") summary.append((name, tf.d, ",".join(tf.tags), y.min(), y.max(), y.std())) pd.DataFrame(summary, columns=["function", "d", "tags", "min", "max", "std"]) @@ -73,9 +73,9 @@ # %% A, B = design.lhs(20, 3, seed=7), design.lhs(20, 3, seed=7) -suite.close("lhs: reproducible with a seed", A, B, rtol=0, atol=0) +suite.check_close("lhs: reproducible with a seed", A, B, rtol=0, atol=0) strata = np.floor(A * 20).astype(int) -suite.true("lhs: one point per stratum", all(len(set(strata[:, j])) == 20 for j in range(3))) +suite.check_true("lhs: one point per stratum", all(len(set(strata[:, j])) == 20 for j in range(3))) # %% fig, axes = plt.subplots(1, 4, figsize=(16, 3.5)) diff --git a/tools/src/update-simulate_Kriging_python.py b/tools/src/update-simulate_Kriging_python.py index a2738a9..2221c0b 100644 --- a/tools/src/update-simulate_Kriging_python.py +++ b/tools/src/update-simulate_Kriging_python.py @@ -28,10 +28,10 @@ import numpy as np import pylibkriging as lk -import lkgallery -from lkgallery import TESTS, design, compat +import gallery +from gallery import TESTS, design, compat -suite = lkgallery.Suite("update-simulate_Kriging_python") +suite = gallery.Suite("update-simulate_Kriging_python") print("pylibkriging", compat.version()) @@ -60,20 +60,20 @@ def fixed_params(k): parameters={"theta": p["theta"], "is_theta_estim": False}) m, s, _ = pred(k, Xt) mr, sr, _ = pred(ref, Xt) - suite.close(f"{case}: update(refit=False) mean == scratch fit, same theta", m, mr, rtol=1e-6, + suite.check_close(f"{case}: update(refit=False) mean == scratch fit, same theta", m, mr, rtol=1e-6, atol=1e-8 * np.std(y)) - suite.close(f"{case}: update(refit=False) stdev == scratch fit, same theta", s, sr, rtol=1e-5, + suite.check_close(f"{case}: update(refit=False) stdev == scratch fit, same theta", s, sr, rtol=1e-5, atol=1e-8 * np.std(y)) - suite.close(f"{case}: update(refit=False) sigma2 == scratch fit sigma2", k.sigma2(), ref.sigma2(), + suite.check_close(f"{case}: update(refit=False) sigma2 == scratch fit sigma2", k.sigma2(), ref.sigma2(), rtol=1e-8) - suite.close(f"{case}: update(refit=False) keeps theta", np.ravel(k.theta()), np.ravel(p["theta"]), + suite.check_close(f"{case}: update(refit=False) keeps theta", np.ravel(k.theta()), np.ravel(p["theta"]), rtol=0, atol=0) - suite.true(f"{case}: model holds n0+nu points", np.asarray(k.X()).shape[0] == n0 + nu) + suite.check_true(f"{case}: model holds n0+nu points", np.asarray(k.X()).shape[0] == n0 + nu) # refit=True vs a fresh fit (same optimizer from the same data, possibly another start) k2 = lk.Kriging(y[:n0], X[:n0], "matern5_2", regmodel=trend) k2.update(y[n0:], X[n0:], refit=True) fresh = lk.Kriging(y, X, "matern5_2", regmodel=trend) - suite.close(f"{case}: update(refit=True) LL == fresh fit LL", k2.logLikelihood(), fresh.logLikelihood(), + suite.check_close(f"{case}: update(refit=True) LL == fresh fit LL", k2.logLikelihood(), fresh.logLikelihood(), rtol=1e-3, severity="warn") # %% [markdown] @@ -87,8 +87,8 @@ def fixed_params(k): for i in range(20, 40, 5): ka.update(y[i:i + 5], X[i:i + 5], refit=False) kb.update(y[20:], X[20:], refit=False) -suite.close("4 updates of 5 == 1 update of 20 (mean)", pred(ka, Xt)[0], pred(kb, Xt)[0], rtol=1e-8, atol=1e-10) -suite.close("4 updates of 5 == 1 update of 20 (stdev)", pred(ka, Xt)[1], pred(kb, Xt)[1], rtol=1e-6, atol=1e-10) +suite.check_close("4 updates of 5 == 1 update of 20 (mean)", pred(ka, Xt)[0], pred(kb, Xt)[0], rtol=1e-8, atol=1e-10) +suite.check_close("4 updates of 5 == 1 update of 20 (stdev)", pred(ka, Xt)[1], pred(kb, Xt)[1], rtol=1e-6, atol=1e-10) # %% [markdown] # ## `simulate`: moments, seeds, conditioning @@ -100,19 +100,19 @@ def fixed_params(k): k = lk.Kriging(y, X, "matern5_2") nsim = 5000 S = np.asarray(k.simulate(nsim, 123, Xt)) - suite.true(f"{fname}: simulate shape (q, nsim)", S.shape == (len(Xt), nsim), detail=str(S.shape)) + suite.check_true(f"{fname}: simulate shape (q, nsim)", S.shape == (len(Xt), nsim), detail=str(S.shape)) m, s, c = pred(k, Xt, cov=True) z = (S.mean(axis=1) - m) / (s / np.sqrt(nsim)) - suite.less(f"{fname}: empirical mean of paths vs predict (max |z|)", np.max(np.abs(z)), 5.0) + suite.check_less(f"{fname}: empirical mean of paths vs predict (max |z|)", np.max(np.abs(z)), 5.0) rel = np.linalg.norm(np.cov(S) - c) / np.linalg.norm(c) - suite.less(f"{fname}: empirical covariance of paths vs predict (rel. Frobenius)", rel, 0.1) + suite.check_less(f"{fname}: empirical covariance of paths vs predict (rel. Frobenius)", rel, 0.1) S2 = np.asarray(k.simulate(nsim, 123, Xt)) - suite.close(f"{fname}: same seed, same paths", S2, S, rtol=0, atol=0) + suite.check_close(f"{fname}: same seed, same paths", S2, S, rtol=0, atol=0) S3 = np.asarray(k.simulate(nsim, 124, Xt)) - suite.true(f"{fname}: another seed, other paths", not np.allclose(S3, S)) + suite.check_true(f"{fname}: another seed, other paths", not np.allclose(S3, S)) Sx = np.asarray(k.simulate(20, 1, X[:5])) # up to the numerical jitter added to the conditional covariance (relative to the prior sd) - suite.close(f"{fname}: paths reproduce y at the design points", Sx, np.tile(y[:5, None], (1, 20)), + suite.check_close(f"{fname}: paths reproduce y at the design points", Sx, np.tile(y[:5, None], (1, 20)), rtol=0, atol=1e-4 * np.sqrt(k.sigma2())) # %% [markdown] @@ -128,16 +128,16 @@ def fixed_params(k): k = lk.Kriging(y[:n0], X[:n0], "matern5_2") p = fixed_params(k) nsim = 5000 -with suite.runs("update_simulate runs"): +with suite.check_runs("update_simulate runs"): k.simulate(nsim, 7, Xt, will_update=True) U = np.asarray(k.update_simulate(y[n0:], X[n0:])) # paths drawn under (theta, sigma2) of the first model, then conditioned on all the data ref = lk.Kriging(y, X, "matern5_2", optim="none", parameters=p) m, s, c = pred(ref, Xt, cov=True) z = (U.mean(axis=1) - m) / (s / np.sqrt(nsim)) - suite.less("update_simulate: mean of corrected paths vs updated model (max |z|)", np.max(np.abs(z)), 5.0) + suite.check_less("update_simulate: mean of corrected paths vs updated model (max |z|)", np.max(np.abs(z)), 5.0) rel = np.linalg.norm(np.cov(U) - c) / np.linalg.norm(c) - suite.less("update_simulate: covariance vs updated model (rel. Frobenius)", rel, 0.1) + suite.check_less("update_simulate: covariance vs updated model (rel. Frobenius)", rel, 0.1) # %% [markdown] # ## In pictures: Forrester, 6 points, then 2 more @@ -154,7 +154,7 @@ def fixed_params(k): k = lk.Kriging(y, X, "matern5_2") P = np.asarray(k.simulate(20, 3, G, will_update=True)) PU = np.asarray(k.update_simulate(yu, Xu)) -suite.less("update_simulate: corrected paths go through the new points (max |path - y_u| / sd(y))", +suite.check_less("update_simulate: corrected paths go through the new points (max |path - y_u| / sd(y))", np.max(np.abs(PU[[140, 180], :] - yu[:, None])) / np.std(y), 1e-3) fig, axes = plt.subplots(1, 2, figsize=(11, 3.8), sharey=True) for ax, paths, title in [(axes[0], P, "6 points: conditional paths"), (axes[1], PU, "+ 2 points: update_simulate")]: diff --git a/tools/src/vs-DiceKriging-RobustGaSP_Kriging_R.R b/tools/src/vs-DiceKriging-RobustGaSP_Kriging_R.R index 1d6e73a..4270d91 100644 --- a/tools/src/vs-DiceKriging-RobustGaSP_Kriging_R.R +++ b/tools/src/vs-DiceKriging-RobustGaSP_Kriging_R.R @@ -43,8 +43,8 @@ suppressPackageStartupMessages({ library(DiceKriging) }) has_rgasp <- requireNamespace("RobustGaSP", quietly = TRUE) -source("tools/r/checks.R") -s <- new_suite("vs-DiceKriging-RobustGaSP_Kriging_R") +source("tools/R/gallery.R") +s <- suite("vs-DiceKriging-RobustGaSP_Kriging_R") cat("rlibkriging", as.character(packageVersion("rlibkriging")), "| DiceKriging", as.character(packageVersion("DiceKriging")), "| RobustGaSP", if (has_rgasp) as.character(packageVersion("RobustGaSP")) else "absent", "\n") DATASETS <- c("branin", "hartmann3", "borehole") @@ -98,11 +98,11 @@ for (name in DATASETS) { r <- tryCatch(fits[[pkg]](), error = function(e) e) t_fit <- as.numeric(difftime(Sys.time(), t0, units = "secs")) if (inherits(r, "error")) { - if (startsWith(pkg, "rlibkriging")) add_check(s, paste0(name, "/", pkg, ": fit"), FALSE, detail = conditionMessage(r)) + if (startsWith(pkg, "rlibkriging")) check(s, paste0(name, "/", pkg, ": fit"), FALSE, detail = conditionMessage(r)) next } m <- as.numeric(r(Xt)) - add_metric(s, name, pkg, q2 = q2(yt, m), rmse = rmse(yt, m), fit_s = t_fit) + metric(s, name, pkg, q2 = q2(yt, m), rmse = rmse(yt, m), fit_s = t_fit) results[[length(results) + 1]] <- data.frame(dataset = name, package = pkg, q2 = q2(yt, m), fit_s = t_fit) } } @@ -133,7 +133,7 @@ for (name in DATASETS) { # All of them are also written to `results/vs-DiceKriging-RobustGaSP_Kriging_R.checks.csv` and `.metrics.csv`. # %% -summary_suite(s) +suite_summary(s) # %% -finalize(s) +suite_finalize(s) diff --git a/tools/src/vs-GaussianProcesses_Kriging_julia.jl b/tools/src/vs-GaussianProcesses_Kriging_julia.jl index 19f957b..288fdf6 100644 --- a/tools/src/vs-GaussianProcesses_Kriging_julia.jl +++ b/tools/src/vs-GaussianProcesses_Kriging_julia.jl @@ -17,7 +17,7 @@ # %% import JLibKriging as LK -include("tools/julia/checks.jl") +include("tools/julia/gallery.jl") s = Suite("vs-GaussianProcesses_Kriging_julia") # %% [markdown] @@ -46,11 +46,11 @@ for name in ("branin", "hartmann3", "borehole") Xd = read_data(name, "X"); yd = vec(read_data(name, "y")); Xt = read_data(name, "Xt"); yt = vec(read_data(name, "yt")) t = @elapsed k = LK.Kriging(yd, Xd, "matern5_2") q_lk = q2(yt, LK.predict(k, Xt; return_stdev=false).mean) - add_metric!(s, name, "JLibKriging"; q2=q_lk, fit_s=t) + metric!(s, name, "JLibKriging"; q2=q_lk, fit_s=t) t = @elapsed k10 = LK.Kriging(yd, Xd, "matern5_2"; optim="BFGS10") q_lk10 = q2(yt, LK.predict(k10, Xt; return_stdev=false).mean) - add_metric!(s, name, "JLibKriging BFGS10"; q2=q_lk10, fit_s=t) - add_check!(s, "$name: Q2(JLibKriging BFGS10) > 0.9", q_lk10 > 0.9; value=q_lk10, expected="> 0.9") + metric!(s, name, "JLibKriging BFGS10"; q2=q_lk10, fit_s=t) + check!(s, "$name: Q2(JLibKriging BFGS10) > 0.9", q_lk10 > 0.9; value=q_lk10, expected="> 0.9") if have_gp try d = size(Xd, 2) @@ -61,15 +61,15 @@ for name in ("branin", "hartmann3", "borehole") end mg, _ = GP.predict_f(gp, permutedims(Xt)) q_gp = q2(yt, mg) - add_metric!(s, name, "GaussianProcesses.jl"; q2=q_gp, fit_s=t) - add_check!(s, "$name: Q2(JLibKriging) - Q2(GaussianProcesses.jl)", q_lk - q_gp > -0.02; + metric!(s, name, "GaussianProcesses.jl"; q2=q_gp, fit_s=t) + check!(s, "$name: Q2(JLibKriging) - Q2(GaussianProcesses.jl)", q_lk - q_gp > -0.02; value=q_lk - q_gp, expected="> -0.02", severity="warn") catch e - add_check!(s, "$name: GaussianProcesses.jl fit", false; value=string(typeof(e)), severity="warn") + check!(s, "$name: GaussianProcesses.jl fit", false; value=string(typeof(e)), severity="warn") end end end -have_gp || add_check!(s, "GaussianProcesses.jl available", false; value="absent", severity="warn") +have_gp || check!(s, "GaussianProcesses.jl available", false; value="absent", severity="warn") foreach(r -> println(join(r[3:6], " ")), s.metrics) # %% [markdown] @@ -81,7 +81,7 @@ foreach(r -> println(join(r[3:6], " ")), s.metrics) # All of them are also written to `results/vs-GaussianProcesses_Kriging_julia.checks.csv` and `.metrics.csv`. # %% -summary!(s) +suite_summary(s) # %% -finalize!(s) +suite_finalize!(s) diff --git a/tools/src/vs-STK_Kriging_octave.m b/tools/src/vs-STK_Kriging_octave.m index e46f8e0..0f30ba5 100644 --- a/tools/src/vs-STK_Kriging_octave.m +++ b/tools/src/vs-STK_Kriging_octave.m @@ -24,8 +24,8 @@ % `tools/install/octave.sh`. % %% -addpath(fullfile(pwd, "tools", "octave")); -s = lkb_suite("vs-STK_Kriging_octave"); +source("tools/octave/gallery.m"); +s = suite("vs-STK_Kriging_octave"); % %% [markdown] % ## Installing STK @@ -48,9 +48,8 @@ run(fullfile(stk_path, "stk_init.m")); printf("STK %s from %s\n", stk_version(), stk_path); else - s = lkb_check(s, "STK available", false, "absent", stk_path, "STK could not be installed: comparison skipped", "warn"); + s = check(s, "STK available", false, "absent", stk_path, "STK could not be installed: comparison skipped", "warn"); end -q2 = @(y, m) 1 - sum((y - m).^2) / sum((y - mean(y)).^2); datasets = {"branin", "hartmann3", "borehole"}; % %% [markdown] @@ -70,8 +69,8 @@ model.param = [log(sigma2); -log(theta(:) * sqrt(2))]; model.lm = stk_lm_constant(); zp = stk_predict(model, X, y, Xt); - s = lkb_close(s, [name ": gauss UK mean == STK"], m_lk, zp.mean, 1e-6, 1e-8 * std(y)); - s = lkb_close(s, [name ": gauss UK variance == STK"], s_lk.^2, zp.var, 1e-4, 1e-8 * sigma2); + s = check_close(s, [name ": gauss UK mean == STK"], m_lk, zp.mean, 1e-6, 1e-8 * std(y)); + s = check_close(s, [name ": gauss UK variance == STK"], s_lk.^2, zp.var, 1e-4, 1e-8 * sigma2); end end @@ -86,12 +85,12 @@ tic; k = Kriging(y, X, "matern5_2"); t = toc; [m, ~] = k.predict(Xt, false, false, false); q_lk = q2(yt, m); - s = lkb_metric(s, name, "mLibKriging", "q2", q_lk); s = lkb_metric(s, name, "mLibKriging", "fit_s", t); + s = metric(s, name, "mLibKriging", "q2", q_lk); s = metric(s, name, "mLibKriging", "fit_s", t); tic; k10 = Kriging(y, X, "matern5_2", "constant", false, "BFGS10", "LL"); t = toc; [m, ~] = k10.predict(Xt, false, false, false); q_lk10 = q2(yt, m); - s = lkb_metric(s, name, "mLibKriging BFGS10", "q2", q_lk10); s = lkb_metric(s, name, "mLibKriging BFGS10", "fit_s", t); - s = lkb_check(s, [name ": Q2(mLibKriging BFGS10) > 0.9"], q_lk10 > 0.9, q_lk10, "> 0.9"); + s = metric(s, name, "mLibKriging BFGS10", "q2", q_lk10); s = metric(s, name, "mLibKriging BFGS10", "fit_s", t); + s = check(s, [name ": Q2(mLibKriging BFGS10) > 0.9"], q_lk10 > 0.9, q_lk10, "> 0.9"); if have_stk tic; model = stk_model(@stk_materncov52_aniso, d); @@ -99,8 +98,8 @@ zp = stk_predict(model, X, y, Xt); t = toc; q_stk = q2(yt, zp.mean); - s = lkb_metric(s, name, "STK", "q2", q_stk); s = lkb_metric(s, name, "STK", "fit_s", t); - s = lkb_check(s, [name ": Q2(mLibKriging) - Q2(STK)"], q_lk - q_stk > -0.02, q_lk - q_stk, "> -0.02", "", "warn"); + s = metric(s, name, "STK", "q2", q_stk); s = metric(s, name, "STK", "fit_s", t); + s = check(s, [name ": Q2(mLibKriging) - Q2(STK)"], q_lk - q_stk > -0.02, q_lk - q_stk, "> -0.02", "", "warn"); end end disp(s.metrics(:, 3:6)) @@ -114,7 +113,7 @@ % All of them are also written to `results/vs-STK_Kriging_octave.checks.csv` and `.metrics.csv`. % %% -lkb_summary(s); +suite_summary(s); % %% -lkb_finalize(s); +suite_finalize(s); diff --git a/tools/src/vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python.py b/tools/src/vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python.py index aa3fe69..f879eb1 100644 --- a/tools/src/vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python.py +++ b/tools/src/vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python.py @@ -54,11 +54,11 @@ import pandas as pd import pylibkriging as lk -import lkgallery -from lkgallery import TESTS, design, metrics, compat +import gallery +from gallery import TESTS, design, metrics, compat warnings.filterwarnings("ignore") -suite = lkgallery.Suite("vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python") +suite = gallery.Suite("vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python") sk = compat.optional("sklearn.gaussian_process") smt = compat.optional("smt.surrogate_models") ot = compat.optional("openturns") @@ -91,9 +91,9 @@ def lk_fixed(y, X, kernel, theta, sigma2, beta): kern = ConstantKernel(sigma2, "fixed") * RBF(theta, "fixed") gp = GaussianProcessRegressor(kern, alpha=1e-12, optimizer=None, normalize_y=False).fit(X, y - beta) m_sk, s_sk = gp.predict(Xt, return_std=True) - suite.close("sklearn RBF == libKriging gauss: simple kriging mean", m_sk + beta, m_lk, rtol=1e-6, + suite.check_close("sklearn RBF == libKriging gauss: simple kriging mean", m_sk + beta, m_lk, rtol=1e-6, atol=1e-7 * np.std(y)) - suite.close("sklearn RBF == libKriging gauss: simple kriging stdev (beta fixed)", s_sk, s_lk, rtol=1e-4, + suite.check_close("sklearn RBF == libKriging gauss: simple kriging stdev (beta fixed)", s_sk, s_lk, rtol=1e-4, atol=1e-6 * np.sqrt(sigma2)) # with beta estimated, libKriging's stdev is the universal kriging one: the simple kriging variance # plus the trend-estimation term (f - F'C^-1 c)^2 / (F'C^-1 F), computed here by hand @@ -102,14 +102,14 @@ def lk_fixed(y, X, kernel, theta, sigma2, beta): one = np.ones((len(y), 1)) v_uk = s_sk ** 2 + ((1 - one.T @ w) ** 2).ravel() / (one.T @ np.linalg.solve(C, one)).item() s_uk = np.ravel(kg.predict(Xt, return_stdev=True)[1]) - suite.close("libKriging gauss (beta estimated) stdev == universal kriging formula", s_uk, np.sqrt(v_uk), + suite.check_close("libKriging gauss (beta estimated) stdev == universal kriging formula", s_uk, np.sqrt(v_uk), rtol=1e-4, atol=1e-6 * np.sqrt(sigma2)) # the (concentrated) log-likelihood at the MLE equals the full Gaussian log-density at (theta, sigma2, beta) lml = gp.log_marginal_likelihood_value_ - suite.close("sklearn log-marginal-likelihood == libKriging logLikelihood()", lml, kg.logLikelihood(), + suite.check_close("sklearn log-marginal-likelihood == libKriging logLikelihood()", lml, kg.logLikelihood(), rtol=1e-6) else: - suite.skip("sklearn exact comparison", "scikit-learn not installed") + suite.check_skip("sklearn exact comparison", "scikit-learn not installed") # %% if ot is not None: @@ -130,12 +130,12 @@ def lk_fixed(y, X, kernel, theta, sigma2, beta): parameters={"theta": th.reshape(1, -1), "is_theta_estim": False, "sigma2": s2, "is_sigma2_estim": False}) m_u, s_u = (np.ravel(a) for a in ku.predict(Xt, return_stdev=True)[:2]) - suite.close("OpenTURNS product Matern 5/2 == libKriging matern5_2: mean", m_ot, m_u, rtol=1e-6, + suite.check_close("OpenTURNS product Matern 5/2 == libKriging matern5_2: mean", m_ot, m_u, rtol=1e-6, atol=1e-6 * np.std(y)) - suite.close("OpenTURNS product Matern 5/2 == libKriging matern5_2: variance (universal kriging)", v_ot, s_u ** 2, + suite.check_close("OpenTURNS product Matern 5/2 == libKriging matern5_2: variance (universal kriging)", v_ot, s_u ** 2, rtol=1e-4, atol=1e-8 * s2) else: - suite.skip("OpenTURNS exact comparison", "openturns not installed") + suite.check_skip("OpenTURNS exact comparison", "openturns not installed") # %% [markdown] # ## 2. Default fits on a set of test functions @@ -218,7 +218,7 @@ def p(Z): except Exception as exc: # noqa: BLE001 rows.append(dict(function=fname, n=n, rep=rep, package=pkg, error=str(exc)[:100])) if pkg.startswith("pylibkriging"): - suite.true(f"{fname}/rep{rep}/{pkg}: fit", False, detail=str(exc)[:200]) + suite.check_true(f"{fname}/rep{rep}/{pkg}: fit", False, detail=str(exc)[:200]) res = pd.DataFrame(rows) summary = res.groupby(["function", "package"], sort=False)[["q2", "nlpd", "coverage95", "fit_s"]].median() for (fname, pkg), r in summary.iterrows(): @@ -239,11 +239,11 @@ def p(Z): s = summary.loc[fname]["q2"] others = s.drop(index=[p for p in s.index if p.startswith("pylibkriging")]) if len(others): - suite.greater(f"{fname}: Q2(pylibkriging) - best other package", s["pylibkriging"] - others.max(), -0.02, + suite.check_greater(f"{fname}: Q2(pylibkriging) - best other package", s["pylibkriging"] - others.max(), -0.02, severity="warn", detail=f"best: {others.idxmax()}") - suite.greater(f"{fname}: Q2(pylibkriging BFGS10) - best other package", s["pylibkriging BFGS10"] - others.max(), + suite.check_greater(f"{fname}: Q2(pylibkriging BFGS10) - best other package", s["pylibkriging BFGS10"] - others.max(), -0.05, detail=f"best: {others.idxmax()}") if len(others) else None - suite.greater(f"{fname}: Q2(pylibkriging BFGS10) > 0.5", s["pylibkriging BFGS10"], 0.5) + suite.check_greater(f"{fname}: Q2(pylibkriging BFGS10) > 0.5", s["pylibkriging BFGS10"], 0.5) # %% [markdown] # ## Summary diff --git a/tools/src/warpings-categorical-ordinal_WarpKriging_python.py b/tools/src/warpings-categorical-ordinal_WarpKriging_python.py index 1ba28fd..c954d90 100644 --- a/tools/src/warpings-categorical-ordinal_WarpKriging_python.py +++ b/tools/src/warpings-categorical-ordinal_WarpKriging_python.py @@ -28,13 +28,13 @@ import matplotlib.pyplot as plt import pylibkriging as lk -import lkgallery -from lkgallery import TESTS, design, metrics, compat +import gallery +from gallery import TESTS, design, metrics, compat -suite = lkgallery.Suite("warpings-categorical-ordinal_WarpKriging_python") +suite = gallery.Suite("warpings-categorical-ordinal_WarpKriging_python") print("pylibkriging", compat.version()) if not compat.has("WarpKriging"): - suite.skip("WarpKriging", "not available in this pylibkriging") + suite.check_skip("WarpKriging", "not available in this pylibkriging") def pred(k, X): @@ -52,13 +52,13 @@ def evaluate(case, label, k, X, y, Xt, yt): if label.startswith("Kriging"): BASE_NLPD[case] = res["nlpd"] else: - suite.less(f"{case}/{label}: NLPD - NLPD(Kriging) (overconfident stdev?)", res["nlpd"] - BASE_NLPD[case], + suite.check_less(f"{case}/{label}: NLPD - NLPD(Kriging) (overconfident stdev?)", res["nlpd"] - BASE_NLPD[case], 2.0, severity="warn", detail=f"min stdev {np.min(s):.3g}, max |error|/stdev " f"{np.max(np.abs(yt - m) / np.maximum(s, 1e-300)):.3g}") suite.metric(case, label, ll=float(k.logLikelihood()), **res) - suite.finite(f"{case}/{label}: predictions finite", m) + suite.check_finite(f"{case}/{label}: predictions finite", m) mX, _ = pred(k, X) - suite.less(f"{case}/{label}: interpolates (max |m(X)-y| / sd(y))", np.max(np.abs(mX - y)) / np.std(y), 1e-3, + suite.check_less(f"{case}/{label}: interpolates (max |m(X)-y| / sd(y))", np.max(np.abs(mX - y)) / np.std(y), 1e-3, severity="warn") return res @@ -72,9 +72,9 @@ def evaluate(case, label, k, X, y, Xt, yt): for kernel in ["gauss", "matern5_2"]: kp = lk.Kriging(y, X, kernel) kw = lk.WarpKriging(y, X, ["none", "none"], kernel) - suite.close(f"{kernel}: WarpKriging(none) LL == Kriging LL", kw.logLikelihood(), kp.logLikelihood(), + suite.check_close(f"{kernel}: WarpKriging(none) LL == Kriging LL", kw.logLikelihood(), kp.logLikelihood(), rtol=1e-3, severity="warn") - suite.close(f"{kernel}: WarpKriging(none) mean == Kriging mean", pred(kw, Xt)[0], pred(kp, Xt)[0], + suite.check_close(f"{kernel}: WarpKriging(none) mean == Kriging mean", pred(kw, Xt)[0], pred(kp, Xt)[0], rtol=1e-2, atol=1e-2 * np.std(y), severity="warn") # %% [markdown] @@ -90,12 +90,12 @@ def evaluate(case, label, k, X, y, Xt, yt): base = evaluate("gl2012", "Kriging", lk.Kriging(y, X, "matern5_2"), X, y, Xt, yt) fits = {} for w in WARPS_1D: - with suite.runs(f"gl2012/{w}: fit"): + with suite.check_runs(f"gl2012/{w}: fit"): fits[w] = lk.WarpKriging(y, X, [w], "matern5_2") evaluate("gl2012", w, fits[w], X, y, Xt, yt) t = suite.table().loc["gl2012"] best = t.drop(index="Kriging")["rmse"].min() - suite.less("gl2012: best warp RMSE / Kriging RMSE", best / base["rmse"], 1.0, severity="warn") + suite.check_less("gl2012: best warp RMSE / Kriging RMSE", best / base["rmse"], 1.0, severity="warn") u = np.linspace(0.01, 0.99, 400)[:, None] fig, ax = plt.subplots(figsize=(11, 4)) ax.plot(u, tf.unit(u), "k--", lw=2, label="truth") @@ -116,10 +116,10 @@ def evaluate(case, label, k, X, y, Xt, yt): X = X * 0.98 + 0.01; Xt = Xt * 0.98 + 0.01 y, yt = tf.unit(X), tf.unit(Xt) base = evaluate("gl2008", "Kriging", lk.Kriging(y, X, "matern5_2"), X, y, Xt, yt) - with suite.runs("gl2008/kumaraswamy: fit"): + with suite.check_runs("gl2008/kumaraswamy: fit"): res = evaluate("gl2008", "kumaraswamy", lk.WarpKriging(y, X, ["kumaraswamy"] * 2, "matern5_2"), X, y, Xt, yt) - suite.less("gl2008: RMSE(kumaraswamy) / RMSE(Kriging)", res["rmse"] / base["rmse"], 1.0, severity="warn") + suite.check_less("gl2008: RMSE(kumaraswamy) / RMSE(Kriging)", res["rmse"] / base["rmse"], 1.0, severity="warn") # %% [markdown] # ## Mixed continuous / categorical @@ -147,12 +147,12 @@ def mixed_design(n, seed): X = mixed_design(60, 34); Xt = mixed_design(1000, 35) y, yt = mixed(X), mixed(Xt) base = evaluate("mixed", "Kriging (level as number)", lk.Kriging(y, X, "matern5_2"), X, y, Xt, yt) - with suite.runs("mixed/categorical: fit"): + with suite.check_runs("mixed/categorical: fit"): kc = lk.WarpKriging(y, X, [f"categorical({L},2)", "none", "none"], "matern5_2") res = evaluate("mixed", "categorical(5,2)", kc, X, y, Xt, yt) - suite.less("mixed: RMSE(categorical) / RMSE(level as number)", res["rmse"] / base["rmse"], 1.0, + suite.check_less("mixed: RMSE(categorical) / RMSE(level as number)", res["rmse"] / base["rmse"], 1.0, severity="warn") - suite.greater("mixed/categorical: Q2", res["q2"], 0.8, severity="warn") + suite.check_greater("mixed/categorical: Q2", res["q2"], 0.8, severity="warn") # %% [markdown] # ## Mixed continuous / ordinal @@ -177,10 +177,10 @@ def f_ord(Xo): y, yt = f_ord(X), f_ord(Xt) base = evaluate("ordinal", "Kriging (rank as number)", lk.Kriging(y, X / [5, 1, 1], "matern5_2"), X / [5, 1, 1], y, Xt / [5, 1, 1], yt) - with suite.runs("ordinal/ordinal(6): fit"): + with suite.check_runs("ordinal/ordinal(6): fit"): ko = lk.WarpKriging(y, X, ["ordinal(6)", "none", "none"], "matern5_2") res = evaluate("ordinal", "ordinal(6)", ko, X, y, Xt, yt) - suite.less("ordinal: RMSE(ordinal) / RMSE(rank as number)", res["rmse"] / base["rmse"], 1.1, + suite.check_less("ordinal: RMSE(ordinal) / RMSE(rank as number)", res["rmse"] / base["rmse"], 1.1, severity="warn") # %% @@ -196,8 +196,8 @@ def f_ord(Xo): # %% if compat.has("WarpKriging"): X, y, Xt, yt = design.dataset(TESTS["hartmann3"], 40, n_test=100, seed=61) - with suite.runs("WarpKriging: fit (persistence)"): - lkgallery.persistence.check(suite, "WarpKriging", lk.WarpKriging(y, X, ["kumaraswamy", "none", "affine"], "matern5_2"), Xt, pred) + with suite.check_runs("WarpKriging: fit (persistence)"): + gallery.persistence.check(suite, "WarpKriging", lk.WarpKriging(y, X, ["kumaraswamy", "none", "affine"], "matern5_2"), Xt, pred) # %% [markdown] # ## Summary diff --git a/update-simulate_Kriging_python.ipynb b/update-simulate_Kriging_python.ipynb index a5c9241..be3d7f8 100644 --- a/update-simulate_Kriging_python.ipynb +++ b/update-simulate_Kriging_python.ipynb @@ -39,7 +39,7 @@ "source": [ "# Setup for running this notebook outside the repository (Colab, Binder, a lone copy):\n", "# nothing to do in the repository itself (CI, local runs). Otherwise the repository\n", - "# (datasets of data/, helpers of tools/) is cloned and the released pylibkriging installed.\n", + "# (datasets of data/, helpers of tools/python/gallery.py) is cloned and the released pylibkriging installed.\n", "import importlib\n", "import os\n", "import subprocess\n", @@ -61,7 +61,7 @@ " \"(PyPI has wheels for CPython 3.10 to 3.12 only). On Colab: Runtime > Change runtime version > \"\n", " \"2026.07 (Python 3.12), then run again.\\n\" + r.stderr[-500:])\n", " importlib.invalidate_caches()\n", - "sys.path.insert(0, str(Path(\"tools\").resolve())) # lkgallery, when not installed (Colab, Binder)\n", + "sys.path.insert(0, str(Path(\"tools/python\").resolve())) # for \"import gallery\" (tools/python/gallery.py)\n", "print(\"working directory:\", os.getcwd())" ] }, @@ -76,10 +76,10 @@ "import numpy as np\n", "import pylibkriging as lk\n", "\n", - "import lkgallery\n", - "from lkgallery import TESTS, design, compat\n", + "import gallery\n", + "from gallery import TESTS, design, compat\n", "\n", - "suite = lkgallery.Suite(\"update-simulate_Kriging_python\")\n", + "suite = gallery.Suite(\"update-simulate_Kriging_python\")\n", "print(\"pylibkriging\", compat.version())\n", "\n", "\n", @@ -120,20 +120,20 @@ " parameters={\"theta\": p[\"theta\"], \"is_theta_estim\": False})\n", " m, s, _ = pred(k, Xt)\n", " mr, sr, _ = pred(ref, Xt)\n", - " suite.close(f\"{case}: update(refit=False) mean == scratch fit, same theta\", m, mr, rtol=1e-6,\n", + " suite.check_close(f\"{case}: update(refit=False) mean == scratch fit, same theta\", m, mr, rtol=1e-6,\n", " atol=1e-8 * np.std(y))\n", - " suite.close(f\"{case}: update(refit=False) stdev == scratch fit, same theta\", s, sr, rtol=1e-5,\n", + " suite.check_close(f\"{case}: update(refit=False) stdev == scratch fit, same theta\", s, sr, rtol=1e-5,\n", " atol=1e-8 * np.std(y))\n", - " suite.close(f\"{case}: update(refit=False) sigma2 == scratch fit sigma2\", k.sigma2(), ref.sigma2(),\n", + " suite.check_close(f\"{case}: update(refit=False) sigma2 == scratch fit sigma2\", k.sigma2(), ref.sigma2(),\n", " rtol=1e-8)\n", - " suite.close(f\"{case}: update(refit=False) keeps theta\", np.ravel(k.theta()), np.ravel(p[\"theta\"]),\n", + " suite.check_close(f\"{case}: update(refit=False) keeps theta\", np.ravel(k.theta()), np.ravel(p[\"theta\"]),\n", " rtol=0, atol=0)\n", - " suite.true(f\"{case}: model holds n0+nu points\", np.asarray(k.X()).shape[0] == n0 + nu)\n", + " suite.check_true(f\"{case}: model holds n0+nu points\", np.asarray(k.X()).shape[0] == n0 + nu)\n", " # refit=True vs a fresh fit (same optimizer from the same data, possibly another start)\n", " k2 = lk.Kriging(y[:n0], X[:n0], \"matern5_2\", regmodel=trend)\n", " k2.update(y[n0:], X[n0:], refit=True)\n", " fresh = lk.Kriging(y, X, \"matern5_2\", regmodel=trend)\n", - " suite.close(f\"{case}: update(refit=True) LL == fresh fit LL\", k2.logLikelihood(), fresh.logLikelihood(),\n", + " suite.check_close(f\"{case}: update(refit=True) LL == fresh fit LL\", k2.logLikelihood(), fresh.logLikelihood(),\n", " rtol=1e-3, severity=\"warn\")" ] }, @@ -159,8 +159,8 @@ "for i in range(20, 40, 5):\n", " ka.update(y[i:i + 5], X[i:i + 5], refit=False)\n", "kb.update(y[20:], X[20:], refit=False)\n", - "suite.close(\"4 updates of 5 == 1 update of 20 (mean)\", pred(ka, Xt)[0], pred(kb, Xt)[0], rtol=1e-8, atol=1e-10)\n", - "suite.close(\"4 updates of 5 == 1 update of 20 (stdev)\", pred(ka, Xt)[1], pred(kb, Xt)[1], rtol=1e-6, atol=1e-10)" + "suite.check_close(\"4 updates of 5 == 1 update of 20 (mean)\", pred(ka, Xt)[0], pred(kb, Xt)[0], rtol=1e-8, atol=1e-10)\n", + "suite.check_close(\"4 updates of 5 == 1 update of 20 (stdev)\", pred(ka, Xt)[1], pred(kb, Xt)[1], rtol=1e-6, atol=1e-10)" ] }, { @@ -184,19 +184,19 @@ " k = lk.Kriging(y, X, \"matern5_2\")\n", " nsim = 5000\n", " S = np.asarray(k.simulate(nsim, 123, Xt))\n", - " suite.true(f\"{fname}: simulate shape (q, nsim)\", S.shape == (len(Xt), nsim), detail=str(S.shape))\n", + " suite.check_true(f\"{fname}: simulate shape (q, nsim)\", S.shape == (len(Xt), nsim), detail=str(S.shape))\n", " m, s, c = pred(k, Xt, cov=True)\n", " z = (S.mean(axis=1) - m) / (s / np.sqrt(nsim))\n", - " suite.less(f\"{fname}: empirical mean of paths vs predict (max |z|)\", np.max(np.abs(z)), 5.0)\n", + " suite.check_less(f\"{fname}: empirical mean of paths vs predict (max |z|)\", np.max(np.abs(z)), 5.0)\n", " rel = np.linalg.norm(np.cov(S) - c) / np.linalg.norm(c)\n", - " suite.less(f\"{fname}: empirical covariance of paths vs predict (rel. Frobenius)\", rel, 0.1)\n", + " suite.check_less(f\"{fname}: empirical covariance of paths vs predict (rel. Frobenius)\", rel, 0.1)\n", " S2 = np.asarray(k.simulate(nsim, 123, Xt))\n", - " suite.close(f\"{fname}: same seed, same paths\", S2, S, rtol=0, atol=0)\n", + " suite.check_close(f\"{fname}: same seed, same paths\", S2, S, rtol=0, atol=0)\n", " S3 = np.asarray(k.simulate(nsim, 124, Xt))\n", - " suite.true(f\"{fname}: another seed, other paths\", not np.allclose(S3, S))\n", + " suite.check_true(f\"{fname}: another seed, other paths\", not np.allclose(S3, S))\n", " Sx = np.asarray(k.simulate(20, 1, X[:5]))\n", " # up to the numerical jitter added to the conditional covariance (relative to the prior sd)\n", - " suite.close(f\"{fname}: paths reproduce y at the design points\", Sx, np.tile(y[:5, None], (1, 20)),\n", + " suite.check_close(f\"{fname}: paths reproduce y at the design points\", Sx, np.tile(y[:5, None], (1, 20)),\n", " rtol=0, atol=1e-4 * np.sqrt(k.sigma2()))" ] }, @@ -224,16 +224,16 @@ "k = lk.Kriging(y[:n0], X[:n0], \"matern5_2\")\n", "p = fixed_params(k)\n", "nsim = 5000\n", - "with suite.runs(\"update_simulate runs\"):\n", + "with suite.check_runs(\"update_simulate runs\"):\n", " k.simulate(nsim, 7, Xt, will_update=True)\n", " U = np.asarray(k.update_simulate(y[n0:], X[n0:]))\n", " # paths drawn under (theta, sigma2) of the first model, then conditioned on all the data\n", " ref = lk.Kriging(y, X, \"matern5_2\", optim=\"none\", parameters=p)\n", " m, s, c = pred(ref, Xt, cov=True)\n", " z = (U.mean(axis=1) - m) / (s / np.sqrt(nsim))\n", - " suite.less(\"update_simulate: mean of corrected paths vs updated model (max |z|)\", np.max(np.abs(z)), 5.0)\n", + " suite.check_less(\"update_simulate: mean of corrected paths vs updated model (max |z|)\", np.max(np.abs(z)), 5.0)\n", " rel = np.linalg.norm(np.cov(U) - c) / np.linalg.norm(c)\n", - " suite.less(\"update_simulate: covariance vs updated model (rel. Frobenius)\", rel, 0.1)" + " suite.check_less(\"update_simulate: covariance vs updated model (rel. Frobenius)\", rel, 0.1)" ] }, { @@ -262,7 +262,7 @@ "k = lk.Kriging(y, X, \"matern5_2\")\n", "P = np.asarray(k.simulate(20, 3, G, will_update=True))\n", "PU = np.asarray(k.update_simulate(yu, Xu))\n", - "suite.less(\"update_simulate: corrected paths go through the new points (max |path - y_u| / sd(y))\",\n", + "suite.check_less(\"update_simulate: corrected paths go through the new points (max |path - y_u| / sd(y))\",\n", " np.max(np.abs(PU[[140, 180], :] - yu[:, None])) / np.std(y), 1e-3)\n", "fig, axes = plt.subplots(1, 2, figsize=(11, 3.8), sharey=True)\n", "for ax, paths, title in [(axes[0], P, \"6 points: conditional paths\"), (axes[1], PU, \"+ 2 points: update_simulate\")]:\n", diff --git a/vs-DiceKriging-RobustGaSP_Kriging_R.ipynb b/vs-DiceKriging-RobustGaSP_Kriging_R.ipynb index d8a3e4e..a954361 100644 --- a/vs-DiceKriging-RobustGaSP_Kriging_R.ipynb +++ b/vs-DiceKriging-RobustGaSP_Kriging_R.ipynb @@ -68,8 +68,8 @@ " library(DiceKriging)\n", "})\n", "has_rgasp <- requireNamespace(\"RobustGaSP\", quietly = TRUE)\n", - "source(\"tools/r/checks.R\")\n", - "s <- new_suite(\"vs-DiceKriging-RobustGaSP_Kriging_R\")\n", + "source(\"tools/R/gallery.R\")\n", + "s <- suite(\"vs-DiceKriging-RobustGaSP_Kriging_R\")\n", "cat(\"rlibkriging\", as.character(packageVersion(\"rlibkriging\")), \"| DiceKriging\", as.character(packageVersion(\"DiceKriging\")),\n", " \"| RobustGaSP\", if (has_rgasp) as.character(packageVersion(\"RobustGaSP\")) else \"absent\", \"\\n\")\n", "DATASETS <- c(\"branin\", \"hartmann3\", \"borehole\")" @@ -147,11 +147,11 @@ " r <- tryCatch(fits[[pkg]](), error = function(e) e)\n", " t_fit <- as.numeric(difftime(Sys.time(), t0, units = \"secs\"))\n", " if (inherits(r, \"error\")) {\n", - " if (startsWith(pkg, \"rlibkriging\")) add_check(s, paste0(name, \"/\", pkg, \": fit\"), FALSE, detail = conditionMessage(r))\n", + " if (startsWith(pkg, \"rlibkriging\")) check(s, paste0(name, \"/\", pkg, \": fit\"), FALSE, detail = conditionMessage(r))\n", " next\n", " }\n", " m <- as.numeric(r(Xt))\n", - " add_metric(s, name, pkg, q2 = q2(yt, m), rmse = rmse(yt, m), fit_s = t_fit)\n", + " metric(s, name, pkg, q2 = q2(yt, m), rmse = rmse(yt, m), fit_s = t_fit)\n", " results[[length(results) + 1]] <- data.frame(dataset = name, package = pkg, q2 = q2(yt, m), fit_s = t_fit)\n", " }\n", "}\n", @@ -201,7 +201,7 @@ "metadata": {}, "outputs": [], "source": [ - "summary_suite(s)" + "suite_summary(s)" ] }, { @@ -211,7 +211,7 @@ "metadata": {}, "outputs": [], "source": [ - "finalize(s)" + "suite_finalize(s)" ] } ], diff --git a/vs-GaussianProcesses_Kriging_julia.ipynb b/vs-GaussianProcesses_Kriging_julia.ipynb index 697874e..54b5645 100644 --- a/vs-GaussianProcesses_Kriging_julia.ipynb +++ b/vs-GaussianProcesses_Kriging_julia.ipynb @@ -30,7 +30,7 @@ "outputs": [], "source": [ "import JLibKriging as LK\n", - "include(\"tools/julia/checks.jl\")\n", + "include(\"tools/julia/gallery.jl\")\n", "s = Suite(\"vs-GaussianProcesses_Kriging_julia\")" ] }, @@ -83,11 +83,11 @@ " Xd = read_data(name, \"X\"); yd = vec(read_data(name, \"y\")); Xt = read_data(name, \"Xt\"); yt = vec(read_data(name, \"yt\"))\n", " t = @elapsed k = LK.Kriging(yd, Xd, \"matern5_2\")\n", " q_lk = q2(yt, LK.predict(k, Xt; return_stdev=false).mean)\n", - " add_metric!(s, name, \"JLibKriging\"; q2=q_lk, fit_s=t)\n", + " metric!(s, name, \"JLibKriging\"; q2=q_lk, fit_s=t)\n", " t = @elapsed k10 = LK.Kriging(yd, Xd, \"matern5_2\"; optim=\"BFGS10\")\n", " q_lk10 = q2(yt, LK.predict(k10, Xt; return_stdev=false).mean)\n", - " add_metric!(s, name, \"JLibKriging BFGS10\"; q2=q_lk10, fit_s=t)\n", - " add_check!(s, \"$name: Q2(JLibKriging BFGS10) > 0.9\", q_lk10 > 0.9; value=q_lk10, expected=\"> 0.9\")\n", + " metric!(s, name, \"JLibKriging BFGS10\"; q2=q_lk10, fit_s=t)\n", + " check!(s, \"$name: Q2(JLibKriging BFGS10) > 0.9\", q_lk10 > 0.9; value=q_lk10, expected=\"> 0.9\")\n", " if have_gp\n", " try\n", " d = size(Xd, 2)\n", @@ -98,15 +98,15 @@ " end\n", " mg, _ = GP.predict_f(gp, permutedims(Xt))\n", " q_gp = q2(yt, mg)\n", - " add_metric!(s, name, \"GaussianProcesses.jl\"; q2=q_gp, fit_s=t)\n", - " add_check!(s, \"$name: Q2(JLibKriging) - Q2(GaussianProcesses.jl)\", q_lk - q_gp > -0.02;\n", + " metric!(s, name, \"GaussianProcesses.jl\"; q2=q_gp, fit_s=t)\n", + " check!(s, \"$name: Q2(JLibKriging) - Q2(GaussianProcesses.jl)\", q_lk - q_gp > -0.02;\n", " value=q_lk - q_gp, expected=\"> -0.02\", severity=\"warn\")\n", " catch e\n", - " add_check!(s, \"$name: GaussianProcesses.jl fit\", false; value=string(typeof(e)), severity=\"warn\")\n", + " check!(s, \"$name: GaussianProcesses.jl fit\", false; value=string(typeof(e)), severity=\"warn\")\n", " end\n", " end\n", "end\n", - "have_gp || add_check!(s, \"GaussianProcesses.jl available\", false; value=\"absent\", severity=\"warn\")\n", + "have_gp || check!(s, \"GaussianProcesses.jl available\", false; value=\"absent\", severity=\"warn\")\n", "foreach(r -> println(join(r[3:6], \" \")), s.metrics)" ] }, @@ -130,7 +130,7 @@ "metadata": {}, "outputs": [], "source": [ - "summary!(s)" + "suite_summary(s)" ] }, { @@ -140,7 +140,7 @@ "metadata": {}, "outputs": [], "source": [ - "finalize!(s)" + "suite_finalize!(s)" ] } ], diff --git a/vs-STK_Kriging_octave.ipynb b/vs-STK_Kriging_octave.ipynb index d992ff6..bb76eec 100644 --- a/vs-STK_Kriging_octave.ipynb +++ b/vs-STK_Kriging_octave.ipynb @@ -37,8 +37,8 @@ "metadata": {}, "outputs": [], "source": [ - "addpath(fullfile(pwd, \"tools\", \"octave\"));\n", - "s = lkb_suite(\"vs-STK_Kriging_octave\");" + "source(\"tools/octave/gallery.m\");\n", + "s = suite(\"vs-STK_Kriging_octave\");" ] }, { @@ -73,9 +73,8 @@ " run(fullfile(stk_path, \"stk_init.m\"));\n", " printf(\"STK %s from %s\\n\", stk_version(), stk_path);\n", "else\n", - " s = lkb_check(s, \"STK available\", false, \"absent\", stk_path, \"STK could not be installed: comparison skipped\", \"warn\");\n", + " s = check(s, \"STK available\", false, \"absent\", stk_path, \"STK could not be installed: comparison skipped\", \"warn\");\n", "end\n", - "q2 = @(y, m) 1 - sum((y - m).^2) / sum((y - mean(y)).^2);\n", "datasets = {\"branin\", \"hartmann3\", \"borehole\"};" ] }, @@ -107,8 +106,8 @@ " model.param = [log(sigma2); -log(theta(:) * sqrt(2))];\n", " model.lm = stk_lm_constant();\n", " zp = stk_predict(model, X, y, Xt);\n", - " s = lkb_close(s, [name \": gauss UK mean == STK\"], m_lk, zp.mean, 1e-6, 1e-8 * std(y));\n", - " s = lkb_close(s, [name \": gauss UK variance == STK\"], s_lk.^2, zp.var, 1e-4, 1e-8 * sigma2);\n", + " s = check_close(s, [name \": gauss UK mean == STK\"], m_lk, zp.mean, 1e-6, 1e-8 * std(y));\n", + " s = check_close(s, [name \": gauss UK variance == STK\"], s_lk.^2, zp.var, 1e-4, 1e-8 * sigma2);\n", " end\n", "end" ] @@ -135,12 +134,12 @@ " tic; k = Kriging(y, X, \"matern5_2\"); t = toc;\n", " [m, ~] = k.predict(Xt, false, false, false);\n", " q_lk = q2(yt, m);\n", - " s = lkb_metric(s, name, \"mLibKriging\", \"q2\", q_lk); s = lkb_metric(s, name, \"mLibKriging\", \"fit_s\", t);\n", + " s = metric(s, name, \"mLibKriging\", \"q2\", q_lk); s = metric(s, name, \"mLibKriging\", \"fit_s\", t);\n", " tic; k10 = Kriging(y, X, \"matern5_2\", \"constant\", false, \"BFGS10\", \"LL\"); t = toc;\n", " [m, ~] = k10.predict(Xt, false, false, false);\n", " q_lk10 = q2(yt, m);\n", - " s = lkb_metric(s, name, \"mLibKriging BFGS10\", \"q2\", q_lk10); s = lkb_metric(s, name, \"mLibKriging BFGS10\", \"fit_s\", t);\n", - " s = lkb_check(s, [name \": Q2(mLibKriging BFGS10) > 0.9\"], q_lk10 > 0.9, q_lk10, \"> 0.9\");\n", + " s = metric(s, name, \"mLibKriging BFGS10\", \"q2\", q_lk10); s = metric(s, name, \"mLibKriging BFGS10\", \"fit_s\", t);\n", + " s = check(s, [name \": Q2(mLibKriging BFGS10) > 0.9\"], q_lk10 > 0.9, q_lk10, \"> 0.9\");\n", " if have_stk\n", " tic;\n", " model = stk_model(@stk_materncov52_aniso, d);\n", @@ -148,8 +147,8 @@ " zp = stk_predict(model, X, y, Xt);\n", " t = toc;\n", " q_stk = q2(yt, zp.mean);\n", - " s = lkb_metric(s, name, \"STK\", \"q2\", q_stk); s = lkb_metric(s, name, \"STK\", \"fit_s\", t);\n", - " s = lkb_check(s, [name \": Q2(mLibKriging) - Q2(STK)\"], q_lk - q_stk > -0.02, q_lk - q_stk, \"> -0.02\", \"\", \"warn\");\n", + " s = metric(s, name, \"STK\", \"q2\", q_stk); s = metric(s, name, \"STK\", \"fit_s\", t);\n", + " s = check(s, [name \": Q2(mLibKriging) - Q2(STK)\"], q_lk - q_stk > -0.02, q_lk - q_stk, \"> -0.02\", \"\", \"warn\");\n", " end\n", "end\n", "disp(s.metrics(:, 3:6))" @@ -175,7 +174,7 @@ "metadata": {}, "outputs": [], "source": [ - "lkb_summary(s);" + "suite_summary(s);" ] }, { @@ -185,7 +184,7 @@ "metadata": {}, "outputs": [], "source": [ - "lkb_finalize(s);" + "suite_finalize(s);" ] } ], diff --git a/vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python.ipynb b/vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python.ipynb index b76b8d6..d003e7f 100644 --- a/vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python.ipynb +++ b/vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python.ipynb @@ -40,7 +40,7 @@ "source": [ "# Setup for running this notebook outside the repository (Colab, Binder, a lone copy):\n", "# nothing to do in the repository itself (CI, local runs). Otherwise the repository\n", - "# (datasets of data/, helpers of tools/) is cloned and the released pylibkriging installed.\n", + "# (datasets of data/, helpers of tools/python/gallery.py) is cloned and the released pylibkriging installed.\n", "import importlib\n", "import os\n", "import subprocess\n", @@ -62,7 +62,7 @@ " \"(PyPI has wheels for CPython 3.10 to 3.12 only). On Colab: Runtime > Change runtime version > \"\n", " \"2026.07 (Python 3.12), then run again.\\n\" + r.stderr[-500:])\n", " importlib.invalidate_caches()\n", - "sys.path.insert(0, str(Path(\"tools\").resolve())) # lkgallery, when not installed (Colab, Binder)\n", + "sys.path.insert(0, str(Path(\"tools/python\").resolve())) # for \"import gallery\" (tools/python/gallery.py)\n", "print(\"working directory:\", os.getcwd())" ] }, @@ -114,11 +114,11 @@ "import pandas as pd\n", "import pylibkriging as lk\n", "\n", - "import lkgallery\n", - "from lkgallery import TESTS, design, metrics, compat\n", + "import gallery\n", + "from gallery import TESTS, design, metrics, compat\n", "\n", "warnings.filterwarnings(\"ignore\")\n", - "suite = lkgallery.Suite(\"vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python\")\n", + "suite = gallery.Suite(\"vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python\")\n", "sk = compat.optional(\"sklearn.gaussian_process\")\n", "smt = compat.optional(\"smt.surrogate_models\")\n", "ot = compat.optional(\"openturns\")\n", @@ -163,9 +163,9 @@ " kern = ConstantKernel(sigma2, \"fixed\") * RBF(theta, \"fixed\")\n", " gp = GaussianProcessRegressor(kern, alpha=1e-12, optimizer=None, normalize_y=False).fit(X, y - beta)\n", " m_sk, s_sk = gp.predict(Xt, return_std=True)\n", - " suite.close(\"sklearn RBF == libKriging gauss: simple kriging mean\", m_sk + beta, m_lk, rtol=1e-6,\n", + " suite.check_close(\"sklearn RBF == libKriging gauss: simple kriging mean\", m_sk + beta, m_lk, rtol=1e-6,\n", " atol=1e-7 * np.std(y))\n", - " suite.close(\"sklearn RBF == libKriging gauss: simple kriging stdev (beta fixed)\", s_sk, s_lk, rtol=1e-4,\n", + " suite.check_close(\"sklearn RBF == libKriging gauss: simple kriging stdev (beta fixed)\", s_sk, s_lk, rtol=1e-4,\n", " atol=1e-6 * np.sqrt(sigma2))\n", " # with beta estimated, libKriging's stdev is the universal kriging one: the simple kriging variance\n", " # plus the trend-estimation term (f - F'C^-1 c)^2 / (F'C^-1 F), computed here by hand\n", @@ -174,14 +174,14 @@ " one = np.ones((len(y), 1))\n", " v_uk = s_sk ** 2 + ((1 - one.T @ w) ** 2).ravel() / (one.T @ np.linalg.solve(C, one)).item()\n", " s_uk = np.ravel(kg.predict(Xt, return_stdev=True)[1])\n", - " suite.close(\"libKriging gauss (beta estimated) stdev == universal kriging formula\", s_uk, np.sqrt(v_uk),\n", + " suite.check_close(\"libKriging gauss (beta estimated) stdev == universal kriging formula\", s_uk, np.sqrt(v_uk),\n", " rtol=1e-4, atol=1e-6 * np.sqrt(sigma2))\n", " # the (concentrated) log-likelihood at the MLE equals the full Gaussian log-density at (theta, sigma2, beta)\n", " lml = gp.log_marginal_likelihood_value_\n", - " suite.close(\"sklearn log-marginal-likelihood == libKriging logLikelihood()\", lml, kg.logLikelihood(),\n", + " suite.check_close(\"sklearn log-marginal-likelihood == libKriging logLikelihood()\", lml, kg.logLikelihood(),\n", " rtol=1e-6)\n", "else:\n", - " suite.skip(\"sklearn exact comparison\", \"scikit-learn not installed\")" + " suite.check_skip(\"sklearn exact comparison\", \"scikit-learn not installed\")" ] }, { @@ -209,12 +209,12 @@ " parameters={\"theta\": th.reshape(1, -1), \"is_theta_estim\": False,\n", " \"sigma2\": s2, \"is_sigma2_estim\": False})\n", " m_u, s_u = (np.ravel(a) for a in ku.predict(Xt, return_stdev=True)[:2])\n", - " suite.close(\"OpenTURNS product Matern 5/2 == libKriging matern5_2: mean\", m_ot, m_u, rtol=1e-6,\n", + " suite.check_close(\"OpenTURNS product Matern 5/2 == libKriging matern5_2: mean\", m_ot, m_u, rtol=1e-6,\n", " atol=1e-6 * np.std(y))\n", - " suite.close(\"OpenTURNS product Matern 5/2 == libKriging matern5_2: variance (universal kriging)\", v_ot, s_u ** 2,\n", + " suite.check_close(\"OpenTURNS product Matern 5/2 == libKriging matern5_2: variance (universal kriging)\", v_ot, s_u ** 2,\n", " rtol=1e-4, atol=1e-8 * s2)\n", "else:\n", - " suite.skip(\"OpenTURNS exact comparison\", \"openturns not installed\")" + " suite.check_skip(\"OpenTURNS exact comparison\", \"openturns not installed\")" ] }, { @@ -309,7 +309,7 @@ " except Exception as exc: # noqa: BLE001\n", " rows.append(dict(function=fname, n=n, rep=rep, package=pkg, error=str(exc)[:100]))\n", " if pkg.startswith(\"pylibkriging\"):\n", - " suite.true(f\"{fname}/rep{rep}/{pkg}: fit\", False, detail=str(exc)[:200])\n", + " suite.check_true(f\"{fname}/rep{rep}/{pkg}: fit\", False, detail=str(exc)[:200])\n", "res = pd.DataFrame(rows)\n", "summary = res.groupby([\"function\", \"package\"], sort=False)[[\"q2\", \"nlpd\", \"coverage95\", \"fit_s\"]].median()\n", "for (fname, pkg), r in summary.iterrows():\n", @@ -351,11 +351,11 @@ " s = summary.loc[fname][\"q2\"]\n", " others = s.drop(index=[p for p in s.index if p.startswith(\"pylibkriging\")])\n", " if len(others):\n", - " suite.greater(f\"{fname}: Q2(pylibkriging) - best other package\", s[\"pylibkriging\"] - others.max(), -0.02,\n", + " suite.check_greater(f\"{fname}: Q2(pylibkriging) - best other package\", s[\"pylibkriging\"] - others.max(), -0.02,\n", " severity=\"warn\", detail=f\"best: {others.idxmax()}\")\n", - " suite.greater(f\"{fname}: Q2(pylibkriging BFGS10) - best other package\", s[\"pylibkriging BFGS10\"] - others.max(),\n", + " suite.check_greater(f\"{fname}: Q2(pylibkriging BFGS10) - best other package\", s[\"pylibkriging BFGS10\"] - others.max(),\n", " -0.05, detail=f\"best: {others.idxmax()}\") if len(others) else None\n", - " suite.greater(f\"{fname}: Q2(pylibkriging BFGS10) > 0.5\", s[\"pylibkriging BFGS10\"], 0.5)" + " suite.check_greater(f\"{fname}: Q2(pylibkriging BFGS10) > 0.5\", s[\"pylibkriging BFGS10\"], 0.5)" ] }, { diff --git a/warpings-categorical-ordinal_WarpKriging_python.ipynb b/warpings-categorical-ordinal_WarpKriging_python.ipynb index b32e75c..187e205 100644 --- a/warpings-categorical-ordinal_WarpKriging_python.ipynb +++ b/warpings-categorical-ordinal_WarpKriging_python.ipynb @@ -38,7 +38,7 @@ "source": [ "# Setup for running this notebook outside the repository (Colab, Binder, a lone copy):\n", "# nothing to do in the repository itself (CI, local runs). Otherwise the repository\n", - "# (datasets of data/, helpers of tools/) is cloned and the released pylibkriging installed.\n", + "# (datasets of data/, helpers of tools/python/gallery.py) is cloned and the released pylibkriging installed.\n", "import importlib\n", "import os\n", "import subprocess\n", @@ -60,7 +60,7 @@ " \"(PyPI has wheels for CPython 3.10 to 3.12 only). On Colab: Runtime > Change runtime version > \"\n", " \"2026.07 (Python 3.12), then run again.\\n\" + r.stderr[-500:])\n", " importlib.invalidate_caches()\n", - "sys.path.insert(0, str(Path(\"tools\").resolve())) # lkgallery, when not installed (Colab, Binder)\n", + "sys.path.insert(0, str(Path(\"tools/python\").resolve())) # for \"import gallery\" (tools/python/gallery.py)\n", "print(\"working directory:\", os.getcwd())" ] }, @@ -76,13 +76,13 @@ "import matplotlib.pyplot as plt\n", "import pylibkriging as lk\n", "\n", - "import lkgallery\n", - "from lkgallery import TESTS, design, metrics, compat\n", + "import gallery\n", + "from gallery import TESTS, design, metrics, compat\n", "\n", - "suite = lkgallery.Suite(\"warpings-categorical-ordinal_WarpKriging_python\")\n", + "suite = gallery.Suite(\"warpings-categorical-ordinal_WarpKriging_python\")\n", "print(\"pylibkriging\", compat.version())\n", "if not compat.has(\"WarpKriging\"):\n", - " suite.skip(\"WarpKriging\", \"not available in this pylibkriging\")\n", + " suite.check_skip(\"WarpKriging\", \"not available in this pylibkriging\")\n", "\n", "\n", "def pred(k, X):\n", @@ -100,13 +100,13 @@ " if label.startswith(\"Kriging\"):\n", " BASE_NLPD[case] = res[\"nlpd\"]\n", " else:\n", - " suite.less(f\"{case}/{label}: NLPD - NLPD(Kriging) (overconfident stdev?)\", res[\"nlpd\"] - BASE_NLPD[case],\n", + " suite.check_less(f\"{case}/{label}: NLPD - NLPD(Kriging) (overconfident stdev?)\", res[\"nlpd\"] - BASE_NLPD[case],\n", " 2.0, severity=\"warn\", detail=f\"min stdev {np.min(s):.3g}, max |error|/stdev \"\n", " f\"{np.max(np.abs(yt - m) / np.maximum(s, 1e-300)):.3g}\")\n", " suite.metric(case, label, ll=float(k.logLikelihood()), **res)\n", - " suite.finite(f\"{case}/{label}: predictions finite\", m)\n", + " suite.check_finite(f\"{case}/{label}: predictions finite\", m)\n", " mX, _ = pred(k, X)\n", - " suite.less(f\"{case}/{label}: interpolates (max |m(X)-y| / sd(y))\", np.max(np.abs(mX - y)) / np.std(y), 1e-3,\n", + " suite.check_less(f\"{case}/{label}: interpolates (max |m(X)-y| / sd(y))\", np.max(np.abs(mX - y)) / np.std(y), 1e-3,\n", " severity=\"warn\")\n", " return res" ] @@ -132,9 +132,9 @@ " for kernel in [\"gauss\", \"matern5_2\"]:\n", " kp = lk.Kriging(y, X, kernel)\n", " kw = lk.WarpKriging(y, X, [\"none\", \"none\"], kernel)\n", - " suite.close(f\"{kernel}: WarpKriging(none) LL == Kriging LL\", kw.logLikelihood(), kp.logLikelihood(),\n", + " suite.check_close(f\"{kernel}: WarpKriging(none) LL == Kriging LL\", kw.logLikelihood(), kp.logLikelihood(),\n", " rtol=1e-3, severity=\"warn\")\n", - " suite.close(f\"{kernel}: WarpKriging(none) mean == Kriging mean\", pred(kw, Xt)[0], pred(kp, Xt)[0],\n", + " suite.check_close(f\"{kernel}: WarpKriging(none) mean == Kriging mean\", pred(kw, Xt)[0], pred(kp, Xt)[0],\n", " rtol=1e-2, atol=1e-2 * np.std(y), severity=\"warn\")" ] }, @@ -162,12 +162,12 @@ " base = evaluate(\"gl2012\", \"Kriging\", lk.Kriging(y, X, \"matern5_2\"), X, y, Xt, yt)\n", " fits = {}\n", " for w in WARPS_1D:\n", - " with suite.runs(f\"gl2012/{w}: fit\"):\n", + " with suite.check_runs(f\"gl2012/{w}: fit\"):\n", " fits[w] = lk.WarpKriging(y, X, [w], \"matern5_2\")\n", " evaluate(\"gl2012\", w, fits[w], X, y, Xt, yt)\n", " t = suite.table().loc[\"gl2012\"]\n", " best = t.drop(index=\"Kriging\")[\"rmse\"].min()\n", - " suite.less(\"gl2012: best warp RMSE / Kriging RMSE\", best / base[\"rmse\"], 1.0, severity=\"warn\")\n", + " suite.check_less(\"gl2012: best warp RMSE / Kriging RMSE\", best / base[\"rmse\"], 1.0, severity=\"warn\")\n", " u = np.linspace(0.01, 0.99, 400)[:, None]\n", " fig, ax = plt.subplots(figsize=(11, 4))\n", " ax.plot(u, tf.unit(u), \"k--\", lw=2, label=\"truth\")\n", @@ -200,10 +200,10 @@ " X = X * 0.98 + 0.01; Xt = Xt * 0.98 + 0.01\n", " y, yt = tf.unit(X), tf.unit(Xt)\n", " base = evaluate(\"gl2008\", \"Kriging\", lk.Kriging(y, X, \"matern5_2\"), X, y, Xt, yt)\n", - " with suite.runs(\"gl2008/kumaraswamy: fit\"):\n", + " with suite.check_runs(\"gl2008/kumaraswamy: fit\"):\n", " res = evaluate(\"gl2008\", \"kumaraswamy\", lk.WarpKriging(y, X, [\"kumaraswamy\"] * 2, \"matern5_2\"),\n", " X, y, Xt, yt)\n", - " suite.less(\"gl2008: RMSE(kumaraswamy) / RMSE(Kriging)\", res[\"rmse\"] / base[\"rmse\"], 1.0, severity=\"warn\")" + " suite.check_less(\"gl2008: RMSE(kumaraswamy) / RMSE(Kriging)\", res[\"rmse\"] / base[\"rmse\"], 1.0, severity=\"warn\")" ] }, { @@ -243,12 +243,12 @@ " X = mixed_design(60, 34); Xt = mixed_design(1000, 35)\n", " y, yt = mixed(X), mixed(Xt)\n", " base = evaluate(\"mixed\", \"Kriging (level as number)\", lk.Kriging(y, X, \"matern5_2\"), X, y, Xt, yt)\n", - " with suite.runs(\"mixed/categorical: fit\"):\n", + " with suite.check_runs(\"mixed/categorical: fit\"):\n", " kc = lk.WarpKriging(y, X, [f\"categorical({L},2)\", \"none\", \"none\"], \"matern5_2\")\n", " res = evaluate(\"mixed\", \"categorical(5,2)\", kc, X, y, Xt, yt)\n", - " suite.less(\"mixed: RMSE(categorical) / RMSE(level as number)\", res[\"rmse\"] / base[\"rmse\"], 1.0,\n", + " suite.check_less(\"mixed: RMSE(categorical) / RMSE(level as number)\", res[\"rmse\"] / base[\"rmse\"], 1.0,\n", " severity=\"warn\")\n", - " suite.greater(\"mixed/categorical: Q2\", res[\"q2\"], 0.8, severity=\"warn\")" + " suite.check_greater(\"mixed/categorical: Q2\", res[\"q2\"], 0.8, severity=\"warn\")" ] }, { @@ -285,10 +285,10 @@ " y, yt = f_ord(X), f_ord(Xt)\n", " base = evaluate(\"ordinal\", \"Kriging (rank as number)\", lk.Kriging(y, X / [5, 1, 1], \"matern5_2\"),\n", " X / [5, 1, 1], y, Xt / [5, 1, 1], yt)\n", - " with suite.runs(\"ordinal/ordinal(6): fit\"):\n", + " with suite.check_runs(\"ordinal/ordinal(6): fit\"):\n", " ko = lk.WarpKriging(y, X, [\"ordinal(6)\", \"none\", \"none\"], \"matern5_2\")\n", " res = evaluate(\"ordinal\", \"ordinal(6)\", ko, X, y, Xt, yt)\n", - " suite.less(\"ordinal: RMSE(ordinal) / RMSE(rank as number)\", res[\"rmse\"] / base[\"rmse\"], 1.1,\n", + " suite.check_less(\"ordinal: RMSE(ordinal) / RMSE(rank as number)\", res[\"rmse\"] / base[\"rmse\"], 1.1,\n", " severity=\"warn\")" ] }, @@ -323,8 +323,8 @@ "source": [ "if compat.has(\"WarpKriging\"):\n", " X, y, Xt, yt = design.dataset(TESTS[\"hartmann3\"], 40, n_test=100, seed=61)\n", - " with suite.runs(\"WarpKriging: fit (persistence)\"):\n", - " lkgallery.persistence.check(suite, \"WarpKriging\", lk.WarpKriging(y, X, [\"kumaraswamy\", \"none\", \"affine\"], \"matern5_2\"), Xt, pred)" + " with suite.check_runs(\"WarpKriging: fit (persistence)\"):\n", + " gallery.persistence.check(suite, \"WarpKriging\", lk.WarpKriging(y, X, [\"kumaraswamy\", \"none\", \"affine\"], \"matern5_2\"), Xt, pred)" ] }, { From 91bb2516d33cdc8c402003e34e8e7a1a63880cc7 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 15 Oct 2026 21:51:35 +0000 Subject: [PATCH 10/16] Notebook sources in src/, notebooks generated and executed by CI only, gallery site on gh-pages - src/ at the repository root; no committed notebook (/*.ipynb ignored) - every source carries a **Tags.** paragraph (class, kernel, trend, objective, method, noise, warping, aggregation, compared to, topic); py2nb --check validates name, title, idea, reference, tags and suite name, and puts them in the notebook metadata - CI: matrix from src/, each job generates its notebook before executing it; new site job builds tools/site.py from the executed master notebooks (artifact 'site') and force-pushes it to gh-pages from main - tools/site.py: cards with first figure, title, idea, tags, check status, tag filters and search; card opens Colab (Python) or nbviewer (others) - README: links to sources, site, gh-pages notebooks Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01QiFEE514dpwS57mcFWDeZG --- .github/workflows/notebooks.yml | 88 +++- .gitignore | 2 + README.md | 125 +++--- ...gations-large-n_NestedKriging_python.ipynb | 309 ------------- ...binding-reference_all-classes_python.ipynb | 288 ------------- cross-binding_all-classes_R.ipynb | 203 --------- cross-binding_all-classes_julia.ipynb | 200 --------- cross-binding_all-classes_octave.ipynb | 218 ---------- ...l-projected-branin_MLPKriging_python.ipynb | 207 --------- ...n-normalize-screening_Kriging_python.ipynb | 225 ---------- ...-trends-interpolation_Kriging_python.ipynb | 363 ---------------- large-n_Kriging-vecchia-nystrom_python.ipynb | 302 ------------- noisy-data_Kriging-nugget-noise_python.ipynb | 305 ------------- ...-LL-LOO-LMP-gradients_Kriging_python.ipynb | 379 ---------------- ...stness-invalid-inputs_Kriging_python.ipynb | 333 -------------- ...pickle-copy-accessors_Kriging_python.ipynb | 239 ---------- sklearn-estimators_Kriging_python.ipynb | 222 ---------- ...gregations-large-n_NestedKriging_python.py | 3 + ...ss-binding-reference_all-classes_python.py | 4 + .../src => src}/cross-binding_all-classes_R.R | 4 + .../cross-binding_all-classes_julia.jl | 4 + .../cross-binding_all-classes_octave.m | 4 + ...rnel-projected-branin_MLPKriging_python.py | 3 + ...sion-normalize-screening_Kriging_python.py | 3 + ...els-trends-interpolation_Kriging_python.py | 3 + .../large-n_Kriging-vecchia-nystrom_python.py | 2 + .../noisy-data_Kriging-nugget-noise_python.py | 3 + ...ves-LL-LOO-LMP-gradients_Kriging_python.py | 2 + ...obustness-invalid-inputs_Kriging_python.py | 2 + ...ad-pickle-copy-accessors_Kriging_python.py | 3 + .../sklearn-estimators_Kriging_python.py | 2 + ...ctions-reference-values_no-model_python.py | 2 + .../update-simulate_Kriging_python.py | 2 + .../vs-DiceKriging-RobustGaSP_Kriging_R.R | 3 + .../vs-GaussianProcesses_Kriging_julia.jl | 2 + {tools/src => src}/vs-STK_Kriging_octave.m | 2 + ...klearn-SMT-OpenTURNS-GPy_Kriging_python.py | 3 + ...-categorical-ordinal_WarpKriging_python.py | 4 + ...ons-reference-values_no-model_python.ipynb | 233 ---------- tools/py2nb.py | 131 ++++-- tools/report.py | 6 +- tools/run_notebooks.py | 2 +- tools/site.py | 293 +++++++++++++ update-simulate_Kriging_python.ipynb | 322 -------------- vs-DiceKriging-RobustGaSP_Kriging_R.ipynb | 230 ---------- vs-GaussianProcesses_Kriging_julia.ipynb | 159 ------- vs-STK_Kriging_octave.ipynb | 203 --------- ...arn-SMT-OpenTURNS-GPy_Kriging_python.ipynb | 407 ------------------ ...tegorical-ordinal_WarpKriging_python.ipynb | 376 ---------------- 49 files changed, 618 insertions(+), 5812 deletions(-) delete mode 100644 aggregations-large-n_NestedKriging_python.ipynb delete mode 100644 cross-binding-reference_all-classes_python.ipynb delete mode 100644 cross-binding_all-classes_R.ipynb delete mode 100644 cross-binding_all-classes_julia.ipynb delete mode 100644 cross-binding_all-classes_octave.ipynb delete mode 100644 deep-kernel-projected-branin_MLPKriging_python.ipynb delete mode 100644 high-dimension-normalize-screening_Kriging_python.ipynb delete mode 100644 kernels-trends-interpolation_Kriging_python.ipynb delete mode 100644 large-n_Kriging-vecchia-nystrom_python.ipynb delete mode 100644 noisy-data_Kriging-nugget-noise_python.ipynb delete mode 100644 objectives-LL-LOO-LMP-gradients_Kriging_python.ipynb delete mode 100644 robustness-invalid-inputs_Kriging_python.ipynb delete mode 100644 save-load-pickle-copy-accessors_Kriging_python.ipynb delete mode 100644 sklearn-estimators_Kriging_python.ipynb rename {tools/src => src}/aggregations-large-n_NestedKriging_python.py (98%) rename {tools/src => src}/cross-binding-reference_all-classes_python.py (97%) rename {tools/src => src}/cross-binding_all-classes_R.R (95%) rename {tools/src => src}/cross-binding_all-classes_julia.jl (95%) rename {tools/src => src}/cross-binding_all-classes_octave.m (95%) rename {tools/src => src}/deep-kernel-projected-branin_MLPKriging_python.py (97%) rename {tools/src => src}/high-dimension-normalize-screening_Kriging_python.py (97%) rename {tools/src => src}/kernels-trends-interpolation_Kriging_python.py (98%) rename {tools/src => src}/large-n_Kriging-vecchia-nystrom_python.py (99%) rename {tools/src => src}/noisy-data_Kriging-nugget-noise_python.py (98%) rename {tools/src => src}/objectives-LL-LOO-LMP-gradients_Kriging_python.py (99%) rename {tools/src => src}/robustness-invalid-inputs_Kriging_python.py (99%) rename {tools/src => src}/save-load-pickle-copy-accessors_Kriging_python.py (97%) rename {tools/src => src}/sklearn-estimators_Kriging_python.py (98%) rename {tools/src => src}/test-functions-reference-values_no-model_python.py (98%) rename {tools/src => src}/update-simulate_Kriging_python.py (99%) rename {tools/src => src}/vs-DiceKriging-RobustGaSP_Kriging_R.R (98%) rename {tools/src => src}/vs-GaussianProcesses_Kriging_julia.jl (97%) rename {tools/src => src}/vs-STK_Kriging_octave.m (98%) rename {tools/src => src}/vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python.py (99%) rename {tools/src => src}/warpings-categorical-ordinal_WarpKriging_python.py (98%) delete mode 100644 test-functions-reference-values_no-model_python.ipynb create mode 100644 tools/site.py delete mode 100644 update-simulate_Kriging_python.ipynb delete mode 100644 vs-DiceKriging-RobustGaSP_Kriging_R.ipynb delete mode 100644 vs-GaussianProcesses_Kriging_julia.ipynb delete mode 100644 vs-STK_Kriging_octave.ipynb delete mode 100644 vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python.ipynb delete mode 100644 warpings-categorical-ordinal_WarpKriging_python.ipynb diff --git a/.github/workflows/notebooks.yml b/.github/workflows/notebooks.yml index d9bcb92..f04e0bc 100644 --- a/.github/workflows/notebooks.yml +++ b/.github/workflows/notebooks.yml @@ -1,10 +1,12 @@ name: notebooks -# Executes every notebook of the repository root against libKriging, one job per -# notebook, each job uploading its executed notebook (outputs saved, failures -# included) and its results as the artifact `executed--`; then -# aggregates the checks of all languages into one report per channel -# (tools/report.py), published as the job summary and as artifacts. +# Generates every notebook from its source (src/, tools/py2nb.py) and executes it +# against libKriging, one job per notebook, each job uploading its executed notebook +# (outputs saved, failures included) and its results as the artifact +# `executed--`; then aggregates the checks of all languages into +# one report per channel (tools/report.py), published as the job summary and as +# artifacts. Finally builds the gallery site from the executed notebooks of the +# master channel (tools/site.py), published on the gh-pages branch from main. # # Channels: # master libKriging and its bindings built from source at `libkriging_ref` @@ -78,11 +80,11 @@ jobs: - uses: actions/setup-python@v5 with: python-version: ${{ env.PYTHON_VERSION }} - - name: Notebooks in sync with their sources, and listed in the README + - name: Notebook sources valid (name, title, idea, reference, tags), and listed in the README run: | pip install nbformat python tools/py2nb.py --check - missing=$(for nb in *.ipynb; do grep -qF "($nb)" README.md || echo "$nb"; done) + missing=$(for f in src/*; do grep -qF "(src/${f#src/})" README.md || echo "$f"; done) if [ -n "$missing" ]; then echo "not linked from README.md: $missing"; exit 1; fi - name: Resolve libKriging refs and channels id: refs @@ -113,8 +115,8 @@ jobs: channels = os.environ["CHANNELS"].split(",") for lang, binding in [("python", "python"), ("r", "R"), ("octave", "octave"), ("julia", "julia")]: chans = [c for c in channels if c == "master" or lang != "julia"] # no Julia release - include = [{"channel": c, "notebook": nb.stem} for c in chans - for nb in sorted(Path(".").glob(f"*_{binding}.ipynb"))] + include = [{"channel": c, "notebook": src.stem, "source": src.name} for c in chans + for src in sorted(Path("src").glob(f"*_{binding}.*"))] print(f"{lang}={json.dumps({'include': include or [{'channel': 'none', 'notebook': 'none'}]})}") EOF cat "$GITHUB_OUTPUT" @@ -275,6 +277,10 @@ jobs: path: dist - name: Install pylibkriging (${{ matrix.channel }}) run: tools/install/python.sh ${{ matrix.channel }} dist + - name: Generate the notebook from src/${{ matrix.source }} + run: | + python -m pip install --quiet nbformat + python tools/py2nb.py "src/${{ matrix.source }}" - name: Execute and save the notebook run: python tools/run_notebooks.py --inplace --timeout 3600 ${{ matrix.notebook }}.ipynb - if: always() @@ -320,6 +326,10 @@ jobs: tools/install/r.sh release "$RUNNER_TEMP/Rlib" echo "R_LIBS_USER=$RUNNER_TEMP/Rlib" >> "$GITHUB_ENV" fi + - name: Generate the notebook from src/${{ matrix.source }} + run: | + python -m pip install --quiet nbformat + python tools/py2nb.py "src/${{ matrix.source }}" - name: Execute and save the notebook run: python tools/run_notebooks.py --inplace --timeout 3600 ${{ matrix.notebook }}.ipynb - if: always() @@ -354,6 +364,10 @@ jobs: sudo apt-get update && sudo apt-get install -y octave liblapack-dev libopenblas-dev tools/install/octave.sh mlibkriging mlibkriging # already built: only the kernel and a smoke test echo "OCTAVE_PATH=$PWD/mlibkriging" >> "$GITHUB_ENV" + - name: Generate the notebook from src/${{ matrix.source }} + run: | + python -m pip install --quiet nbformat + python tools/py2nb.py "src/${{ matrix.source }}" - name: Execute and save the notebook run: python tools/run_notebooks.py --inplace --timeout 3600 ${{ matrix.notebook }}.ipynb - if: always() @@ -398,6 +412,10 @@ jobs: run: | sudo apt-get update && sudo apt-get install -y liblapack-dev libopenblas-dev tools/install/julia.sh libKriging + - name: Generate the notebook from src/${{ matrix.source }} + run: | + python -m pip install --quiet nbformat + python tools/py2nb.py "src/${{ matrix.source }}" - name: Execute and save the notebook run: python tools/run_notebooks.py --inplace --timeout 3600 --kernel julia-${{ env.JULIA_VERSION }} ${{ matrix.notebook }}.ipynb - if: always() @@ -463,3 +481,55 @@ jobs: name: executed-notebooks-${{ matrix.channel }} path: executed/ retention-days: 90 + + # --------------------------------------------------------------------------- site + # The gallery site (tools/site.py): one card per notebook, from the executed notebooks + # of the master channel (failed ones included: their card shows it). Built on every + # run (artifact `site`, for a preview), published as the gh-pages branch (a single + # commit, replaced each time) only from main and from libKriging/libKriging itself. + site: + needs: [setup, python, r, octave, julia] + if: always() && needs.setup.result == 'success' && contains(needs.setup.outputs.channels, 'master') + runs-on: ubuntu-24.04 + permissions: + contents: write + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + - uses: actions/download-artifact@v4 + with: + pattern: executed-master-* + path: artifacts + - name: Build the site + env: + SHA: ${{ needs.setup.outputs.sha }} + run: | + pip install nbformat + mkdir -p results executed + find artifacts -path '*/results/*' -name '*.checks.csv' -exec cp {} results/ \; + find artifacts -maxdepth 2 -name '*.ipynb' -exec cp {} executed/ \; + python tools/site.py --notebooks executed --results results --out _site \ + --repo "${{ github.repository }}" --branch gh-pages \ + --libkriging "${LIBKRIGING_REPO}@${LIBKRIGING_REF} (${SHA:0:10})" + - uses: actions/upload-artifact@v4 + with: + name: site + path: _site/ + include-hidden-files: true + retention-days: 30 + - name: Publish on gh-pages + if: >- + github.ref == 'refs/heads/main' && github.event_name != 'pull_request' + && env.LIBKRIGING_REPO == 'libKriging/libKriging' + env: + GH_TOKEN: ${{ github.token }} + run: | + cd _site + git init -q -b gh-pages + git add -A + git -c user.name='github-actions[bot]' -c user.email='41898282+github-actions[bot]@users.noreply.github.com' \ + commit -q -m "Gallery site, ${{ github.sha }}, libKriging ${LIBKRIGING_REF} (${{ needs.setup.outputs.sha }})" + git push -f "https://x-access-token:${GH_TOKEN}@github.com/${{ github.repository }}.git" gh-pages + echo "Site published: https://${{ github.repository_owner }}.github.io/${{ github.event.repository.name }}/" >> "$GITHUB_STEP_SUMMARY" diff --git a/.gitignore b/.gitignore index 24aa0c7..c0e3251 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,5 @@ __pycache__/ new-baseline/ executed/ report.md +/*.ipynb +_site/ diff --git a/README.md b/README.md index de0ce1b..b786930 100644 --- a/README.md +++ b/README.md @@ -12,10 +12,17 @@ CI runs every notebook against the **development head of libKriging and of its bindings** (and weekly against the releases), saves it with its outputs, and turns the checks into a report: the gallery is libKriging's end-to-end acceptance test. -Every notebook starts with its **idea** and its **reference**, and ends with a -**summary** of its checks; error-level checks fail the run, warn-level ones are +Every notebook starts with its **idea**, its **reference** and its **tags**, and ends +with a **summary** of its checks; error-level checks fail the run, warn-level ones are reported (see [Checks and statuses](#checks-and-statuses)). +**Gallery site: **: one card per notebook +(first figure, title, idea, tags: language, Kriging class, kernel, objective, method, +noise/nugget, warping, aggregation, compared packages; status of its checks), tag +filters; a card opens its notebook, executed by the last CI run on `main`, in Colab +(Python) or nbviewer (R, Octave, Julia). Built by [`tools/site.py`](tools/site.py), +published on the `gh-pages` branch. + ## The examples **executed**: the notebook as run by the last CI run on `main` against libKriging @@ -25,7 +32,10 @@ stable URL; [all runs on `main`](https://github.com/libKriging/gallery/actions/w keep them 90 days; all of a run at once: [executed-notebooks-master](https://nightly.link/libKriging/gallery/workflows/notebooks/main/executed-notebooks-master.zip), and the aggregated report: [report-master](https://nightly.link/libKriging/gallery/workflows/notebooks/main/report-master.zip)). -**open in**: run it in a free hosted service (see [below](#running-a-notebook-in-colab-or-binder)). +**open in**: the executed notebook of the site, run in a free hosted service (see +[below](#running-a-notebook-in-colab-or-binder)), or read in nbviewer. The +**notebook** column links to its source in [`src/`](src): the notebooks themselves +are generated and executed by CI only, never committed. One binding per use case: Python, the most complete binding, unless the reference belongs to another language (DiceKriging and RobustGaSP in R, STK in Octave, @@ -36,74 +46,75 @@ binding gives the same numbers as Python, for every class and fit method. | notebook | binding | kriging type | what it shows | validated against | executed | open in | |---|---|---|---|---|---|---| -| [update-simulate_Kriging_python](update-simulate_Kriging_python.ipynb) | python | `Kriging` | the workflow on 1D to 6D functions: fit, `predict`, conditional paths, `update` with new data, `update_simulate`, in pictures | fit from scratch, Monte Carlo moments | [zip](https://nightly.link/libKriging/gallery/workflows/notebooks/main/executed-master-update-simulate_Kriging_python.zip) | [Colab](https://colab.research.google.com/github/libKriging/gallery/blob/main/update-simulate_Kriging_python.ipynb) · [Binder](https://mybinder.org/v2/gh/libKriging/gallery/main?labpath=update-simulate_Kriging_python.ipynb) | -| [kernels-trends-interpolation_Kriging_python](kernels-trends-interpolation_Kriging_python.ipynb) | python | `Kriging` | choosing a kernel and a trend: 4 × 4 combinations on 8 functions, interpolation, predicted covariance and derivatives | finite differences, Q² floors | [zip](https://nightly.link/libKriging/gallery/workflows/notebooks/main/executed-master-kernels-trends-interpolation_Kriging_python.zip) | [Colab](https://colab.research.google.com/github/libKriging/gallery/blob/main/kernels-trends-interpolation_Kriging_python.ipynb) · [Binder](https://mybinder.org/v2/gh/libKriging/gallery/main?labpath=kernels-trends-interpolation_Kriging_python.ipynb) | -| [objectives-LL-LOO-LMP-gradients_Kriging_python](objectives-LL-LOO-LMP-gradients_Kriging_python.ipynb) | python | `Kriging` | choosing the estimation criterion (`LL`, `LOO`, `LMP`), its profile around θ̂, multistart | finite differences, explicit leave-one-out refits | [zip](https://nightly.link/libKriging/gallery/workflows/notebooks/main/executed-master-objectives-LL-LOO-LMP-gradients_Kriging_python.zip) | [Colab](https://colab.research.google.com/github/libKriging/gallery/blob/main/objectives-LL-LOO-LMP-gradients_Kriging_python.ipynb) · [Binder](https://mybinder.org/v2/gh/libKriging/gallery/main?labpath=objectives-LL-LOO-LMP-gradients_Kriging_python.ipynb) | -| [high-dimension-normalize-screening_Kriging_python](high-dimension-normalize-screening_Kriging_python.ipynb) | python | `Kriging` | 6 to 20 inputs in physical units: `normalize`, reading the fitted θ to screen inputs (Welch) | explicit rescaling, known active inputs | [zip](https://nightly.link/libKriging/gallery/workflows/notebooks/main/executed-master-high-dimension-normalize-screening_Kriging_python.zip) | [Colab](https://colab.research.google.com/github/libKriging/gallery/blob/main/high-dimension-normalize-screening_Kriging_python.ipynb) · [Binder](https://mybinder.org/v2/gh/libKriging/gallery/main?labpath=high-dimension-normalize-screening_Kriging_python.ipynb) | -| [save-load-pickle-copy-accessors_Kriging_python](save-load-pickle-copy-accessors_Kriging_python.ipynb) | python | `Kriging` | saving, loading, pickling, copying a model; reading its settings back | the original model (to 1e-12) | [zip](https://nightly.link/libKriging/gallery/workflows/notebooks/main/executed-master-save-load-pickle-copy-accessors_Kriging_python.zip) | [Colab](https://colab.research.google.com/github/libKriging/gallery/blob/main/save-load-pickle-copy-accessors_Kriging_python.ipynb) · [Binder](https://mybinder.org/v2/gh/libKriging/gallery/main?labpath=save-load-pickle-copy-accessors_Kriging_python.ipynb) | -| [sklearn-estimators_Kriging_python](sklearn-estimators_Kriging_python.ipynb) | python | `Kriging` | libKriging inside scikit-learn: `Pipeline`, `GridSearchCV` over kernels, `cross_val_score`, `sample_y` | the native class | [zip](https://nightly.link/libKriging/gallery/workflows/notebooks/main/executed-master-sklearn-estimators_Kriging_python.zip) | [Colab](https://colab.research.google.com/github/libKriging/gallery/blob/main/sklearn-estimators_Kriging_python.ipynb) · [Binder](https://mybinder.org/v2/gh/libKriging/gallery/main?labpath=sklearn-estimators_Kriging_python.ipynb) | +| [update-simulate_Kriging_python](src/update-simulate_Kriging_python.py) | python | `Kriging` | the workflow on 1D to 6D functions: fit, `predict`, conditional paths, `update` with new data, `update_simulate`, in pictures | fit from scratch, Monte Carlo moments | [zip](https://nightly.link/libKriging/gallery/workflows/notebooks/main/executed-master-update-simulate_Kriging_python.zip) | [Colab](https://colab.research.google.com/github/libKriging/gallery/blob/gh-pages/notebooks/update-simulate_Kriging_python.ipynb) · [Binder](https://mybinder.org/v2/gh/libKriging/gallery/gh-pages?labpath=notebooks%2Fupdate-simulate_Kriging_python.ipynb) | +| [kernels-trends-interpolation_Kriging_python](src/kernels-trends-interpolation_Kriging_python.py) | python | `Kriging` | choosing a kernel and a trend: 4 × 4 combinations on 8 functions, interpolation, predicted covariance and derivatives | finite differences, Q² floors | [zip](https://nightly.link/libKriging/gallery/workflows/notebooks/main/executed-master-kernels-trends-interpolation_Kriging_python.zip) | [Colab](https://colab.research.google.com/github/libKriging/gallery/blob/gh-pages/notebooks/kernels-trends-interpolation_Kriging_python.ipynb) · [Binder](https://mybinder.org/v2/gh/libKriging/gallery/gh-pages?labpath=notebooks%2Fkernels-trends-interpolation_Kriging_python.ipynb) | +| [objectives-LL-LOO-LMP-gradients_Kriging_python](src/objectives-LL-LOO-LMP-gradients_Kriging_python.py) | python | `Kriging` | choosing the estimation criterion (`LL`, `LOO`, `LMP`), its profile around θ̂, multistart | finite differences, explicit leave-one-out refits | [zip](https://nightly.link/libKriging/gallery/workflows/notebooks/main/executed-master-objectives-LL-LOO-LMP-gradients_Kriging_python.zip) | [Colab](https://colab.research.google.com/github/libKriging/gallery/blob/gh-pages/notebooks/objectives-LL-LOO-LMP-gradients_Kriging_python.ipynb) · [Binder](https://mybinder.org/v2/gh/libKriging/gallery/gh-pages?labpath=notebooks%2Fobjectives-LL-LOO-LMP-gradients_Kriging_python.ipynb) | +| [high-dimension-normalize-screening_Kriging_python](src/high-dimension-normalize-screening_Kriging_python.py) | python | `Kriging` | 6 to 20 inputs in physical units: `normalize`, reading the fitted θ to screen inputs (Welch) | explicit rescaling, known active inputs | [zip](https://nightly.link/libKriging/gallery/workflows/notebooks/main/executed-master-high-dimension-normalize-screening_Kriging_python.zip) | [Colab](https://colab.research.google.com/github/libKriging/gallery/blob/gh-pages/notebooks/high-dimension-normalize-screening_Kriging_python.ipynb) · [Binder](https://mybinder.org/v2/gh/libKriging/gallery/gh-pages?labpath=notebooks%2Fhigh-dimension-normalize-screening_Kriging_python.ipynb) | +| [save-load-pickle-copy-accessors_Kriging_python](src/save-load-pickle-copy-accessors_Kriging_python.py) | python | `Kriging` | saving, loading, pickling, copying a model; reading its settings back | the original model (to 1e-12) | [zip](https://nightly.link/libKriging/gallery/workflows/notebooks/main/executed-master-save-load-pickle-copy-accessors_Kriging_python.zip) | [Colab](https://colab.research.google.com/github/libKriging/gallery/blob/gh-pages/notebooks/save-load-pickle-copy-accessors_Kriging_python.ipynb) · [Binder](https://mybinder.org/v2/gh/libKriging/gallery/gh-pages?labpath=notebooks%2Fsave-load-pickle-copy-accessors_Kriging_python.ipynb) | +| [sklearn-estimators_Kriging_python](src/sklearn-estimators_Kriging_python.py) | python | `Kriging` | libKriging inside scikit-learn: `Pipeline`, `GridSearchCV` over kernels, `cross_val_score`, `sample_y` | the native class | [zip](https://nightly.link/libKriging/gallery/workflows/notebooks/main/executed-master-sklearn-estimators_Kriging_python.zip) | [Colab](https://colab.research.google.com/github/libKriging/gallery/blob/gh-pages/notebooks/sklearn-estimators_Kriging_python.ipynb) · [Binder](https://mybinder.org/v2/gh/libKriging/gallery/gh-pages?labpath=notebooks%2Fsklearn-estimators_Kriging_python.ipynb) | ### Noisy observations | notebook | binding | kriging type | what it shows | validated against | executed | open in | |---|---|---|---|---|---|---| -| [noisy-data_Kriging-nugget-noise_python](noisy-data_Kriging-nugget-noise_python.ipynb) | python | `Kriging` with `noise="nugget"` or variances | estimated nugget vs known noise variances, replicates, calibrated intervals on the noise-free function | true noise, replicate averaging, interpolating fit | [zip](https://nightly.link/libKriging/gallery/workflows/notebooks/main/executed-master-noisy-data_Kriging-nugget-noise_python.zip) | [Colab](https://colab.research.google.com/github/libKriging/gallery/blob/main/noisy-data_Kriging-nugget-noise_python.ipynb) · [Binder](https://mybinder.org/v2/gh/libKriging/gallery/main?labpath=noisy-data_Kriging-nugget-noise_python.ipynb) | +| [noisy-data_Kriging-nugget-noise_python](src/noisy-data_Kriging-nugget-noise_python.py) | python | `Kriging` with `noise="nugget"` or variances | estimated nugget vs known noise variances, replicates, calibrated intervals on the noise-free function | true noise, replicate averaging, interpolating fit | [zip](https://nightly.link/libKriging/gallery/workflows/notebooks/main/executed-master-noisy-data_Kriging-nugget-noise_python.zip) | [Colab](https://colab.research.google.com/github/libKriging/gallery/blob/gh-pages/notebooks/noisy-data_Kriging-nugget-noise_python.ipynb) · [Binder](https://mybinder.org/v2/gh/libKriging/gallery/gh-pages?labpath=notebooks%2Fnoisy-data_Kriging-nugget-noise_python.ipynb) | ### Large datasets | notebook | binding | kriging type | what it shows | validated against | executed | open in | |---|---|---|---|---|---|---| -| [large-n_Kriging-vecchia-nystrom_python](large-n_Kriging-vecchia-nystrom_python.ipynb) | python | `Kriging` with `method="vecchia(m)"` / `"nystrom(k)"` | n up to 5000: Vecchia and Nyström fits, their convergence to the exact model as m and k grow, `subsetOfData` | exact model (limits m = n − 1, k = n), exact fit | [zip](https://nightly.link/libKriging/gallery/workflows/notebooks/main/executed-master-large-n_Kriging-vecchia-nystrom_python.zip) | [Colab](https://colab.research.google.com/github/libKriging/gallery/blob/main/large-n_Kriging-vecchia-nystrom_python.ipynb) · [Binder](https://mybinder.org/v2/gh/libKriging/gallery/main?labpath=large-n_Kriging-vecchia-nystrom_python.ipynb) | -| [aggregations-large-n_NestedKriging_python](aggregations-large-n_NestedKriging_python.ipynb) | python | `NestedKriging` | splitting a design into groups: `NK`, `PoE`, `gPoE`, `BCM`, `rBCM`, from 1 group to 32 | exact predictor (limit: 1 group), exact fit | [zip](https://nightly.link/libKriging/gallery/workflows/notebooks/main/executed-master-aggregations-large-n_NestedKriging_python.zip) | [Colab](https://colab.research.google.com/github/libKriging/gallery/blob/main/aggregations-large-n_NestedKriging_python.ipynb) · [Binder](https://mybinder.org/v2/gh/libKriging/gallery/main?labpath=aggregations-large-n_NestedKriging_python.ipynb) | +| [large-n_Kriging-vecchia-nystrom_python](src/large-n_Kriging-vecchia-nystrom_python.py) | python | `Kriging` with `method="vecchia(m)"` / `"nystrom(k)"` | n up to 5000: Vecchia and Nyström fits, their convergence to the exact model as m and k grow, `subsetOfData` | exact model (limits m = n − 1, k = n), exact fit | [zip](https://nightly.link/libKriging/gallery/workflows/notebooks/main/executed-master-large-n_Kriging-vecchia-nystrom_python.zip) | [Colab](https://colab.research.google.com/github/libKriging/gallery/blob/gh-pages/notebooks/large-n_Kriging-vecchia-nystrom_python.ipynb) · [Binder](https://mybinder.org/v2/gh/libKriging/gallery/gh-pages?labpath=notebooks%2Flarge-n_Kriging-vecchia-nystrom_python.ipynb) | +| [aggregations-large-n_NestedKriging_python](src/aggregations-large-n_NestedKriging_python.py) | python | `NestedKriging` | splitting a design into groups: `NK`, `PoE`, `gPoE`, `BCM`, `rBCM`, from 1 group to 32 | exact predictor (limit: 1 group), exact fit | [zip](https://nightly.link/libKriging/gallery/workflows/notebooks/main/executed-master-aggregations-large-n_NestedKriging_python.zip) | [Colab](https://colab.research.google.com/github/libKriging/gallery/blob/gh-pages/notebooks/aggregations-large-n_NestedKriging_python.ipynb) · [Binder](https://mybinder.org/v2/gh/libKriging/gallery/gh-pages?labpath=notebooks%2Faggregations-large-n_NestedKriging_python.ipynb) | ### Non-stationary, categorical and ordinal inputs | notebook | binding | kriging type | what it shows | validated against | executed | open in | |---|---|---|---|---|---|---| -| [warpings-categorical-ordinal_WarpKriging_python](warpings-categorical-ordinal_WarpKriging_python.ipynb) | python | `WarpKriging` | input warpings (affine, boxcox, kumaraswamy, knots, neural_mono), categorical and ordinal inputs | plain `Kriging` (accuracy, NLPD), `none` warping = `Kriging` | [zip](https://nightly.link/libKriging/gallery/workflows/notebooks/main/executed-master-warpings-categorical-ordinal_WarpKriging_python.zip) | [Colab](https://colab.research.google.com/github/libKriging/gallery/blob/main/warpings-categorical-ordinal_WarpKriging_python.ipynb) · [Binder](https://mybinder.org/v2/gh/libKriging/gallery/main?labpath=warpings-categorical-ordinal_WarpKriging_python.ipynb) | +| [warpings-categorical-ordinal_WarpKriging_python](src/warpings-categorical-ordinal_WarpKriging_python.py) | python | `WarpKriging` | input warpings (affine, boxcox, kumaraswamy, knots, neural_mono), categorical and ordinal inputs | plain `Kriging` (accuracy, NLPD), `none` warping = `Kriging` | [zip](https://nightly.link/libKriging/gallery/workflows/notebooks/main/executed-master-warpings-categorical-ordinal_WarpKriging_python.zip) | [Colab](https://colab.research.google.com/github/libKriging/gallery/blob/gh-pages/notebooks/warpings-categorical-ordinal_WarpKriging_python.ipynb) · [Binder](https://mybinder.org/v2/gh/libKriging/gallery/gh-pages?labpath=notebooks%2Fwarpings-categorical-ordinal_WarpKriging_python.ipynb) | ### Deep kernels | notebook | binding | kriging type | what it shows | validated against | executed | open in | |---|---|---|---|---|---|---| -| [deep-kernel-projected-branin_MLPKriging_python](deep-kernel-projected-branin_MLPKriging_python.ipynb) | python | `MLPKriging` | an MLP feature map finding the 2D structure of a 10D function; activations, multistart | plain `Kriging` | [zip](https://nightly.link/libKriging/gallery/workflows/notebooks/main/executed-master-deep-kernel-projected-branin_MLPKriging_python.zip) | [Colab](https://colab.research.google.com/github/libKriging/gallery/blob/main/deep-kernel-projected-branin_MLPKriging_python.ipynb) · [Binder](https://mybinder.org/v2/gh/libKriging/gallery/main?labpath=deep-kernel-projected-branin_MLPKriging_python.ipynb) | +| [deep-kernel-projected-branin_MLPKriging_python](src/deep-kernel-projected-branin_MLPKriging_python.py) | python | `MLPKriging` | an MLP feature map finding the 2D structure of a 10D function; activations, multistart | plain `Kriging` | [zip](https://nightly.link/libKriging/gallery/workflows/notebooks/main/executed-master-deep-kernel-projected-branin_MLPKriging_python.zip) | [Colab](https://colab.research.google.com/github/libKriging/gallery/blob/gh-pages/notebooks/deep-kernel-projected-branin_MLPKriging_python.ipynb) · [Binder](https://mybinder.org/v2/gh/libKriging/gallery/gh-pages?labpath=notebooks%2Fdeep-kernel-projected-branin_MLPKriging_python.ipynb) | ### libKriging and other kriging packages | notebook | binding | kriging type | what it shows | validated against | executed | open in | |---|---|---|---|---|---|---| -| [vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python](vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python.ipynb) | python | `Kriging` | same hyperparameters → same numbers, then default fits on 10 functions (accuracy, calibration, cost) | scikit-learn, SMT, OpenTURNS, GPy | [zip](https://nightly.link/libKriging/gallery/workflows/notebooks/main/executed-master-vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python.zip) | [Colab](https://colab.research.google.com/github/libKriging/gallery/blob/main/vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python.ipynb) · [Binder](https://mybinder.org/v2/gh/libKriging/gallery/main?labpath=vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python.ipynb) | -| [vs-DiceKriging-RobustGaSP_Kriging_R](vs-DiceKriging-RobustGaSP_Kriging_R.ipynb) | R | `Kriging` | rlibkriging next to the R references: equality at fixed hyperparameters, default fits | DiceKriging, RobustGaSP | [zip](https://nightly.link/libKriging/gallery/workflows/notebooks/main/executed-master-vs-DiceKriging-RobustGaSP_Kriging_R.zip) | — | -| [vs-STK_Kriging_octave](vs-STK_Kriging_octave.ipynb) | octave | `Kriging` | mLibKriging next to STK: equality at fixed hyperparameters (Gaussian kernel), default fits | STK | [zip](https://nightly.link/libKriging/gallery/workflows/notebooks/main/executed-master-vs-STK_Kriging_octave.zip) | — | -| [vs-GaussianProcesses_Kriging_julia](vs-GaussianProcesses_Kriging_julia.ipynb) | julia | `Kriging` | JLibKriging next to GaussianProcesses.jl: default fits | GaussianProcesses.jl | [zip](https://nightly.link/libKriging/gallery/workflows/notebooks/main/executed-master-vs-GaussianProcesses_Kriging_julia.zip) | — | +| [vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python](src/vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python.py) | python | `Kriging` | same hyperparameters → same numbers, then default fits on 10 functions (accuracy, calibration, cost) | scikit-learn, SMT, OpenTURNS, GPy | [zip](https://nightly.link/libKriging/gallery/workflows/notebooks/main/executed-master-vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python.zip) | [Colab](https://colab.research.google.com/github/libKriging/gallery/blob/gh-pages/notebooks/vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python.ipynb) · [Binder](https://mybinder.org/v2/gh/libKriging/gallery/gh-pages?labpath=notebooks%2Fvs-sklearn-SMT-OpenTURNS-GPy_Kriging_python.ipynb) | +| [vs-DiceKriging-RobustGaSP_Kriging_R](src/vs-DiceKriging-RobustGaSP_Kriging_R.R) | R | `Kriging` | rlibkriging next to the R references: equality at fixed hyperparameters, default fits | DiceKriging, RobustGaSP | [zip](https://nightly.link/libKriging/gallery/workflows/notebooks/main/executed-master-vs-DiceKriging-RobustGaSP_Kriging_R.zip) | [nbviewer](https://nbviewer.org/github/libKriging/gallery/blob/gh-pages/notebooks/vs-DiceKriging-RobustGaSP_Kriging_R.ipynb) | +| [vs-STK_Kriging_octave](src/vs-STK_Kriging_octave.m) | octave | `Kriging` | mLibKriging next to STK: equality at fixed hyperparameters (Gaussian kernel), default fits | STK | [zip](https://nightly.link/libKriging/gallery/workflows/notebooks/main/executed-master-vs-STK_Kriging_octave.zip) | [nbviewer](https://nbviewer.org/github/libKriging/gallery/blob/gh-pages/notebooks/vs-STK_Kriging_octave.ipynb) | +| [vs-GaussianProcesses_Kriging_julia](src/vs-GaussianProcesses_Kriging_julia.jl) | julia | `Kriging` | JLibKriging next to GaussianProcesses.jl: default fits | GaussianProcesses.jl | [zip](https://nightly.link/libKriging/gallery/workflows/notebooks/main/executed-master-vs-GaussianProcesses_Kriging_julia.zip) | [nbviewer](https://nbviewer.org/github/libKriging/gallery/blob/gh-pages/notebooks/vs-GaussianProcesses_Kriging_julia.ipynb) | ### Validation only | notebook | binding | kriging type | what it shows | validated against | executed | open in | |---|---|---|---|---|---|---| -| [test-functions-reference-values_no-model_python](test-functions-reference-values_no-model_python.ipynb) | python | — | the 21 test functions used throughout, plotted and checked | published optima / values | [zip](https://nightly.link/libKriging/gallery/workflows/notebooks/main/executed-master-test-functions-reference-values_no-model_python.zip) | [Colab](https://colab.research.google.com/github/libKriging/gallery/blob/main/test-functions-reference-values_no-model_python.ipynb) · [Binder](https://mybinder.org/v2/gh/libKriging/gallery/main?labpath=test-functions-reference-values_no-model_python.ipynb) | -| [robustness-invalid-inputs_Kriging_python](robustness-invalid-inputs_Kriging_python.ipynb) | python | `Kriging` | invalid and degenerate inputs, ill-conditioning, units, extrapolation, prediction cost in q | equivariance identities, O(q) cost | [zip](https://nightly.link/libKriging/gallery/workflows/notebooks/main/executed-master-robustness-invalid-inputs_Kriging_python.zip) | [Colab](https://colab.research.google.com/github/libKriging/gallery/blob/main/robustness-invalid-inputs_Kriging_python.ipynb) · [Binder](https://mybinder.org/v2/gh/libKriging/gallery/main?labpath=robustness-invalid-inputs_Kriging_python.ipynb) | -| [cross-binding-reference_all-classes_python](cross-binding-reference_all-classes_python.ipynb) | python | all classes | the same data and options in every binding, every class and fit method: the Python values | R, Octave, Julia (report) | [zip](https://nightly.link/libKriging/gallery/workflows/notebooks/main/executed-master-cross-binding-reference_all-classes_python.zip) | [Colab](https://colab.research.google.com/github/libKriging/gallery/blob/main/cross-binding-reference_all-classes_python.ipynb) · [Binder](https://mybinder.org/v2/gh/libKriging/gallery/main?labpath=cross-binding-reference_all-classes_python.ipynb) | -| [cross-binding_all-classes_R](cross-binding_all-classes_R.ipynb) | R | all classes | the R side of the cross-binding protocol | Python (report) | [zip](https://nightly.link/libKriging/gallery/workflows/notebooks/main/executed-master-cross-binding_all-classes_R.zip) | — | -| [cross-binding_all-classes_octave](cross-binding_all-classes_octave.ipynb) | octave | all classes | the Octave side | Python (report) | [zip](https://nightly.link/libKriging/gallery/workflows/notebooks/main/executed-master-cross-binding_all-classes_octave.zip) | — | -| [cross-binding_all-classes_julia](cross-binding_all-classes_julia.ipynb) | julia | all classes | the Julia side | Python (report) | [zip](https://nightly.link/libKriging/gallery/workflows/notebooks/main/executed-master-cross-binding_all-classes_julia.zip) | — | - -Notebooks are named `__.ipynb`. Those comparing +| [test-functions-reference-values_no-model_python](src/test-functions-reference-values_no-model_python.py) | python | — | the 21 test functions used throughout, plotted and checked | published optima / values | [zip](https://nightly.link/libKriging/gallery/workflows/notebooks/main/executed-master-test-functions-reference-values_no-model_python.zip) | [Colab](https://colab.research.google.com/github/libKriging/gallery/blob/gh-pages/notebooks/test-functions-reference-values_no-model_python.ipynb) · [Binder](https://mybinder.org/v2/gh/libKriging/gallery/gh-pages?labpath=notebooks%2Ftest-functions-reference-values_no-model_python.ipynb) | +| [robustness-invalid-inputs_Kriging_python](src/robustness-invalid-inputs_Kriging_python.py) | python | `Kriging` | invalid and degenerate inputs, ill-conditioning, units, extrapolation, prediction cost in q | equivariance identities, O(q) cost | [zip](https://nightly.link/libKriging/gallery/workflows/notebooks/main/executed-master-robustness-invalid-inputs_Kriging_python.zip) | [Colab](https://colab.research.google.com/github/libKriging/gallery/blob/gh-pages/notebooks/robustness-invalid-inputs_Kriging_python.ipynb) · [Binder](https://mybinder.org/v2/gh/libKriging/gallery/gh-pages?labpath=notebooks%2Frobustness-invalid-inputs_Kriging_python.ipynb) | +| [cross-binding-reference_all-classes_python](src/cross-binding-reference_all-classes_python.py) | python | all classes | the same data and options in every binding, every class and fit method: the Python values | R, Octave, Julia (report) | [zip](https://nightly.link/libKriging/gallery/workflows/notebooks/main/executed-master-cross-binding-reference_all-classes_python.zip) | [Colab](https://colab.research.google.com/github/libKriging/gallery/blob/gh-pages/notebooks/cross-binding-reference_all-classes_python.ipynb) · [Binder](https://mybinder.org/v2/gh/libKriging/gallery/gh-pages?labpath=notebooks%2Fcross-binding-reference_all-classes_python.ipynb) | +| [cross-binding_all-classes_R](src/cross-binding_all-classes_R.R) | R | all classes | the R side of the cross-binding protocol | Python (report) | [zip](https://nightly.link/libKriging/gallery/workflows/notebooks/main/executed-master-cross-binding_all-classes_R.zip) | [nbviewer](https://nbviewer.org/github/libKriging/gallery/blob/gh-pages/notebooks/cross-binding_all-classes_R.ipynb) | +| [cross-binding_all-classes_octave](src/cross-binding_all-classes_octave.m) | octave | all classes | the Octave side | Python (report) | [zip](https://nightly.link/libKriging/gallery/workflows/notebooks/main/executed-master-cross-binding_all-classes_octave.zip) | [nbviewer](https://nbviewer.org/github/libKriging/gallery/blob/gh-pages/notebooks/cross-binding_all-classes_octave.ipynb) | +| [cross-binding_all-classes_julia](src/cross-binding_all-classes_julia.jl) | julia | all classes | the Julia side | Python (report) | [zip](https://nightly.link/libKriging/gallery/workflows/notebooks/main/executed-master-cross-binding_all-classes_julia.zip) | [nbviewer](https://nbviewer.org/github/libKriging/gallery/blob/gh-pages/notebooks/cross-binding_all-classes_julia.ipynb) | + +Notebooks are named `__`. Those comparing with another package install it themselves, in their first cells (pip, CRAN, `Pkg.add`, or a clone of STK); a package that cannot be installed is reported (warn-level) and its comparison skipped. Test functions: [`tools/python/gallery.py`](tools/python/gallery.py) (section 1), after the [Virtual Library of Simulation Experiments](https://www.sfu.ca/~ssurjano/) -(Surjanovic & Bingham). Everything that is not a notebook is under -[`tools/`](tools) (sources of the notebooks, helpers, installation of the bindings, -CI scripts, known issues, baselines), the shared datasets under [`data/`](data), -the Binder environment under [`.binder/`](.binder). +(Surjanovic & Bingham). The notebook sources are in [`src/`](src); everything else is under +[`tools/`](tools) (helpers, notebook generation, installation of the bindings, +CI scripts, site generator, known issues, baselines), the shared datasets under +[`data/`](data), the Binder environment under [`.binder/`](.binder). ## Running a notebook in Colab or Binder -The **open in** links run a Python notebook in a free hosted service, against the -**released** pylibkriging (from PyPI), not against libKriging master as CI does: a -feature not released yet is reported as SKIP. The first code cell of each Python +The **open in** links (and the cards of the site) open the executed notebook of the +`gh-pages` branch, outputs of the last CI run included; re-running it in a free +hosted service uses the **released** pylibkriging (from PyPI), not libKriging master +as CI does: a feature not released yet is reported as SKIP. The first code cell of each Python notebook ([`tools/notebook_setup.py`](tools/notebook_setup.py), a no-op in this repository) clones the repository for its datasets and helpers and installs pylibkriging when missing; the notebooks that compare with other packages install @@ -113,10 +124,12 @@ them themselves. default runtime has used Python 3.13 since August-September 2026: choose *Runtime > Change runtime version > 2026.07* (Python 3.12) before running, otherwise the setup cell stops with this message. -* **Binder**: the environment of [`.binder/`](.binder) (Python 3.12, pylibkriging); - the first start builds the image, which takes a few minutes. +* **Binder**: the environment of [`.binder/`](.binder) (Python 3.12, pylibkriging), + copied to `gh-pages` by the site generator; the first start builds the image, which + takes a few minutes. -Not offered for the R, Octave and Julia notebooks: Octave is not available on Colab, +Not offered for the R, Octave and Julia notebooks (nbviewer instead, read only): +Octave is not available on Colab, and the installation of rlibkriging (compiled from CRAN sources) and of JLibKriging.jl (which compiles libKriging) on these services has not been checked. @@ -156,7 +169,8 @@ against [`tools/baseline/`](tools/baseline). ## Running locally -From the repository root (the notebooks' paths are relative to it): +From the repository root (the notebooks' paths are relative to it; generated +notebooks land there, ignored by git): ```sh git clone --recurse-submodules https://github.com/libKriging/libKriging ../libKriging # for a master build @@ -164,8 +178,11 @@ tools/install/python.sh master ../libKriging # or: tools/install/python.sh tools/install/r.sh master ~/Rlib ../libKriging # then R_LIBS_USER=~/Rlib tools/install/octave.sh ../libKriging ~/mlib # then OCTAVE_PATH=~/mlib tools/install/julia.sh ../libKriging +python tools/py2nb.py src/update-simulate_Kriging_python.py # or no argument: every source python tools/run_notebooks.py --inplace update-simulate_Kriging_python.ipynb python tools/report.py --expect python +mkdir -p executed && cp *.ipynb executed/ +python tools/site.py --notebooks executed --results results --out _site # then open _site/index.html ``` The scripts of [`tools/install/`](tools/install) are the ones CI runs; each says @@ -173,22 +190,26 @@ what it needs (BLAS/LAPACK headers, Octave, R, Julia). ## Editing notebooks -Notebooks are generated from plain-text sources ([`tools/src/`](tools/src), cells -separated by `# %%` / `# %% [markdown]`, or `% %%` in Octave), so that reviews are -plain diffs, and get the setup cell of `tools/notebook_setup.py` (Python); outputs -are not committed (CI archives the executed notebooks as artifacts, see -[The examples](#the-examples)). +A notebook exists only as a plain-text source in [`src/`](src) (cells separated by +`# %%` / `# %% [markdown]`, or `% %%` in Octave), so that reviews are plain diffs. +[`tools/py2nb.py`](tools/py2nb.py) turns it into a notebook (Python ones get the setup +cell of `tools/notebook_setup.py`); CI does so in each notebook job, executes it, and +archives and publishes the executed notebook (see [The examples](#the-examples)). ```sh -python tools/py2nb.py # regenerate every notebook from tools/src/ -python tools/py2nb.py --check # CI: sources and notebooks in sync, no notebook without source +python tools/py2nb.py # every notebook of src/, at the repository root +python tools/py2nb.py --check # CI: every source valid (name, title, idea, reference, tags, suite) ``` -A new example is a new source in `tools/src/` named `__`, -with its suite named the same, an **Idea** and a **Reference** paragraph after its -title, plots and tables of its results, its checks, a final `## Summary` cell, and a -line in the tables above (CI checks that -every notebook is linked from this README); CI picks it up automatically. +A new example is a new source in `src/` named `__`, +with its suite named the same, a first markdown cell holding its title +(`# · — `) and an **Idea**, a **Reference** and a **Tags** +paragraph, then plots and tables of its results, its checks, a final `## Summary` +cell, and a line in the tables above (CI checks that every source is linked from this +README); CI picks it up automatically. Tags are `key: value, value` groups separated +by ` · `, with the keys `class` (required), `kernel`, `trend`, `objective`, `method`, +`noise`, `warping`, `aggregation`, `compared to`, `topic`; the language comes from the +file extension. They become the tags and filters of the site. Shared datasets (`data/`, used by the cross-binding notebooks of every language) are written once by `tools/make_data.py` and committed: the files are the @@ -212,7 +233,11 @@ One build job per binding and channel, then **one job per notebook**, each uploading its executed notebook and its results as the artifact `executed--`; then one report per channel (job summary, `report-` artifact with the report, all CSV results and a refreshed -baseline, and `executed-notebooks-` with all the executed notebooks). +baseline, and `executed-notebooks-` with all the executed notebooks); then +the **site** job builds the gallery site from the executed notebooks of the master +channel (artifact `site`, for a preview, on every run) and, on `main` with +libKriging/libKriging as source, force-pushes it as the single commit of the +`gh-pages` branch (`contents: write` for this job only). The initial `tools/baseline/` files come from a local run (Python notebooks only, 4-core container); to refresh them, commit the `new-baseline/` files of a trusted CI run. diff --git a/aggregations-large-n_NestedKriging_python.ipynb b/aggregations-large-n_NestedKriging_python.ipynb deleted file mode 100644 index b7ef835..0000000 --- a/aggregations-large-n_NestedKriging_python.ipynb +++ /dev/null @@ -1,309 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "c000", - "metadata": {}, - "source": [ - "# Python · NestedKriging — aggregations, large n\n", - "\n", - "**Idea.** NestedKriging splits a large design into groups and aggregates their\n", - "predictions; how far is it from the exact predictor as the number of groups grows,\n", - "and which aggregation is closest?\n", - "\n", - "**Reference.** The exact predictor at the same θ (limit: 1 group), and the exact\n", - "fit.\n", - "\n", - "`NestedKriging` partitions the design into groups, fits one `Kriging`\n", - "sub-model per group (common hyperparameters), and aggregates the sub-model\n", - "predictions: `NK` (nested kriging: kriging of the sub-predictors, a proper\n", - "kriging predictor), or the products of experts `PoE`, `gPoE`, `BCM`, `rBCM`\n", - "(cheaper, variances not fully consistent).\n", - "\n", - "Problems: as in `large-n_Kriging-vecchia-nystrom_python.ipynb`, Hartmann 3\n", - "and the OTL circuit with n = 1500 (exact fit as reference) and n = 5000; then\n", - "every aggregation on Branin.\n", - "\n", - "Checks:\n", - "* **limit and asymptotic behaviour** at fixed θ (Hartmann 3, n = 300): with one\n", - " group, `NK` *is* the exact kriging predictor; the gap to the exact predictor\n", - " as the number of groups grows, against the fit and prediction cost;\n", - "* accuracy close to the exact fit (warn-level gap, error-level floor), and\n", - " faster than the exact fit at n = 1500 (warn-level);\n", - "* `NK` interpolates the data (it is a proper kriging predictor); the PoE family\n", - " does not need to; all stdev ≥ 0;\n", - "* `NK` with a non-constant trend is rejected (simple-kriging theory)." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c001", - "metadata": {}, - "outputs": [], - "source": [ - "# Setup for running this notebook outside the repository (Colab, Binder, a lone copy):\n", - "# nothing to do in the repository itself (CI, local runs). Otherwise the repository\n", - "# (datasets of data/, helpers of tools/python/gallery.py) is cloned and the released pylibkriging installed.\n", - "import importlib\n", - "import os\n", - "import subprocess\n", - "import sys\n", - "from pathlib import Path\n", - "\n", - "if not Path(\"tools/known_issues.csv\").is_file():\n", - " if not Path(\"gallery/tools/known_issues.csv\").is_file():\n", - " subprocess.run([\"git\", \"clone\", \"--depth\", \"1\", \"https://github.com/libKriging/gallery\"], check=True)\n", - " os.chdir(\"gallery\")\n", - " try:\n", - " import pylibkriging # noqa: F401\n", - " except ImportError:\n", - " r = subprocess.run([sys.executable, \"-m\", \"pip\", \"install\", \"--quiet\", \"pylibkriging\"],\n", - " capture_output=True, text=True)\n", - " if r.returncode != 0:\n", - " raise RuntimeError(\n", - " f\"pylibkriging could not be installed for Python {sys.version_info.major}.{sys.version_info.minor} \"\n", - " \"(PyPI has wheels for CPython 3.10 to 3.12 only). On Colab: Runtime > Change runtime version > \"\n", - " \"2026.07 (Python 3.12), then run again.\\n\" + r.stderr[-500:])\n", - " importlib.invalidate_caches()\n", - "sys.path.insert(0, str(Path(\"tools/python\").resolve())) # for \"import gallery\" (tools/python/gallery.py)\n", - "print(\"working directory:\", os.getcwd())" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c002", - "metadata": {}, - "outputs": [], - "source": [ - "import time\n", - "\n", - "import matplotlib.pyplot as plt\n", - "import numpy as np\n", - "import pandas as pd\n", - "import pylibkriging as lk\n", - "\n", - "import gallery\n", - "from gallery import TESTS, design, metrics, compat\n", - "\n", - "suite = gallery.Suite(\"aggregations-large-n_NestedKriging_python\")\n", - "print(\"pylibkriging\", compat.version())\n", - "if not compat.has(\"NestedKriging\"):\n", - " suite.check_skip(\"NestedKriging\", \"not available in this pylibkriging\")\n", - "\n", - "\n", - "def pred(k, X, **kw):\n", - " m, s, *_ = k.predict(X, return_stdev=True, **kw)\n", - " return np.ravel(m), np.ravel(s)\n", - "\n", - "\n", - "def timed(f):\n", - " t0 = time.time()\n", - " out = f()\n", - " return out, time.time() - t0" - ] - }, - { - "cell_type": "markdown", - "id": "c003", - "metadata": {}, - "source": [ - "## From one group (the exact predictor) to many, at fixed θ\n", - "\n", - "Hartmann 3, n = 300, θ of the exact fit kept fixed in every sub-model: the gap\n", - "is that of the aggregation alone. RMS gap of the predicted mean to the exact\n", - "predictor on 500 test points, relative to its standard deviation, and fit time." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c004", - "metadata": {}, - "outputs": [], - "source": [ - "if compat.has(\"NestedKriging\"):\n", - " tf = TESTS[\"hartmann3\"]\n", - " X, y, Xt, yt = design.dataset(tf, 300, n_test=500, seed=54)\n", - " theta = np.asarray(lk.Kriging(y, X, \"matern5_2\").theta()).reshape(1, -1)\n", - " fixed = {\"theta\": theta, \"is_theta_estim\": False}\n", - " m_e = pred(lk.Kriging(y, X, \"matern5_2\", optim=\"none\", parameters=fixed), Xt)[0]\n", - " rows = []\n", - " for agg in [\"NK\", \"rBCM\"]:\n", - " for g in [1, 2, 4, 8, 16, 32]:\n", - " with suite.check_runs(f\"nested {agg}, {g} groups, fixed theta: fit + predict\"):\n", - " k, t = timed(lambda: lk.NestedKriging(y, X, \"matern5_2\", nb_groups=g, aggregation=agg, seed=1,\n", - " optim=\"none\", parameters=fixed))\n", - " m = pred(k, Xt)[0]\n", - " rows.append(dict(aggregation=agg, groups=g, time_s=t,\n", - " mean_gap=np.sqrt(np.mean((m - m_e) ** 2)) / np.std(m_e),\n", - " q2=metrics.q2(yt, m)))\n", - " conv = pd.DataFrame(rows)\n", - " nk = conv[conv[\"aggregation\"] == \"NK\"]\n", - " suite.check_less(\"nested NK, 1 group == exact predictor (RMS gap / sd)\", nk[\"mean_gap\"].iloc[0], 1e-6)\n", - " suite.check_less(\"nested NK, 32 groups: RMS gap to the exact predictor / sd\", nk[\"mean_gap\"].iloc[-1], 0.1,\n", - " severity=\"warn\")\n", - " rb = conv[conv[\"aggregation\"] == \"rBCM\"]\n", - " suite.check_true(\"nested NK closer to the exact predictor than rBCM (2 to 32 groups)\",\n", - " bool(np.all(nk[\"mean_gap\"].to_numpy()[1:] <= rb[\"mean_gap\"].to_numpy()[1:])), severity=\"warn\")\n", - " fig, axes = plt.subplots(1, 2, figsize=(10, 3.8))\n", - " for agg, g in conv.groupby(\"aggregation\"):\n", - " axes[0].semilogy(g[\"groups\"], np.maximum(g[\"mean_gap\"], 1e-12), \"o-\", label=agg)\n", - " axes[1].plot(g[\"groups\"], g[\"time_s\"], \"o-\", label=agg)\n", - " axes[0].set_ylabel(\"RMS(mean − mean(exact)) / sd\"); axes[1].set_ylabel(\"fit time (s)\")\n", - " for ax in axes:\n", - " ax.set_xscale(\"log\", base=2); ax.set_xlabel(\"number of groups\"); ax.grid(True, which=\"both\", alpha=.3)\n", - " axes[0].legend(); fig.suptitle(\"Hartmann 3, n = 300, θ fixed: NestedKriging vs the exact predictor\"); plt.show()\n", - " display(conv.round(5))" - ] - }, - { - "cell_type": "markdown", - "id": "c005", - "metadata": {}, - "source": [ - "## Accuracy and cost against the exact fit" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c006", - "metadata": {}, - "outputs": [], - "source": [ - "CASES = {\"hartmann3\": (1500, 5000), \"otl_circuit\": (1500, 5000)}\n", - "FLOORS = {\"hartmann3\": 0.98, \"otl_circuit\": 0.99}\n", - "if compat.has(\"NestedKriging\"):\n", - " for fname, (n_mid, n_big) in CASES.items():\n", - " tf = TESTS[fname]\n", - " X, y, Xt, yt = design.dataset(tf, n_big, n_test=2000, seed=52)\n", - " ke, t_exact = timed(lambda: lk.Kriging(y[:n_mid], X[:n_mid], \"matern5_2\"))\n", - " res_e = metrics.all_metrics(yt, *pred(ke, Xt))\n", - " suite.metric(f\"{fname}/n={n_mid}\", \"exact\", fit_s=t_exact, **res_e)\n", - " for agg in [\"NK\", \"rBCM\"]:\n", - " label = f\"nested {agg} (20 groups)\"\n", - " for n in (n_mid, n_big):\n", - " case = f\"{fname}/n={n}\"\n", - " with suite.check_runs(f\"{case}/{label}: fit + predict\"):\n", - " k, t = timed(lambda: lk.NestedKriging(y[:n], X[:n], \"matern5_2\", nb_groups=20, aggregation=agg,\n", - " seed=1))\n", - " m, s = pred(k, Xt)\n", - " suite.check_finite(f\"{case}/{label}: finite predictions\", m)\n", - " res = metrics.all_metrics(yt, m, s)\n", - " suite.metric(case, label, fit_s=t, **res)\n", - " suite.check_greater(f\"{case}/{label}: Q2\", res[\"q2\"], FLOORS[fname] - 0.05)\n", - " if n == n_mid:\n", - " suite.check_greater(f\"{case}/{label}: Q2 - Q2(exact)\", res[\"q2\"] - res_e[\"q2\"], -0.02,\n", - " severity=\"warn\")\n", - " suite.check_less(f\"{case}/{label}: fit time / exact fit time\", t / t_exact, 1.0, severity=\"warn\")\n", - " display(suite.table().round(4))" - ] - }, - { - "cell_type": "markdown", - "id": "c007", - "metadata": {}, - "source": [ - "## `NestedKriging` aggregations: interpolation and variance" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c008", - "metadata": {}, - "outputs": [], - "source": [ - "if compat.has(\"NestedKriging\"):\n", - " tf = TESTS[\"branin\"]\n", - " X, y, Xt, yt = design.dataset(tf, 200, n_test=500, seed=53)\n", - " for agg in [\"NK\", \"PoE\", \"gPoE\", \"BCM\", \"rBCM\"]:\n", - " with suite.check_runs(f\"nested {agg}: fit\"):\n", - " k = lk.NestedKriging(y, X, \"matern5_2\", nb_groups=8, aggregation=agg, seed=2)\n", - " m, s = (np.ravel(a) for a in k.predict(Xt, return_stdev=True)[:2])\n", - " suite.check_true(f\"nested {agg}: stdev >= 0\", np.all(s >= 0))\n", - " suite.metric(\"branin/nested\", agg, **metrics.all_metrics(yt, m, s))\n", - " if agg == \"NK\":\n", - " mX = np.ravel(k.predict(X, return_stdev=True)[0])\n", - " suite.check_less(\"nested NK: interpolates (max |m(X)-y| / sd(y))\", np.max(np.abs(mX - y)) / np.std(y),\n", - " 1e-4)\n", - " display(suite.table().loc[\"branin/nested\"].round(4))\n", - " suite.check_raises(\"nested NK with a linear trend is rejected\",\n", - " lambda: lk.NestedKriging(y, X, \"matern5_2\", nb_groups=4, aggregation=\"NK\", regmodel=\"linear\"),\n", - " severity=\"warn\")" - ] - }, - { - "cell_type": "markdown", - "id": "c009", - "metadata": {}, - "source": [ - "## save / load, pickle, copy\n", - "\n", - "On Hartmann 3 (n = 40): a model saved and loaded back (class-specific and\n", - "generic `pylibkriging.load`), pickled, or deep-copied predicts exactly as the\n", - "original." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c010", - "metadata": {}, - "outputs": [], - "source": [ - "if compat.has(\"NestedKriging\"):\n", - " X, y, Xt, yt = design.dataset(TESTS[\"hartmann3\"], 40, n_test=100, seed=61)\n", - " with suite.check_runs(\"NestedKriging: fit (persistence)\"):\n", - " gallery.persistence.check(suite, \"NestedKriging\", lk.NestedKriging(y, X, \"matern5_2\", nb_groups=4, seed=1), Xt, pred)" - ] - }, - { - "cell_type": "markdown", - "id": "c011", - "metadata": {}, - "source": [ - "## Summary\n", - "\n", - "The metrics of this notebook, the number of checks per status, and every check that\n", - "did not pass (FAIL: error-level, fails the run; WARN: warn-level; KNOWN: listed in\n", - "`tools/known_issues.csv`, with its issue; SKIP: feature absent from the installed version).\n", - "All of them are also written to `results/aggregations-large-n_NestedKriging_python.checks.csv` and `.metrics.csv`." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c012", - "metadata": {}, - "outputs": [], - "source": [ - "suite.summary()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c013", - "metadata": {}, - "outputs": [], - "source": [ - "suite.finalize()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/cross-binding-reference_all-classes_python.ipynb b/cross-binding-reference_all-classes_python.ipynb deleted file mode 100644 index b6c9026..0000000 --- a/cross-binding-reference_all-classes_python.ipynb +++ /dev/null @@ -1,288 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "c000", - "metadata": {}, - "source": [ - "# Python · all classes — cross-binding consistency: the Python reference\n", - "\n", - "**Idea.** All bindings wrap the same C++ library: on the same data with the same\n", - "options, they must give the same numbers, for every model class and fit method. This\n", - "notebook writes the Python values, the reference of the comparison.\n", - "\n", - "**Reference.** The R, Octave and Julia notebooks `cross-binding_all-classes_*`,\n", - "compared by `tools/report.py`.\n", - "\n", - "All bindings (Python, R, Octave, Julia) wrap the same C++ library: on the\n", - "same data with the same options they must return the same numbers. Each\n", - "language runs the protocol below on the shared datasets of `data/` and writes\n", - "`results/xbind_.csv` (`lang,dataset,quantity,index,value`);\n", - "`tools/report.py` then compares every language to Python.\n", - "\n", - "Protocol, for each dataset (Branin n=25, Hartmann 3 n=40, Borehole n=80, on\n", - "the unit cube), kernel `matern5_2`, trend `constant`, θ₀ = 0.3 in every\n", - "dimension:\n", - "\n", - "| quantity | how | tolerance in the report |\n", - "|---|---|---|\n", - "| `fixed_sigma2`, `fixed_beta`, `fixed_ll`, `fixed_loo` | `optim=\"none\"`, θ = θ₀ fixed | rel. 1e-8 (error) |\n", - "| `fixed_mean`, `fixed_stdev` | predict at the first 50 test points | rel. 1e-8 (error) |\n", - "| `llfun`, `llfun_grad` | `logLikelihoodFun(θ₀)` with gradient | rel. 1e-8 (error) |\n", - "| `fit_theta`, `fit_ll`, `fit_mean` | default fit (`BFGS`) | rel. 1e-4 (warn: optimizer paths may differ) |\n", - "| `sim` | `simulate(nsim=3, seed=123)` at the first 5 test points (5 × 3, column-major) | rel. 1e-8 (warn: RNG may be binding-specific) |\n", - "\n", - "Then, on Branin only, every other model class and fit method, so that each\n", - "binding is checked on all of them (the class-specific notebooks are in Python\n", - "only): with fixed hyperparameters the numbers must be identical (error); a\n", - "fit is compared at warn level, as above.\n", - "\n", - "| quantity | how | tolerance in the report |\n", - "|---|---|---|\n", - "| `nugget_fixed_mean`, `nugget_fixed_stdev` | `noise=\"nugget\"`, θ = θ₀, σ² = var(y), nugget = 0.01 var(y), all fixed | rel. 1e-8 (error) |\n", - "| `noise_fixed_mean`, `noise_fixed_stdev` | known noise variances 0.01 var(y), θ = θ₀ and σ² fixed | rel. 1e-8 (error) |\n", - "| `nugget_fit_ll`, `nugget_fit_mean` | `noise=\"nugget\"`, default fit | rel. 1e-4 (warn) |\n", - "| `vecchia_fixed_ll`, `vecchia_fixed_mean`, `nystrom_fixed_ll`, `nystrom_fixed_mean` | `method=\"vecchia(10)\"` / `\"nystrom(10)\"`, θ = θ₀ fixed; likelihood and prediction with that method | rel. 1e-8 (error) |\n", - "| `warp_fit_mean` | `WarpKriging`, `kumaraswamy` on both inputs, default fit | rel. 1e-4 (warn) |\n", - "| `mlp_fit_mean` | `MLPKriging`, hidden layer of 8 (`tanh`), 2 features, `BFGS10` | rel. 1e-4 (warn) |\n", - "| `nested_fit_mean` | `NestedKriging`, 2 groups (`kmeans`, seed 1), `NK` | rel. 1e-4 (warn) |" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c001", - "metadata": {}, - "outputs": [], - "source": [ - "# Setup for running this notebook outside the repository (Colab, Binder, a lone copy):\n", - "# nothing to do in the repository itself (CI, local runs). Otherwise the repository\n", - "# (datasets of data/, helpers of tools/python/gallery.py) is cloned and the released pylibkriging installed.\n", - "import importlib\n", - "import os\n", - "import subprocess\n", - "import sys\n", - "from pathlib import Path\n", - "\n", - "if not Path(\"tools/known_issues.csv\").is_file():\n", - " if not Path(\"gallery/tools/known_issues.csv\").is_file():\n", - " subprocess.run([\"git\", \"clone\", \"--depth\", \"1\", \"https://github.com/libKriging/gallery\"], check=True)\n", - " os.chdir(\"gallery\")\n", - " try:\n", - " import pylibkriging # noqa: F401\n", - " except ImportError:\n", - " r = subprocess.run([sys.executable, \"-m\", \"pip\", \"install\", \"--quiet\", \"pylibkriging\"],\n", - " capture_output=True, text=True)\n", - " if r.returncode != 0:\n", - " raise RuntimeError(\n", - " f\"pylibkriging could not be installed for Python {sys.version_info.major}.{sys.version_info.minor} \"\n", - " \"(PyPI has wheels for CPython 3.10 to 3.12 only). On Colab: Runtime > Change runtime version > \"\n", - " \"2026.07 (Python 3.12), then run again.\\n\" + r.stderr[-500:])\n", - " importlib.invalidate_caches()\n", - "sys.path.insert(0, str(Path(\"tools/python\").resolve())) # for \"import gallery\" (tools/python/gallery.py)\n", - "print(\"working directory:\", os.getcwd())" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c002", - "metadata": {}, - "outputs": [], - "source": [ - "import csv\n", - "\n", - "import matplotlib.pyplot as plt\n", - "import numpy as np\n", - "import pandas as pd\n", - "import pylibkriging as lk\n", - "\n", - "import gallery\n", - "from gallery import compat\n", - "from gallery import DATA_DIR as DATA, RESULTS_DIR\n", - "\n", - "suite = gallery.Suite(\"cross-binding-reference_all-classes_python\")\n", - "print(\"pylibkriging\", compat.version())\n", - "DATASETS = [\"branin\", \"hartmann3\", \"borehole\"]\n", - "rows = []\n", - "\n", - "\n", - "def put(dataset, quantity, values):\n", - " for i, v in enumerate(np.ravel(values), start=1):\n", - " rows.append((\"python\", dataset, quantity, i, f\"{float(v):.17g}\"))\n", - "\n", - "\n", - "def load(name, stem):\n", - " return np.loadtxt(DATA / f\"{name}_{stem}.csv\", delimiter=\",\", ndmin=2)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c003", - "metadata": {}, - "outputs": [], - "source": [ - "for name in DATASETS:\n", - " X, y, Xt = load(name, \"X\"), load(name, \"y\").ravel(), load(name, \"Xt\")[:50]\n", - " d = X.shape[1]\n", - " theta0 = np.full((1, d), 0.3)\n", - " kf = lk.Kriging(y, X, \"matern5_2\", regmodel=\"constant\", optim=\"none\",\n", - " parameters={\"theta\": theta0, \"is_theta_estim\": False})\n", - " put(name, \"fixed_sigma2\", kf.sigma2())\n", - " put(name, \"fixed_beta\", kf.beta())\n", - " put(name, \"fixed_ll\", kf.logLikelihood())\n", - " put(name, \"fixed_loo\", kf.leaveOneOut())\n", - " m, s, *_ = kf.predict(Xt, return_stdev=True)\n", - " put(name, \"fixed_mean\", m)\n", - " put(name, \"fixed_stdev\", s)\n", - " ll, grad, _ = kf.logLikelihoodFun(theta0.ravel(), True)\n", - " put(name, \"llfun\", ll)\n", - " put(name, \"llfun_grad\", grad)\n", - " suite.check_close(f\"{name}: logLikelihood() == logLikelihoodFun(theta0)\", kf.logLikelihood(), ll, rtol=1e-10)\n", - " k = lk.Kriging(y, X, \"matern5_2\")\n", - " put(name, \"fit_theta\", k.theta())\n", - " put(name, \"fit_ll\", k.logLikelihood())\n", - " put(name, \"fit_mean\", k.predict(Xt, return_stdev=False)[0])\n", - " put(name, \"sim\", np.ravel(np.asarray(kf.simulate(3, 123, Xt[:5])), order=\"F\")) # column-major, as R/Octave/Julia" - ] - }, - { - "cell_type": "markdown", - "id": "c004", - "metadata": {}, - "source": [ - "## Other classes and fit methods (Branin)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c005", - "metadata": {}, - "outputs": [], - "source": [ - "name = \"branin\"\n", - "X, y, Xt = load(name, \"X\"), load(name, \"y\").ravel(), load(name, \"Xt\")[:50]\n", - "theta0 = np.full((1, 2), 0.3)\n", - "s2 = float(np.var(y)); nug = 0.01 * s2\n", - "fixed = {\"theta\": theta0, \"is_theta_estim\": False, \"sigma2\": s2, \"is_sigma2_estim\": False}\n", - "kn = lk.Kriging(y, X, \"matern5_2\", noise=\"nugget\", optim=\"none\",\n", - " parameters={**fixed, \"nugget\": nug, \"is_nugget_estim\": False})\n", - "m, s, *_ = kn.predict(Xt, return_stdev=True)\n", - "put(name, \"nugget_fixed_mean\", m); put(name, \"nugget_fixed_stdev\", s)\n", - "kh = lk.Kriging(y, X, \"matern5_2\", noise=np.full(len(y), nug), optim=\"none\", parameters=fixed)\n", - "m, s, *_ = kh.predict(Xt, return_stdev=True)\n", - "put(name, \"noise_fixed_mean\", m); put(name, \"noise_fixed_stdev\", s)\n", - "kf = lk.Kriging(y, X, \"matern5_2\", noise=\"nugget\")\n", - "put(name, \"nugget_fit_ll\", kf.logLikelihood()); put(name, \"nugget_fit_mean\", kf.predict(Xt, return_stdev=False)[0])\n", - "for meth in [\"vecchia\", \"nystrom\"] if compat.has(\"method\") else []:\n", - " ka = lk.Kriging(y, X, \"matern5_2\", optim=\"none\", parameters={\"theta\": theta0, \"is_theta_estim\": False},\n", - " method=f\"{meth}(10)\")\n", - " put(name, f\"{meth}_fixed_ll\", ka.logLikelihood(method=meth))\n", - " put(name, f\"{meth}_fixed_mean\", ka.predict(Xt, return_stdev=False, method=meth)[0])\n", - "if not compat.has(\"method\"):\n", - " suite.check_skip(\"vecchia / nystrom quantities\", \"pylibkriging without the method= argument\")\n", - "kw = lk.WarpKriging(y, X, [\"kumaraswamy\", \"kumaraswamy\"], \"matern5_2\")\n", - "put(name, \"warp_fit_mean\", kw.predict(Xt, return_stdev=False)[0])\n", - "km = lk.MLPKriging(y, X, hidden_dims=[8], d_out=2, activation=\"tanh\", kernel=\"matern5_2\", optim=\"BFGS10\")\n", - "put(name, \"mlp_fit_mean\", km.predict(Xt, return_stdev=False)[0])\n", - "kk = lk.NestedKriging(y, X, \"matern5_2\", nb_groups=2, aggregation=\"NK\", seed=1)\n", - "put(name, \"nested_fit_mean\", kk.predict(Xt, return_stdev=False)[0])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c006", - "metadata": {}, - "outputs": [], - "source": [ - "RESULTS_DIR.mkdir(exist_ok=True)\n", - "with open(RESULTS_DIR / \"xbind_python.csv\", \"w\", newline=\"\") as fh:\n", - " w = csv.writer(fh)\n", - " w.writerow([\"lang\", \"dataset\", \"quantity\", \"index\", \"value\"])\n", - " w.writerows(rows)\n", - "print(len(rows), \"values written\")" - ] - }, - { - "cell_type": "markdown", - "id": "c007", - "metadata": {}, - "source": [ - "## The reference values\n", - "\n", - "Number of values per quantity and dataset (the report compares each of them\n", - "with the other bindings), and the default fits they come from: predicted vs\n", - "true values on the 50 test points." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c008", - "metadata": {}, - "outputs": [], - "source": [ - "ref = pd.DataFrame(rows, columns=[\"lang\", \"dataset\", \"quantity\", \"index\", \"value\"])\n", - "ref[\"value\"] = ref[\"value\"].astype(float)\n", - "display(ref.groupby([\"quantity\", \"dataset\"])[\"value\"].agg([\"size\", \"first\"]).unstack(\"dataset\"))\n", - "fig, axes = plt.subplots(1, len(DATASETS), figsize=(12, 3.6))\n", - "for ax, name in zip(axes, DATASETS):\n", - " yt = load(name, \"yt\").ravel()[:50]\n", - " m = ref[(ref.dataset == name) & (ref.quantity == \"fit_mean\")][\"value\"].to_numpy()\n", - " ax.plot(yt, m, \"o\", ms=4); ax.plot([yt.min(), yt.max()], [yt.min(), yt.max()], \"k--\", lw=.8)\n", - " q2 = 1 - np.sum((yt - m) ** 2) / np.sum((yt - yt.mean()) ** 2)\n", - " suite.metric(name, \"pylibkriging default fit\", q2=q2)\n", - " ax.set_title(f\"{name}: Q² = {q2:.3f}\"); ax.set_xlabel(\"true\"); ax.set_ylabel(\"predicted\")\n", - "plt.tight_layout(); plt.show()" - ] - }, - { - "cell_type": "markdown", - "id": "c009", - "metadata": {}, - "source": [ - "## Summary\n", - "\n", - "The metrics of this notebook, the number of checks per status, and every check that\n", - "did not pass (FAIL: error-level, fails the run; WARN: warn-level; KNOWN: listed in\n", - "`tools/known_issues.csv`, with its issue; SKIP: feature absent from the installed version).\n", - "All of them are also written to `results/cross-binding-reference_all-classes_python.checks.csv` and `.metrics.csv`." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c010", - "metadata": {}, - "outputs": [], - "source": [ - "suite.summary()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c011", - "metadata": {}, - "outputs": [], - "source": [ - "suite.finalize()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/cross-binding_all-classes_R.ipynb b/cross-binding_all-classes_R.ipynb deleted file mode 100644 index ddca363..0000000 --- a/cross-binding_all-classes_R.ipynb +++ /dev/null @@ -1,203 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "c000", - "metadata": {}, - "source": [ - "# R · all classes — cross-binding consistency: the R side\n", - "\n", - "**Idea.** The R side of the cross-binding protocol: rlibkriging must give the\n", - "numbers of pylibkriging on the same data, for every model class and fit method.\n", - "\n", - "**Reference.** The Python values (`cross-binding-reference_all-\n", - "classes_python.ipynb`), compared by `tools/report.py`.\n", - "\n", - "Same protocol as `cross-binding-reference_all-classes_python.ipynb` (see there), on the\n", - "shared datasets of `data/`; writes `results/xbind_R.csv`, compared with the\n", - "Python values by `tools/report.py`." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c001", - "metadata": {}, - "outputs": [], - "source": [ - "suppressPackageStartupMessages(library(rlibkriging))\n", - "source(\"tools/R/gallery.R\")\n", - "s <- suite(\"cross-binding_all-classes_R\")\n", - "cat(\"rlibkriging\", as.character(packageVersion(\"rlibkriging\")), \"\\n\")\n", - "rows <- list()\n", - "put <- function(dataset, quantity, values) {\n", - " v <- as.numeric(values)\n", - " rows[[length(rows) + 1]] <<- data.frame(lang = \"R\", dataset = dataset, quantity = quantity,\n", - " index = seq_along(v), value = sprintf(\"%.17g\", v))\n", - "}" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c002", - "metadata": {}, - "outputs": [], - "source": [ - "for (name in c(\"branin\", \"hartmann3\", \"borehole\")) {\n", - " X <- read_data(name, \"X\"); y <- as.numeric(read_data(name, \"y\")); Xt <- read_data(name, \"Xt\")[1:50, , drop = FALSE]\n", - " d <- ncol(X)\n", - " theta0 <- matrix(0.3, nrow = 1, ncol = d)\n", - " kf <- Kriging(y, X, \"matern5_2\", regmodel = \"constant\", optim = \"none\",\n", - " parameters = list(theta = theta0, is_theta_estim = FALSE))\n", - " put(name, \"fixed_sigma2\", kf$sigma2())\n", - " put(name, \"fixed_beta\", kf$beta())\n", - " put(name, \"fixed_ll\", logLikelihood(kf))\n", - " put(name, \"fixed_loo\", leaveOneOut(kf))\n", - " p <- predict(kf, Xt, return_stdev = TRUE)\n", - " put(name, \"fixed_mean\", p$mean)\n", - " put(name, \"fixed_stdev\", p$stdev)\n", - " llf <- logLikelihoodFun(kf, theta0, return_grad = TRUE)\n", - " put(name, \"llfun\", llf$logLikelihood)\n", - " put(name, \"llfun_grad\", llf$logLikelihoodGrad)\n", - " check_close(s, paste0(name, \": logLikelihood() == logLikelihoodFun(theta0)\"), logLikelihood(kf), llf$logLikelihood, rtol = 1e-10)\n", - " k <- Kriging(y, X, \"matern5_2\")\n", - " put(name, \"fit_theta\", k$theta())\n", - " put(name, \"fit_ll\", logLikelihood(k))\n", - " put(name, \"fit_mean\", predict(k, Xt, return_stdev = FALSE)$mean)\n", - " put(name, \"sim\", simulate(kf, nsim = 3, seed = 123, x = Xt[1:5, , drop = FALSE]))\n", - "}" - ] - }, - { - "cell_type": "markdown", - "id": "c003", - "metadata": {}, - "source": [ - "## Other classes and fit methods (Branin)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c004", - "metadata": {}, - "outputs": [], - "source": [ - "name <- \"branin\"\n", - "X <- read_data(name, \"X\"); y <- as.numeric(read_data(name, \"y\")); Xt <- read_data(name, \"Xt\")[1:50, , drop = FALSE]\n", - "theta0 <- matrix(0.3, nrow = 1, ncol = 2)\n", - "s2 <- mean((y - mean(y))^2); nug <- 0.01 * s2 # population variance, as numpy.var\n", - "fixed <- list(theta = theta0, is_theta_estim = FALSE, sigma2 = s2, is_sigma2_estim = FALSE)\n", - "kn <- Kriging(y, X, \"matern5_2\", noise = \"nugget\", optim = \"none\",\n", - " parameters = c(fixed, list(nugget = nug, is_nugget_estim = FALSE)))\n", - "p <- predict(kn, Xt, return_stdev = TRUE)\n", - "put(name, \"nugget_fixed_mean\", p$mean); put(name, \"nugget_fixed_stdev\", p$stdev)\n", - "kh <- Kriging(y, X, \"matern5_2\", noise = rep(nug, length(y)), optim = \"none\", parameters = fixed)\n", - "p <- predict(kh, Xt, return_stdev = TRUE)\n", - "put(name, \"noise_fixed_mean\", p$mean); put(name, \"noise_fixed_stdev\", p$stdev)\n", - "kf <- Kriging(y, X, \"matern5_2\", noise = \"nugget\")\n", - "put(name, \"nugget_fit_ll\", logLikelihood(kf)); put(name, \"nugget_fit_mean\", predict(kf, Xt, return_stdev = FALSE)$mean)\n", - "for (meth in c(\"vecchia\", \"nystrom\")) tryCatch({ # method= is absent from older releases\n", - " ka <- Kriging(y, X, \"matern5_2\", optim = \"none\", parameters = list(theta = theta0, is_theta_estim = FALSE),\n", - " method = paste0(meth, \"(10)\"))\n", - " put(name, paste0(meth, \"_fixed_ll\"), ka$logLikelihood(method = meth))\n", - " put(name, paste0(meth, \"_fixed_mean\"), predict(ka, Xt, return_stdev = FALSE, method = meth)$mean)\n", - "}, error = function(e) message(meth, \" skipped (reported as missing): \", conditionMessage(e)))\n", - "kw <- WarpKriging(y, X, warping = c(\"kumaraswamy\", \"kumaraswamy\"), kernel = \"matern5_2\")\n", - "put(name, \"warp_fit_mean\", predict(kw, Xt, return_stdev = FALSE)$mean)\n", - "km <- MLPKriging(y, X, hidden_dims = c(8), d_out = 2, activation = \"tanh\", kernel = \"matern5_2\", optim = \"BFGS10\")\n", - "put(name, \"mlp_fit_mean\", predict(km, Xt, return_stdev = FALSE)$mean)\n", - "kk <- NestedKriging(y, X, kernel = \"matern5_2\", nb_groups = 2, aggregation = \"NK\", seed = 1)\n", - "put(name, \"nested_fit_mean\", predict(kk, Xt, return_stdev = FALSE)$mean)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c005", - "metadata": {}, - "outputs": [], - "source": [ - "out <- do.call(rbind, rows)\n", - "write.csv(out, file.path(results_dir, \"xbind_R.csv\"), row.names = FALSE, quote = FALSE)\n", - "cat(nrow(out), \"values written\\n\")" - ] - }, - { - "cell_type": "markdown", - "id": "c006", - "metadata": {}, - "source": [ - "## The values written\n", - "\n", - "Number of values per quantity and dataset (each compared with Python by the\n", - "report), and the default fits they come from: predicted vs true on the 50 test points." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c007", - "metadata": {}, - "outputs": [], - "source": [ - "print(with(out, table(quantity, dataset)))\n", - "op <- par(mfrow = c(1, 3), mar = c(4, 4, 2, 1))\n", - "for (name in c(\"branin\", \"hartmann3\", \"borehole\")) {\n", - " yt <- as.numeric(read_data(name, \"yt\"))[1:50]\n", - " m <- as.numeric(out$value[out$dataset == name & out$quantity == \"fit_mean\"])\n", - " q <- q2(yt, m); metric(s, name, \"rlibkriging default fit\", q2 = q)\n", - " plot(yt, m, xlab = \"true\", ylab = \"predicted\", main = sprintf(\"%s: Q2 = %.3f\", name, q), pch = 16, cex = .7)\n", - " abline(0, 1, lty = 2)\n", - "}\n", - "par(op)" - ] - }, - { - "cell_type": "markdown", - "id": "c008", - "metadata": {}, - "source": [ - "## Summary\n", - "\n", - "The metrics of this notebook, the number of checks per status, and every check that\n", - "did not pass (FAIL: error-level, fails the run; WARN: warn-level; KNOWN: listed in\n", - "`tools/known_issues.csv`, with its issue; SKIP: feature absent from the installed version).\n", - "All of them are also written to `results/cross-binding_all-classes_R.checks.csv` and `.metrics.csv`." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c009", - "metadata": {}, - "outputs": [], - "source": [ - "suite_summary(s)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c010", - "metadata": {}, - "outputs": [], - "source": [ - "suite_finalize(s)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "R", - "language": "R", - "name": "ir" - }, - "language_info": { - "name": "R" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/cross-binding_all-classes_julia.ipynb b/cross-binding_all-classes_julia.ipynb deleted file mode 100644 index 85f00df..0000000 --- a/cross-binding_all-classes_julia.ipynb +++ /dev/null @@ -1,200 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "c000", - "metadata": {}, - "source": [ - "# Julia · all classes — cross-binding consistency: the Julia side\n", - "\n", - "**Idea.** The Julia side of the cross-binding protocol: JLibKriging must give the\n", - "numbers of pylibkriging on the same data, for every model class and fit method.\n", - "\n", - "**Reference.** The Python values (`cross-binding-reference_all-\n", - "classes_python.ipynb`), compared by `tools/report.py`.\n", - "\n", - "Same protocol as `cross-binding-reference_all-classes_python.ipynb` (see there), on\n", - "the shared datasets of `data/`; writes `results/xbind_julia.csv`, compared with\n", - "the Python values by `tools/report.py`.\n", - "\n", - "The module is `JLibKriging` (the packaged binding, [JLibKriging.jl](https://github.com/libKriging/JLibKriging.jl)),\n", - "imported under a short name so that its exported accessors (`X`, `y`, `theta`, ...)\n", - "do not clash with the variables of the notebook." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c001", - "metadata": {}, - "outputs": [], - "source": [ - "import JLibKriging as LK\n", - "include(\"tools/julia/gallery.jl\")\n", - "s = Suite(\"cross-binding_all-classes_julia\")\n", - "rows = Vector{Vector{String}}()\n", - "record!(dataset, quantity, values) =\n", - " for (i, v) in enumerate(vec(Float64.(collect(values))))\n", - " push!(rows, [\"julia\", dataset, quantity, string(i), @sprintf(\"%.17g\", v)])\n", - " end" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c002", - "metadata": {}, - "outputs": [], - "source": [ - "for name in (\"branin\", \"hartmann3\", \"borehole\")\n", - " Xd = read_data(name, \"X\"); yd = vec(read_data(name, \"y\")); Xt = read_data(name, \"Xt\")[1:50, :]\n", - " d = size(Xd, 2)\n", - " theta0 = fill(0.3, d)\n", - " kf = LK.Kriging(yd, Xd, \"matern5_2\"; regmodel=\"constant\", optim=\"none\", theta=theta0, is_theta_estim=false)\n", - " record!(name, \"fixed_sigma2\", LK.sigma2(kf))\n", - " record!(name, \"fixed_beta\", LK.beta(kf))\n", - " record!(name, \"fixed_ll\", LK.log_likelihood(kf))\n", - " record!(name, \"fixed_loo\", LK.leave_one_out(kf))\n", - " p = LK.predict(kf, Xt; return_stdev=true)\n", - " record!(name, \"fixed_mean\", p.mean)\n", - " record!(name, \"fixed_stdev\", p.stdev)\n", - " llf = LK.log_likelihood_fun(kf, theta0; return_grad=true)\n", - " record!(name, \"llfun\", llf.ll)\n", - " record!(name, \"llfun_grad\", llf.grad)\n", - " check_close!(s, \"$name: log_likelihood() == log_likelihood_fun(theta0)\", LK.log_likelihood(kf), llf.ll; rtol=1e-10)\n", - " k = LK.Kriging(yd, Xd, \"matern5_2\")\n", - " record!(name, \"fit_theta\", LK.theta(k))\n", - " record!(name, \"fit_ll\", LK.log_likelihood(k))\n", - " record!(name, \"fit_mean\", LK.predict(k, Xt; return_stdev=false).mean)\n", - " record!(name, \"sim\", LK.simulate(kf, 3, 123, Xt[1:5, :]))\n", - "end" - ] - }, - { - "cell_type": "markdown", - "id": "c003", - "metadata": {}, - "source": [ - "## Other classes and fit methods (Branin)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c004", - "metadata": {}, - "outputs": [], - "source": [ - "name = \"branin\"\n", - "Xd = read_data(name, \"X\"); yd = vec(read_data(name, \"y\")); Xt = read_data(name, \"Xt\")[1:50, :]\n", - "theta0 = [0.3, 0.3]\n", - "s2 = sum((yd .- sum(yd) / length(yd)) .^ 2) / length(yd); nug = 0.01 * s2 # population variance, as numpy.var\n", - "kn = LK.Kriging(yd, Xd, \"matern5_2\"; noise=\"nugget\", optim=\"none\", theta=theta0, is_theta_estim=false,\n", - " sigma2=s2, is_sigma2_estim=false, nugget=nug, is_nugget_estim=false)\n", - "p = LK.predict(kn, Xt; return_stdev=true)\n", - "record!(name, \"nugget_fixed_mean\", p.mean); record!(name, \"nugget_fixed_stdev\", p.stdev)\n", - "kh = LK.Kriging(yd, Xd, \"matern5_2\"; noise=fill(nug, length(yd)), optim=\"none\", theta=theta0, is_theta_estim=false,\n", - " sigma2=s2, is_sigma2_estim=false)\n", - "p = LK.predict(kh, Xt; return_stdev=true)\n", - "record!(name, \"noise_fixed_mean\", p.mean); record!(name, \"noise_fixed_stdev\", p.stdev)\n", - "kf = LK.Kriging(yd, Xd, \"matern5_2\"; noise=\"nugget\")\n", - "record!(name, \"nugget_fit_ll\", LK.log_likelihood(kf))\n", - "record!(name, \"nugget_fit_mean\", LK.predict(kf, Xt; return_stdev=false).mean)\n", - "for meth in (\"vecchia\", \"nystrom\")\n", - " ka = LK.Kriging(yd, Xd, \"matern5_2\"; optim=\"none\", theta=theta0, is_theta_estim=false, method=\"$meth(10)\")\n", - " record!(name, \"$(meth)_fixed_ll\", LK.log_likelihood(ka; method=meth))\n", - " record!(name, \"$(meth)_fixed_mean\", LK.predict(ka, Xt; return_stdev=false, method=meth).mean)\n", - "end\n", - "kw = LK.WarpKriging(yd, Xd, [\"kumaraswamy\", \"kumaraswamy\"], \"matern5_2\")\n", - "record!(name, \"warp_fit_mean\", LK.predict(kw, Xt; return_stdev=false).mean)\n", - "km = LK.MLPKriging(yd, Xd, [8], 2; activation=\"tanh\", kernel=\"matern5_2\", optim=\"BFGS10\")\n", - "record!(name, \"mlp_fit_mean\", LK.predict(km, Xt; return_stdev=false).mean)\n", - "kk = LK.NestedKriging(yd, Xd, \"matern5_2\", 2; aggregation=\"NK\", seed=1)\n", - "record!(name, \"nested_fit_mean\", LK.predict(kk, Xt; return_stdev=false).mean)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c005", - "metadata": {}, - "outputs": [], - "source": [ - "open(joinpath(RESULTS_DIR, \"xbind_julia.csv\"), \"w\") do io\n", - " println(io, \"lang,dataset,quantity,index,value\")\n", - " for r in rows; println(io, join(r, \",\")); end\n", - "end\n", - "println(length(rows), \" values written\")" - ] - }, - { - "cell_type": "markdown", - "id": "c006", - "metadata": {}, - "source": [ - "## The values written\n", - "\n", - "First value of each quantity and dataset (each compared with Python by the\n", - "report; plots are left to the Python reference, to avoid a plotting package here)." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c007", - "metadata": {}, - "outputs": [], - "source": [ - "for key in unique([(r[2], r[3]) for r in rows])\n", - " vals = [r[5] for r in rows if (r[2], r[3]) == key]\n", - " println(rpad(\"$(key[1]) $(key[2])\", 36), vals[1], \" (\", length(vals), \" values)\")\n", - "end" - ] - }, - { - "cell_type": "markdown", - "id": "c008", - "metadata": {}, - "source": [ - "## Summary\n", - "\n", - "The metrics of this notebook, the number of checks per status, and every check that\n", - "did not pass (FAIL: error-level, fails the run; WARN: warn-level; KNOWN: listed in\n", - "`tools/known_issues.csv`, with its issue; SKIP: feature absent from the installed version).\n", - "All of them are also written to `results/cross-binding_all-classes_julia.checks.csv` and `.metrics.csv`." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c009", - "metadata": {}, - "outputs": [], - "source": [ - "suite_summary(s)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c010", - "metadata": {}, - "outputs": [], - "source": [ - "suite_finalize!(s)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Julia 1.11", - "language": "julia", - "name": "julia-1.11" - }, - "language_info": { - "name": "julia" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/cross-binding_all-classes_octave.ipynb b/cross-binding_all-classes_octave.ipynb deleted file mode 100644 index f806f0d..0000000 --- a/cross-binding_all-classes_octave.ipynb +++ /dev/null @@ -1,218 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "c000", - "metadata": {}, - "source": [ - "# Octave · all classes — cross-binding consistency: the Octave side\n", - "\n", - "**Idea.** The Octave side of the cross-binding protocol: mLibKriging must give the\n", - "numbers of pylibkriging on the same data, for every model class and fit method.\n", - "\n", - "**Reference.** The Python values (`cross-binding-reference_all-\n", - "classes_python.ipynb`), compared by `tools/report.py`.\n", - "\n", - "Same protocol as `cross-binding-reference_all-classes_python.ipynb` (see there), on the\n", - "shared datasets of `data/`; writes `results/xbind_octave.csv`, compared with\n", - "the Python values by `tools/report.py`. mLibKriging must be on the path\n", - "(`OCTAVE_PATH` or `addpath`)." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c001", - "metadata": {}, - "outputs": [], - "source": [ - "source(\"tools/octave/gallery.m\");\n", - "s = suite(\"cross-binding_all-classes_octave\");\n", - "out = {};\n", - "function out = put(out, dataset, quantity, values)\n", - " v = double(values(:));\n", - " for i = 1:numel(v)\n", - " out(end + 1, :) = {\"octave\", dataset, quantity, i, sprintf(\"%.17g\", v(i))};\n", - " end\n", - "end" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c002", - "metadata": {}, - "outputs": [], - "source": [ - "for name = {\"branin\", \"hartmann3\", \"borehole\"}\n", - " name = name{1};\n", - " X = read_data(name, \"X\"); y = read_data(name, \"y\"); Xt = read_data(name, \"Xt\")(1:50, :);\n", - " d = columns(X);\n", - " theta0 = 0.3 * ones(1, d);\n", - " kf = Kriging(y, X, \"matern5_2\", \"constant\", false, \"none\", \"LL\", Params(\"theta\", theta0, \"is_theta_estim\", false));\n", - " v = kf.sigma2(); out = put(out, name, \"fixed_sigma2\", v);\n", - " v = kf.beta(); out = put(out, name, \"fixed_beta\", v);\n", - " ll = kf.logLikelihood();\n", - " out = put(out, name, \"fixed_ll\", ll);\n", - " loo = kf.leaveOneOut();\n", - " out = put(out, name, \"fixed_loo\", loo);\n", - " [m, sd] = kf.predict(Xt, true, false, false);\n", - " out = put(out, name, \"fixed_mean\", m);\n", - " out = put(out, name, \"fixed_stdev\", sd);\n", - " [llf, grad] = kf.logLikelihoodFun(theta0', true, false);\n", - " out = put(out, name, \"llfun\", llf);\n", - " out = put(out, name, \"llfun_grad\", grad);\n", - " s = check_close(s, [name \": logLikelihood() == logLikelihoodFun(theta0)\"], ll, llf, 1e-10);\n", - " k = Kriging(y, X, \"matern5_2\");\n", - " v = k.theta(); out = put(out, name, \"fit_theta\", v);\n", - " llk = k.logLikelihood();\n", - " out = put(out, name, \"fit_ll\", llk);\n", - " [mk, ~] = k.predict(Xt, false, false, false);\n", - " out = put(out, name, \"fit_mean\", mk);\n", - " S = kf.simulate(int32(3), int32(123), Xt(1:5, :), false);\n", - " out = put(out, name, \"sim\", S);\n", - "end" - ] - }, - { - "cell_type": "markdown", - "id": "c003", - "metadata": {}, - "source": [ - "## Other classes and fit methods (Branin)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c004", - "metadata": {}, - "outputs": [], - "source": [ - "name = \"branin\";\n", - "X = read_data(name, \"X\"); y = read_data(name, \"y\"); Xt = read_data(name, \"Xt\")(1:50, :);\n", - "theta0 = [0.3, 0.3];\n", - "s2 = mean((y - mean(y)).^2); nug = 0.01 * s2; % population variance, as numpy.var\n", - "kn = Kriging(y, X, \"matern5_2\", \"constant\", false, \"none\", \"LL\", ...\n", - " Params(\"theta\", theta0, \"is_theta_estim\", false, \"sigma2\", s2, \"is_sigma2_estim\", false, ...\n", - " \"nugget\", nug, \"is_nugget_estim\", false), \"nugget\");\n", - "[m, sd] = kn.predict(Xt, true, false, false);\n", - "out = put(out, name, \"nugget_fixed_mean\", m); out = put(out, name, \"nugget_fixed_stdev\", sd);\n", - "kh = Kriging(y, X, \"matern5_2\", \"constant\", false, \"none\", \"LL\", ...\n", - " Params(\"theta\", theta0, \"is_theta_estim\", false, \"sigma2\", s2, \"is_sigma2_estim\", false), ...\n", - " \"heterogeneous\", nug * ones(size(y)));\n", - "[m, sd] = kh.predict(Xt, true, false, false);\n", - "out = put(out, name, \"noise_fixed_mean\", m); out = put(out, name, \"noise_fixed_stdev\", sd);\n", - "kf = Kriging(y, X, \"matern5_2\", \"constant\", false, \"BFGS\", \"LL\", [], \"nugget\");\n", - "ll = kf.logLikelihood(); out = put(out, name, \"nugget_fit_ll\", ll);\n", - "m = kf.predict(Xt, false, false, false); out = put(out, name, \"nugget_fit_mean\", m);\n", - "for meth = {\"vecchia\", \"nystrom\"}\n", - " meth = meth{1};\n", - " try % method is absent from older releases\n", - " ka = Kriging(y, X, \"matern5_2\", \"constant\", false, \"none\", \"LL\", Params(\"theta\", theta0, \"is_theta_estim\", false), ...\n", - " \"none\", [], [meth \"(10)\"]);\n", - " ll = ka.logLikelihood(meth); out = put(out, name, [meth \"_fixed_ll\"], ll);\n", - " m = ka.predict(Xt, false, false, false, meth); out = put(out, name, [meth \"_fixed_mean\"], m);\n", - " catch err\n", - " printf(\"%s skipped (reported as missing): %s\\n\", meth, err.message);\n", - " end\n", - "end\n", - "kw = WarpKriging(y, X, {\"kumaraswamy\", \"kumaraswamy\"}, \"matern5_2\");\n", - "m = kw.predict(Xt, false, false, false); out = put(out, name, \"warp_fit_mean\", m);\n", - "km = MLPKriging(y, X, [8], 2, \"tanh\", \"matern5_2\", \"constant\", false, \"BFGS10\");\n", - "m = km.predict(Xt, false, false, false); out = put(out, name, \"mlp_fit_mean\", m);\n", - "kk = NestedKriging(y, X, \"matern5_2\", 2, \"NK\", \"kmeans\", 1);\n", - "m = kk.predict(Xt); out = put(out, name, \"nested_fit_mean\", m);" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c005", - "metadata": {}, - "outputs": [], - "source": [ - "d = getenv(\"LKGALLERY_RESULTS\"); if isempty(d), d = \"results\"; end\n", - "if ~exist(d, \"dir\"), mkdir(d); end\n", - "fid = fopen(fullfile(d, \"xbind_octave.csv\"), \"w\");\n", - "fprintf(fid, \"lang,dataset,quantity,index,value\\n\");\n", - "for i = 1:size(out, 1)\n", - " fprintf(fid, \"%s,%s,%s,%d,%s\\n\", out{i, :});\n", - "end\n", - "fclose(fid);\n", - "printf(\"%d values written\\n\", size(out, 1));" - ] - }, - { - "cell_type": "markdown", - "id": "c006", - "metadata": {}, - "source": [ - "## The values written\n", - "\n", - "First value of each quantity and dataset (each compared with Python by the\n", - "report; plots are left to the Python reference, Octave's graphics toolkit not\n", - "being available on every runner)." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c007", - "metadata": {}, - "outputs": [], - "source": [ - "keys = strcat(out(:, 2), \" \", out(:, 3));\n", - "[u, first] = unique(keys, \"first\");\n", - "for i = 1:numel(u)\n", - " printf(\"%-36s %s (%d values)\\n\", u{i}, out{first(i), 5}, sum(strcmp(keys, u{i})));\n", - "end" - ] - }, - { - "cell_type": "markdown", - "id": "c008", - "metadata": {}, - "source": [ - "## Summary\n", - "\n", - "The metrics of this notebook, the number of checks per status, and every check that\n", - "did not pass (FAIL: error-level, fails the run; WARN: warn-level; KNOWN: listed in\n", - "`tools/known_issues.csv`, with its issue; SKIP: feature absent from the installed version).\n", - "All of them are also written to `results/cross-binding_all-classes_octave.checks.csv` and `.metrics.csv`." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c009", - "metadata": {}, - "outputs": [], - "source": [ - "suite_summary(s);" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c010", - "metadata": {}, - "outputs": [], - "source": [ - "suite_finalize(s);" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Octave", - "language": "octave", - "name": "octave" - }, - "language_info": { - "name": "octave" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/deep-kernel-projected-branin_MLPKriging_python.ipynb b/deep-kernel-projected-branin_MLPKriging_python.ipynb deleted file mode 100644 index 6db067c..0000000 --- a/deep-kernel-projected-branin_MLPKriging_python.ipynb +++ /dev/null @@ -1,207 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "c000", - "metadata": {}, - "source": [ - "# Python · MLPKriging — deep kernel on a 10D function with a 2D structure\n", - "\n", - "**Idea.** A deep kernel (an MLP feature map before the kernel) can find a low-\n", - "dimensional structure that a per-axis kernel cannot: Branin of a 2D projection of 10\n", - "inputs.\n", - "\n", - "**Reference.** Plain `Kriging` on the same data.\n", - "\n", - "f(x) = Branin(Wx) for x ∈ [0,1]¹⁰ and a fixed 2 × 10 projection W: the\n", - "function only varies in a 2D subspace, which a joint feature map (an MLP from\n", - "ℝ¹⁰ to ℝ², then a stationary kernel) can find, whereas a per-axis kernel\n", - "cannot.\n", - "\n", - "Checks: finite predictions; with a multistart optimizer, a Q² floor\n", - "(error-level); the single-start fits compared with plain `Kriging`\n", - "(warn-level), for two activations (`tanh`, smooth; `selu`, the default, with\n", - "a kink at 0)." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c001", - "metadata": {}, - "outputs": [], - "source": [ - "# Setup for running this notebook outside the repository (Colab, Binder, a lone copy):\n", - "# nothing to do in the repository itself (CI, local runs). Otherwise the repository\n", - "# (datasets of data/, helpers of tools/python/gallery.py) is cloned and the released pylibkriging installed.\n", - "import importlib\n", - "import os\n", - "import subprocess\n", - "import sys\n", - "from pathlib import Path\n", - "\n", - "if not Path(\"tools/known_issues.csv\").is_file():\n", - " if not Path(\"gallery/tools/known_issues.csv\").is_file():\n", - " subprocess.run([\"git\", \"clone\", \"--depth\", \"1\", \"https://github.com/libKriging/gallery\"], check=True)\n", - " os.chdir(\"gallery\")\n", - " try:\n", - " import pylibkriging # noqa: F401\n", - " except ImportError:\n", - " r = subprocess.run([sys.executable, \"-m\", \"pip\", \"install\", \"--quiet\", \"pylibkriging\"],\n", - " capture_output=True, text=True)\n", - " if r.returncode != 0:\n", - " raise RuntimeError(\n", - " f\"pylibkriging could not be installed for Python {sys.version_info.major}.{sys.version_info.minor} \"\n", - " \"(PyPI has wheels for CPython 3.10 to 3.12 only). On Colab: Runtime > Change runtime version > \"\n", - " \"2026.07 (Python 3.12), then run again.\\n\" + r.stderr[-500:])\n", - " importlib.invalidate_caches()\n", - "sys.path.insert(0, str(Path(\"tools/python\").resolve())) # for \"import gallery\" (tools/python/gallery.py)\n", - "print(\"working directory:\", os.getcwd())" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c002", - "metadata": {}, - "outputs": [], - "source": [ - "import time\n", - "\n", - "import matplotlib.pyplot as plt\n", - "import numpy as np\n", - "import pylibkriging as lk\n", - "\n", - "import gallery\n", - "from gallery import TESTS, design, metrics, compat\n", - "\n", - "suite = gallery.Suite(\"deep-kernel-projected-branin_MLPKriging_python\")\n", - "print(\"pylibkriging\", compat.version())\n", - "\n", - "\n", - "def pred(k, X):\n", - " m, s, *_ = k.predict(X, return_stdev=True)\n", - " return np.ravel(m), np.ravel(s)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c003", - "metadata": {}, - "outputs": [], - "source": [ - "if compat.has(\"MLPKriging\"):\n", - " rng = np.random.default_rng(43)\n", - " W = rng.normal(size=(2, 10))\n", - " W /= np.abs(W).sum(axis=1, keepdims=True)\n", - " tfb = TESTS[\"branin\"]\n", - "\n", - " def projected(X):\n", - " Z = X @ W.T\n", - " Z = (Z - (W.clip(max=0).sum(axis=1))) / np.abs(W).sum(axis=1) # back to [0,1]^2\n", - " return tfb.unit(Z)\n", - "\n", - " X = design.lhs(120, 10, seed=44); Xt = design.uniform(2000, 10, seed=45)\n", - " y, yt = projected(X), projected(Xt)\n", - " kp = lk.Kriging(y, X, \"matern5_2\")\n", - " res_k = metrics.all_metrics(yt, *pred(kp, Xt))\n", - " suite.metric(\"projected_branin_10d\", \"Kriging\", **res_k)\n", - " for act in [\"tanh\", \"selu\"]:\n", - " for optim in [\"BFGS\", \"BFGS10\"]:\n", - " case = f\"MLPKriging({act}, {optim})\"\n", - " with suite.check_runs(f\"{case}: fit\"):\n", - " t0 = time.time()\n", - " km = lk.MLPKriging(y, X, hidden_dims=[16, 8], d_out=2, activation=act, kernel=\"matern5_2\",\n", - " optim=optim)\n", - " fit_s = time.time() - t0\n", - " m, s = pred(km, Xt)\n", - " suite.check_finite(f\"{case}: predictions finite\", m)\n", - " res = metrics.all_metrics(yt, m, s)\n", - " suite.metric(\"projected_branin_10d\", f\"MLPKriging {act} {optim}\", fit_s=fit_s,\n", - " ll=float(km.logLikelihood()), **res)\n", - " if optim == \"BFGS10\":\n", - " suite.check_greater(f\"{case}: Q2\", res[\"q2\"], 0.8)\n", - " else:\n", - " suite.check_greater(f\"{case}: Q2 - Q2(Kriging)\", res[\"q2\"] - res_k[\"q2\"], -0.1, severity=\"warn\")\n", - " tab = suite.table().loc[\"projected_branin_10d\"]\n", - " tab[\"q2\"].plot.barh(figsize=(6, 3)); plt.xlabel(\"Q² (test set)\"); plt.xlim(min(0, tab[\"q2\"].min()), 1)\n", - " plt.title(\"Branin(Wx), x in [0,1]^10\"); plt.show()\n", - " display(tab.round(4))\n", - "else:\n", - " suite.check_skip(\"MLPKriging\", \"not available in this pylibkriging\")" - ] - }, - { - "cell_type": "markdown", - "id": "c004", - "metadata": {}, - "source": [ - "## save / load, pickle, copy\n", - "\n", - "On Hartmann 3 (n = 40): a model saved and loaded back (class-specific and\n", - "generic `pylibkriging.load`), pickled, or deep-copied predicts exactly as the\n", - "original." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c005", - "metadata": {}, - "outputs": [], - "source": [ - "if compat.has(\"MLPKriging\"):\n", - " X, y, Xt, yt = design.dataset(TESTS[\"hartmann3\"], 40, n_test=100, seed=61)\n", - " with suite.check_runs(\"MLPKriging: fit (persistence)\"):\n", - " gallery.persistence.check(suite, \"MLPKriging\", lk.MLPKriging(y, X, hidden_dims=[8], d_out=2, activation=\"tanh\",\n", - " kernel=\"matern5_2\"), Xt, pred)" - ] - }, - { - "cell_type": "markdown", - "id": "c006", - "metadata": {}, - "source": [ - "## Summary\n", - "\n", - "The metrics of this notebook, the number of checks per status, and every check that\n", - "did not pass (FAIL: error-level, fails the run; WARN: warn-level; KNOWN: listed in\n", - "`tools/known_issues.csv`, with its issue; SKIP: feature absent from the installed version).\n", - "All of them are also written to `results/deep-kernel-projected-branin_MLPKriging_python.checks.csv` and `.metrics.csv`." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c007", - "metadata": {}, - "outputs": [], - "source": [ - "suite.summary()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c008", - "metadata": {}, - "outputs": [], - "source": [ - "suite.finalize()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/high-dimension-normalize-screening_Kriging_python.ipynb b/high-dimension-normalize-screening_Kriging_python.ipynb deleted file mode 100644 index cdbc8ec..0000000 --- a/high-dimension-normalize-screening_Kriging_python.ipynb +++ /dev/null @@ -1,225 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "c000", - "metadata": {}, - "source": [ - "# Python · Kriging — moderate and high dimension: engineering functions, `normalize`, screening\n", - "\n", - "**Idea.** From 6 to 20 inputs on engineering functions: accuracy, the effect of\n", - "`normalize` on inputs in physical units, and the ability of the fitted ranges θ to\n", - "tell influential inputs from inactive ones.\n", - "\n", - "**Reference.** The same model on inputs rescaled by hand, and the known active\n", - "inputs of the Welch function.\n", - "\n", - "| function | d | character |\n", - "|---|---|---|\n", - "| OTL circuit | 6 | smooth, inputs on very different scales |\n", - "| Piston | 7 | smooth, strongly non-linear in a few inputs |\n", - "| Borehole | 8 | smooth, 2–3 dominant inputs, raw scales from 0.05 to 10⁵ |\n", - "| Robot arm | 8 | oscillating (angles): hard for a stationary GP |\n", - "| Dette & Pepelyshev | 8 | curved, logarithmic terms |\n", - "| Wing weight | 10 | smooth, nearly additive |\n", - "| Welch et al. (1992) | 20 | screening problem: few active inputs, interactions |\n", - "\n", - "Checks:\n", - "* **`normalize=True` on raw inputs** gives the same accuracy as an explicit\n", - " rescaling to [0, 1] (they are the same model up to the optimizer);\n", - " inputs in native units (Borehole spans 0.05 to 50 000) must not break a fit;\n", - "* Q² floors per function (regression detection);\n", - "* the fitted θ of Welch's **inactive** inputs are larger than those of the\n", - " active ones (screening ability, warn-level).\n", - "\n", - "`MLPKriging` on a high-dimensional function with a low-dimensional structure:\n", - "`deep-kernel-projected-branin_MLPKriging_python.ipynb`." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c001", - "metadata": {}, - "outputs": [], - "source": [ - "# Setup for running this notebook outside the repository (Colab, Binder, a lone copy):\n", - "# nothing to do in the repository itself (CI, local runs). Otherwise the repository\n", - "# (datasets of data/, helpers of tools/python/gallery.py) is cloned and the released pylibkriging installed.\n", - "import importlib\n", - "import os\n", - "import subprocess\n", - "import sys\n", - "from pathlib import Path\n", - "\n", - "if not Path(\"tools/known_issues.csv\").is_file():\n", - " if not Path(\"gallery/tools/known_issues.csv\").is_file():\n", - " subprocess.run([\"git\", \"clone\", \"--depth\", \"1\", \"https://github.com/libKriging/gallery\"], check=True)\n", - " os.chdir(\"gallery\")\n", - " try:\n", - " import pylibkriging # noqa: F401\n", - " except ImportError:\n", - " r = subprocess.run([sys.executable, \"-m\", \"pip\", \"install\", \"--quiet\", \"pylibkriging\"],\n", - " capture_output=True, text=True)\n", - " if r.returncode != 0:\n", - " raise RuntimeError(\n", - " f\"pylibkriging could not be installed for Python {sys.version_info.major}.{sys.version_info.minor} \"\n", - " \"(PyPI has wheels for CPython 3.10 to 3.12 only). On Colab: Runtime > Change runtime version > \"\n", - " \"2026.07 (Python 3.12), then run again.\\n\" + r.stderr[-500:])\n", - " importlib.invalidate_caches()\n", - "sys.path.insert(0, str(Path(\"tools/python\").resolve())) # for \"import gallery\" (tools/python/gallery.py)\n", - "print(\"working directory:\", os.getcwd())" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c002", - "metadata": {}, - "outputs": [], - "source": [ - "import time\n", - "\n", - "import matplotlib.pyplot as plt\n", - "import numpy as np\n", - "import pandas as pd\n", - "import pylibkriging as lk\n", - "\n", - "import gallery\n", - "from gallery import TESTS, design, metrics, compat\n", - "\n", - "suite = gallery.Suite(\"high-dimension-normalize-screening_Kriging_python\")\n", - "print(\"pylibkriging\", compat.version())\n", - "\n", - "\n", - "def pred(k, X):\n", - " m, s, *_ = k.predict(X, return_stdev=True)\n", - " return np.ravel(m), np.ravel(s)" - ] - }, - { - "cell_type": "markdown", - "id": "c003", - "metadata": {}, - "source": [ - "## Accuracy, unit-cube inputs vs native inputs with `normalize=True`" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c004", - "metadata": {}, - "outputs": [], - "source": [ - "CASES = { # function: (n, Q2 floor on the unit cube)\n", - " \"otl_circuit\": (60, 0.999),\n", - " \"piston\": (70, 0.98),\n", - " \"borehole\": (80, 0.995),\n", - " \"robot_arm\": (160, 0.45),\n", - " \"dette_pepelyshev\": (80, 0.99),\n", - " \"wing_weight\": (100, 0.999),\n", - " \"welch\": (200, 0.98),\n", - "}\n", - "for fname, (n, floor) in CASES.items():\n", - " tf = TESTS[fname]\n", - " U, y, Ut, yt = design.dataset(tf, n, n_test=2000, seed=41)\n", - " t0 = time.time()\n", - " k = lk.Kriging(y, U, \"matern5_2\")\n", - " fit_s = time.time() - t0\n", - " res = metrics.all_metrics(yt, *pred(k, Ut))\n", - " suite.metric(fname, \"unit cube\", fit_s=fit_s, **res)\n", - " suite.check_greater(f\"{fname}: Q2 (unit cube, matern5_2)\", res[\"q2\"], floor)\n", - " # same problem in native units, rescaled internally\n", - " Xn, Xtn = tf.scale(U), tf.scale(Ut)\n", - " with suite.check_runs(f\"{fname}: fit on native inputs with normalize=True\"):\n", - " kn = lk.Kriging(y, Xn, \"matern5_2\", normalize=True)\n", - " res_n = metrics.all_metrics(yt, *pred(kn, Xtn))\n", - " suite.metric(fname, \"native + normalize\", **res_n)\n", - " suite.check_greater(f\"{fname}: Q2(normalize) - Q2(unit cube)\", res_n[\"q2\"] - res[\"q2\"], -0.02)\n", - "tab = suite.table()\n", - "tab.round(4)" - ] - }, - { - "cell_type": "markdown", - "id": "c005", - "metadata": {}, - "source": [ - "## Screening: Welch et al. (1992)\n", - "\n", - "Inputs 8 and 16 do not appear in the function; inputs 1, 4, 5, 12, 19, 20 are\n", - "the most influential ones. A small θ_j (short correlation length) means an\n", - "influential input, so the inactive inputs should get the largest θ." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c006", - "metadata": {}, - "outputs": [], - "source": [ - "tf = TESTS[\"welch\"]\n", - "U, y, _, _ = design.dataset(tf, 200, seed=42)\n", - "k = lk.Kriging(y, U, \"matern5_2\")\n", - "theta = np.ravel(k.theta())\n", - "inactive = [7, 15] # 0-based indices of x8, x16\n", - "active = [0, 3, 4, 11, 18, 19] # x1, x4, x5, x12, x19, x20\n", - "suite.check_greater(\"welch: min theta(inactive) / max theta(active)\", theta[inactive].min() / theta[active].max(), 1.0,\n", - " severity=\"warn\")\n", - "colors = [\"tab:red\" if i in active else \"tab:gray\" if i in inactive else \"tab:blue\" for i in range(20)]\n", - "plt.figure(figsize=(9, 3))\n", - "plt.bar([f\"x{i + 1}\" for i in range(20)], theta, color=colors)\n", - "plt.yscale(\"log\"); plt.ylabel(\"fitted θ (log)\")\n", - "plt.title(\"Welch (20D): red = most influential inputs, gray = inactive (x8, x16)\"); plt.show()\n", - "pd.Series(theta, index=[f\"x{i + 1}\" for i in range(20)]).round(3).to_frame(\"theta\").T" - ] - }, - { - "cell_type": "markdown", - "id": "c007", - "metadata": {}, - "source": [ - "## Summary\n", - "\n", - "The metrics of this notebook, the number of checks per status, and every check that\n", - "did not pass (FAIL: error-level, fails the run; WARN: warn-level; KNOWN: listed in\n", - "`tools/known_issues.csv`, with its issue; SKIP: feature absent from the installed version).\n", - "All of them are also written to `results/high-dimension-normalize-screening_Kriging_python.checks.csv` and `.metrics.csv`." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c008", - "metadata": {}, - "outputs": [], - "source": [ - "suite.summary()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c009", - "metadata": {}, - "outputs": [], - "source": [ - "suite.finalize()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/kernels-trends-interpolation_Kriging_python.ipynb b/kernels-trends-interpolation_Kriging_python.ipynb deleted file mode 100644 index 7242018..0000000 --- a/kernels-trends-interpolation_Kriging_python.ipynb +++ /dev/null @@ -1,363 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "c000", - "metadata": {}, - "source": [ - "# Python · Kriging — kernels × trends, interpolation, predict (covariance, derivatives)\n", - "\n", - "**Idea.** The basic promise of noise-free Kriging, over the whole grid of kernels\n", - "and trends: it interpolates the data, its predictive variance vanishes at the design\n", - "points, its predicted derivatives are the derivatives of its prediction, and it is\n", - "accurate on smooth 1D to 3D functions.\n", - "\n", - "**Reference.** Finite differences of the prediction, the data themselves, and per-\n", - "function Q² floors.\n", - "\n", - "Plain noise-free `Kriging`, every kernel (`gauss`, `exp`, `matern3_2`,\n", - "`matern5_2`) crossed with every trend (`constant`, `linear`, `interactive`,\n", - "`quadratic`) on functions of different regularity:\n", - "\n", - "| function | d | character |\n", - "|---|---|---|\n", - "| Forrester | 1 | smooth, one oscillation |\n", - "| Gramacy & Lee (2012) | 1 | fast oscillation then polynomial: non-stationary |\n", - "| Branin | 2 | smooth, three global minima |\n", - "| Six-hump camel | 2 | polynomial, steep borders |\n", - "| log-Goldstein-Price | 2 | strongly varying |\n", - "| Lim polynomial | 2 | low-degree polynomial: the trend should help |\n", - "| Ishigami | 3 | strongly non-linear, interactions |\n", - "| Hartmann 3 | 3 | multimodal, localized |\n", - "\n", - "Checks:\n", - "* **interpolation**: a noise-free kriging mean reproduces the observations, with\n", - " a (near) zero predictive standard deviation at the design points. Error-level\n", - " for every kernel but `gauss`: its correlation matrices are notoriously\n", - " ill-conditioned, and the numerical jitter libKriging adds to factorize them\n", - " breaks exact interpolation (warn-level, with the size of the gap);\n", - "* **accuracy**: Q² on a test set above a per-function floor for\n", - " `matern5_2` / `constant` with a multistart optimizer (`BFGS10`): a\n", - " regression shows up here first. The same floor for the default single-start\n", - " `BFGS` is warn-level: a single start can end in a poor local optimum of the\n", - " likelihood (see `objectives-LL-LOO-LMP-gradients_Kriging_python.ipynb`),\n", - " which is worth seeing in the report but is not a defect of the model;\n", - "* **sanity of the fitted hyperparameters**: finite, positive θ and σ²;\n", - "* **consistency**: `predict` on a batch equals `predict` point by point, the\n", - " returned covariance has the predicted variances on its diagonal, and is\n", - " symmetric positive semi-definite." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c001", - "metadata": {}, - "outputs": [], - "source": [ - "# Setup for running this notebook outside the repository (Colab, Binder, a lone copy):\n", - "# nothing to do in the repository itself (CI, local runs). Otherwise the repository\n", - "# (datasets of data/, helpers of tools/python/gallery.py) is cloned and the released pylibkriging installed.\n", - "import importlib\n", - "import os\n", - "import subprocess\n", - "import sys\n", - "from pathlib import Path\n", - "\n", - "if not Path(\"tools/known_issues.csv\").is_file():\n", - " if not Path(\"gallery/tools/known_issues.csv\").is_file():\n", - " subprocess.run([\"git\", \"clone\", \"--depth\", \"1\", \"https://github.com/libKriging/gallery\"], check=True)\n", - " os.chdir(\"gallery\")\n", - " try:\n", - " import pylibkriging # noqa: F401\n", - " except ImportError:\n", - " r = subprocess.run([sys.executable, \"-m\", \"pip\", \"install\", \"--quiet\", \"pylibkriging\"],\n", - " capture_output=True, text=True)\n", - " if r.returncode != 0:\n", - " raise RuntimeError(\n", - " f\"pylibkriging could not be installed for Python {sys.version_info.major}.{sys.version_info.minor} \"\n", - " \"(PyPI has wheels for CPython 3.10 to 3.12 only). On Colab: Runtime > Change runtime version > \"\n", - " \"2026.07 (Python 3.12), then run again.\\n\" + r.stderr[-500:])\n", - " importlib.invalidate_caches()\n", - "sys.path.insert(0, str(Path(\"tools/python\").resolve())) # for \"import gallery\" (tools/python/gallery.py)\n", - "print(\"working directory:\", os.getcwd())" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c002", - "metadata": {}, - "outputs": [], - "source": [ - "import time\n", - "\n", - "import numpy as np\n", - "import pandas as pd\n", - "import matplotlib.pyplot as plt\n", - "import pylibkriging as lk\n", - "\n", - "import gallery\n", - "from gallery import TESTS, design, metrics, compat\n", - "\n", - "suite = gallery.Suite(\"kernels-trends-interpolation_Kriging_python\")\n", - "print(\"pylibkriging\", compat.version())\n", - "\n", - "KERNELS = [\"gauss\", \"exp\", \"matern3_2\", \"matern5_2\"]\n", - "TRENDS = [\"constant\", \"linear\", \"interactive\", \"quadratic\"]\n", - "CASES = { # function: (n, Q2 floor for matern5_2/constant)\n", - " \"forrester\": (12, 0.95),\n", - " \"gramacy_lee_2012\": (20, 0.60),\n", - " \"branin\": (30, 0.97),\n", - " \"six_hump_camel\": (30, 0.95),\n", - " \"log_goldstein_price\": (40, 0.85),\n", - " \"lim_polynomial\": (15, 0.99),\n", - " \"ishigami\": (80, 0.75),\n", - " \"hartmann3\": (40, 0.85),\n", - "}\n", - "\n", - "\n", - "def predict(model, X, **kw):\n", - " m, s, cov, *_ = model.predict(X, return_stdev=True, **kw)\n", - " return np.asarray(m).ravel(), np.asarray(s).ravel(), cov" - ] - }, - { - "cell_type": "markdown", - "id": "c003", - "metadata": {}, - "source": [ - "## Kernel × trend grid" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c004", - "metadata": {}, - "outputs": [], - "source": [ - "for fname, (n, q2_floor) in CASES.items():\n", - " tf = TESTS[fname]\n", - " X, y, Xt, yt = design.dataset(tf, n, n_test=1000, seed=1)\n", - " for kernel in KERNELS:\n", - " for trend in TRENDS:\n", - " n_beta = {\"constant\": 1, \"linear\": 1 + tf.d, \"interactive\": 1 + tf.d + tf.d * (tf.d - 1) // 2,\n", - " \"quadratic\": 1 + 2 * tf.d + tf.d * (tf.d - 1) // 2}[trend]\n", - " case = f\"{fname}/{kernel}/{trend}\"\n", - " if n_beta >= n - 1:\n", - " suite.check_skip(case, f\"{n_beta} trend coefficients for n={n}\")\n", - " continue\n", - " t0 = time.time()\n", - " try:\n", - " k = lk.Kriging(y, X, kernel, regmodel=trend, optim=\"BFGS\")\n", - " except Exception as exc: # noqa: BLE001\n", - " suite.check_true(f\"{case}: fit\", False, detail=str(exc)[:200])\n", - " continue\n", - " fit_s = time.time() - t0\n", - " m, s, _ = predict(k, Xt)\n", - " res = metrics.all_metrics(yt, m, s)\n", - " suite.metric(case, \"pylibkriging\", fit_s=fit_s, **res)\n", - " theta, sigma2 = np.asarray(k.theta()).ravel(), float(k.sigma2())\n", - " suite.check_true(f\"{case}: theta, sigma2 finite and > 0\",\n", - " np.all(np.isfinite(theta)) and np.all(theta > 0) and np.isfinite(sigma2) and sigma2 > 0,\n", - " detail=f\"theta={theta}, sigma2={sigma2:.3g}\")\n", - " # interpolation at the design points (relative to the spread of y)\n", - " md, sd, _ = predict(k, X)\n", - " scale = np.std(y)\n", - " sev = \"warn\" if kernel == \"gauss\" else \"error\"\n", - " suite.check_less(f\"{case}: interpolates y (max |m(X)-y| / sd(y))\", np.max(np.abs(md - y)) / scale, 1e-4,\n", - " severity=sev)\n", - " suite.check_less(f\"{case}: stdev(X) ~ 0 (max sd / sd(y))\", np.max(sd) / scale, 1e-2, severity=sev)\n", - " if kernel == \"matern5_2\" and trend == \"constant\":\n", - " suite.check_greater(f\"{case}: Q2 (single-start BFGS)\", res[\"q2\"], q2_floor, severity=\"warn\")\n", - " k10 = lk.Kriging(y, X, kernel, regmodel=trend, optim=\"BFGS10\")\n", - " res10 = metrics.all_metrics(yt, *predict(k10, Xt)[:2])\n", - " suite.metric(case, \"pylibkriging BFGS10\", **res10)\n", - " suite.check_greater(f\"{case}: Q2 (BFGS10)\", res10[\"q2\"], q2_floor)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c005", - "metadata": {}, - "outputs": [], - "source": [ - "tab = suite.table().reset_index()\n", - "tab = tab[tab[\"package\"] == \"pylibkriging\"]\n", - "tab[[\"function\", \"kernel\", \"trend\"]] = tab[\"case\"].str.split(\"/\", expand=True)\n", - "pivot = tab.pivot_table(index=[\"function\", \"trend\"], columns=\"kernel\", values=\"q2\", sort=False)\n", - "pivot.style.format(\"{:.3f}\").background_gradient(axis=None, vmin=0.5, vmax=1, cmap=\"RdYlGn\")" - ] - }, - { - "cell_type": "markdown", - "id": "c006", - "metadata": {}, - "source": [ - "Expectations that should hold whatever the version (warn-level: they are\n", - "empirical, not mathematical, statements):\n", - "* on the smooth Branin the `exp` kernel (non-differentiable paths) is the least accurate;\n", - "* on the Lim polynomial a `quadratic` trend is at least as good as a `constant` one." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c007", - "metadata": {}, - "outputs": [], - "source": [ - "q = tab.set_index([\"function\", \"kernel\", \"trend\"])[\"q2\"]\n", - "suite.check_true(\"branin: exp is the worst kernel (constant trend)\",\n", - " q[\"branin\", \"exp\", \"constant\"] <= min(q[\"branin\", k, \"constant\"] for k in KERNELS),\n", - " severity=\"warn\")\n", - "suite.check_greater(\"lim_polynomial/matern5_2: Q2(quadratic) - Q2(constant)\",\n", - " q[\"lim_polynomial\", \"matern5_2\", \"quadratic\"] - q[\"lim_polynomial\", \"matern5_2\", \"constant\"], -1e-3,\n", - " severity=\"warn\")" - ] - }, - { - "cell_type": "markdown", - "id": "c008", - "metadata": {}, - "source": [ - "## Consistency of `predict`" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c009", - "metadata": {}, - "outputs": [], - "source": [ - "tf = TESTS[\"branin\"]\n", - "X, y, Xt, yt = design.dataset(tf, 30, n_test=50, seed=3)\n", - "k = lk.Kriging(y, X, \"matern5_2\")\n", - "m, s, cov = predict(k, Xt, return_cov=True)\n", - "cov = np.asarray(cov)\n", - "suite.check_close(\"predict: diag(cov) == stdev^2\", np.diag(cov), s ** 2, rtol=1e-6, atol=1e-10)\n", - "suite.check_close(\"predict: cov symmetric\", cov, cov.T, rtol=1e-10, atol=1e-12)\n", - "suite.check_greater(\"predict: cov PSD (min eigenvalue / max)\", np.linalg.eigvalsh(cov).min() / np.abs(cov).max(), -1e-8)\n", - "m1 = np.array([predict(k, Xt[i:i + 1])[0][0] for i in range(len(Xt))])\n", - "s1 = np.array([predict(k, Xt[i:i + 1])[1][0] for i in range(len(Xt))])\n", - "suite.check_close(\"predict: batch == point by point (mean)\", m, m1, rtol=1e-8, atol=1e-10 * np.std(y))\n", - "suite.check_close(\"predict: batch == point by point (stdev)\", s, s1, rtol=1e-6, atol=1e-10 * np.std(y))\n", - "# the mean is a function of the point only: duplicating/permuting the batch changes nothing\n", - "perm = np.random.default_rng(0).permutation(len(Xt))\n", - "suite.check_close(\"predict: invariant to the order of the batch\", predict(k, Xt[perm])[0], m[perm], rtol=1e-10, atol=1e-10)" - ] - }, - { - "cell_type": "markdown", - "id": "c010", - "metadata": {}, - "source": [ - "### Derivatives of the predictor\n", - "\n", - "`return_deriv=True` gives the gradients of the mean and of the stdev w.r.t. x;\n", - "compared with central finite differences." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c011", - "metadata": {}, - "outputs": [], - "source": [ - "with suite.check_runs(\"predict(return_deriv=True)\"):\n", - " out = k.predict(Xt[:5], True, False, True)\n", - " dm, ds = np.asarray(out[3]), np.asarray(out[4])\n", - " h = 1e-6\n", - " fd_m, fd_s = np.zeros_like(dm), np.zeros_like(ds)\n", - " for j in range(2):\n", - " e = np.zeros(2); e[j] = h\n", - " mp, sp, _ = predict(k, Xt[:5] + e)\n", - " mm, sm, _ = predict(k, Xt[:5] - e)\n", - " fd_m[:, j], fd_s[:, j] = (mp - mm) / (2 * h), (sp - sm) / (2 * h)\n", - " suite.check_close(\"d mean / dx == finite differences\", dm, fd_m, rtol=1e-4, atol=1e-4 * np.abs(fd_m).max())\n", - " suite.check_close(\"d stdev / dx == finite differences\", ds, fd_s, rtol=1e-3, atol=1e-3 * np.abs(fd_s).max())" - ] - }, - { - "cell_type": "markdown", - "id": "c012", - "metadata": {}, - "source": [ - "## A look at the 1D fits" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c013", - "metadata": {}, - "outputs": [], - "source": [ - "fig, axes = plt.subplots(1, 2, figsize=(13, 4))\n", - "for ax, fname in zip(axes, [\"forrester\", \"gramacy_lee_2012\"]):\n", - " tf = TESTS[fname]\n", - " n = CASES[fname][0]\n", - " X, y, _, _ = design.dataset(tf, n, seed=1)\n", - " u = np.linspace(0, 1, 400)[:, None]\n", - " ax.plot(u, tf.unit(u), \"k--\", label=\"truth\")\n", - " for kernel in [\"gauss\", \"matern5_2\", \"exp\"]:\n", - " mm, ss, _ = predict(lk.Kriging(y, X, kernel), u)\n", - " ax.plot(u, mm, label=kernel)\n", - " if kernel == \"matern5_2\":\n", - " ax.fill_between(u[:, 0], mm - 2 * ss, mm + 2 * ss, alpha=0.2)\n", - " ax.plot(X, y, \"ko\")\n", - " ax.set_title(fname)\n", - " ax.legend()" - ] - }, - { - "cell_type": "markdown", - "id": "c014", - "metadata": {}, - "source": [ - "## Summary\n", - "\n", - "The metrics of this notebook, the number of checks per status, and every check that\n", - "did not pass (FAIL: error-level, fails the run; WARN: warn-level; KNOWN: listed in\n", - "`tools/known_issues.csv`, with its issue; SKIP: feature absent from the installed version).\n", - "All of them are also written to `results/kernels-trends-interpolation_Kriging_python.checks.csv` and `.metrics.csv`." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c015", - "metadata": {}, - "outputs": [], - "source": [ - "suite.summary()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c016", - "metadata": {}, - "outputs": [], - "source": [ - "suite.finalize()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/large-n_Kriging-vecchia-nystrom_python.ipynb b/large-n_Kriging-vecchia-nystrom_python.ipynb deleted file mode 100644 index d83bfe0..0000000 --- a/large-n_Kriging-vecchia-nystrom_python.ipynb +++ /dev/null @@ -1,302 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "c000", - "metadata": {}, - "source": [ - "# Python · Kriging with `method=\"vecchia\"` / `\"nystrom\"` — large n\n", - "\n", - "**Idea.** Vecchia and Nyström approximations make Kriging affordable for large n;\n", - "how much accuracy do they cost, and do they converge to the exact model as their\n", - "size parameter grows?\n", - "\n", - "**Reference.** The exact model at the same θ (limits m = n − 1 and k = n), and the\n", - "exact fit.\n", - "\n", - "The exact fit costs O(n³). `Kriging` offers:\n", - "\n", - "* `method=\"vecchia(m)\"`: Vecchia likelihood with m conditioning neighbours\n", - " (local, recommended in low dimension);\n", - "* `method=\"nystrom(k)\"`: low-rank Nyström approximation with k landmarks\n", - " (global);\n", - "* `Kriging.subsetOfData`: a representative subset before an exact fit.\n", - "\n", - "(`NestedKriging`, the partition-based alternative, on the same problems:\n", - "`aggregations-large-n_NestedKriging_python.ipynb`.)\n", - "\n", - "Problems: Hartmann 3 (3D, local structure: Vecchia's ground) and the OTL\n", - "circuit (6D, smooth: Nyström's ground), n = 1500 (exact fit still affordable,\n", - "for reference) and n = 5000 (approximations only).\n", - "\n", - "Checks:\n", - "* **convergence to the exact model** at fixed θ (Hartmann 3, n = 300): the gap\n", - " between the approximate and the exact log-likelihood and predicted mean,\n", - " as m (Vecchia) and k (Nyström) grow, down to the limit cases m = n − 1 and\n", - " k = n where the approximation *is* the exact model;\n", - "* **limit cases**: Vecchia with m = n − 1 neighbours *is* the exact likelihood;\n", - "* the method is explicit at prediction time: `predict(method=\"exact\")` on a\n", - " Nyström/Vecchia fit raises, `method=` gives finite predictions;\n", - "* accuracy of each approximation close to the exact fit (warn-level gaps,\n", - " error-level floors) and faster than the exact fit at n = 1500 (warn-level)." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c001", - "metadata": {}, - "outputs": [], - "source": [ - "# Setup for running this notebook outside the repository (Colab, Binder, a lone copy):\n", - "# nothing to do in the repository itself (CI, local runs). Otherwise the repository\n", - "# (datasets of data/, helpers of tools/python/gallery.py) is cloned and the released pylibkriging installed.\n", - "import importlib\n", - "import os\n", - "import subprocess\n", - "import sys\n", - "from pathlib import Path\n", - "\n", - "if not Path(\"tools/known_issues.csv\").is_file():\n", - " if not Path(\"gallery/tools/known_issues.csv\").is_file():\n", - " subprocess.run([\"git\", \"clone\", \"--depth\", \"1\", \"https://github.com/libKriging/gallery\"], check=True)\n", - " os.chdir(\"gallery\")\n", - " try:\n", - " import pylibkriging # noqa: F401\n", - " except ImportError:\n", - " r = subprocess.run([sys.executable, \"-m\", \"pip\", \"install\", \"--quiet\", \"pylibkriging\"],\n", - " capture_output=True, text=True)\n", - " if r.returncode != 0:\n", - " raise RuntimeError(\n", - " f\"pylibkriging could not be installed for Python {sys.version_info.major}.{sys.version_info.minor} \"\n", - " \"(PyPI has wheels for CPython 3.10 to 3.12 only). On Colab: Runtime > Change runtime version > \"\n", - " \"2026.07 (Python 3.12), then run again.\\n\" + r.stderr[-500:])\n", - " importlib.invalidate_caches()\n", - "sys.path.insert(0, str(Path(\"tools/python\").resolve())) # for \"import gallery\" (tools/python/gallery.py)\n", - "print(\"working directory:\", os.getcwd())" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c002", - "metadata": {}, - "outputs": [], - "source": [ - "import time\n", - "\n", - "import matplotlib.pyplot as plt\n", - "import numpy as np\n", - "import pandas as pd\n", - "import pylibkriging as lk\n", - "\n", - "import gallery\n", - "from gallery import TESTS, design, metrics, compat\n", - "\n", - "suite = gallery.Suite(\"large-n_Kriging-vecchia-nystrom_python\")\n", - "print(\"pylibkriging\", compat.version())\n", - "HAS_METHOD = compat.has(\"method\")\n", - "if not HAS_METHOD:\n", - " suite.check_skip(\"method=vecchia/nystrom\", \"pylibkriging without the method= argument\")\n", - "\n", - "\n", - "def pred(k, X, **kw):\n", - " m, s, *_ = k.predict(X, return_stdev=True, **kw)\n", - " return np.ravel(m), np.ravel(s)\n", - "\n", - "\n", - "def timed(f):\n", - " t0 = time.time()\n", - " out = f()\n", - " return out, time.time() - t0" - ] - }, - { - "cell_type": "markdown", - "id": "c003", - "metadata": {}, - "source": [ - "## Limit case: Vecchia with all the previous points as neighbours is exact" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c004", - "metadata": {}, - "outputs": [], - "source": [ - "if HAS_METHOD:\n", - " tf = TESTS[\"branin\"]\n", - " X, y, Xt, _ = design.dataset(tf, 40, n_test=100, seed=51)\n", - " ke = lk.Kriging(y, X, \"matern5_2\")\n", - " fixed = {\"theta\": np.asarray(ke.theta()).reshape(1, -1), \"is_theta_estim\": False}\n", - " kv = lk.Kriging(y, X, \"matern5_2\", optim=\"none\", parameters=fixed, method=f\"vecchia({len(y) - 1})\")\n", - " suite.check_close(\"vecchia(n-1) LL == exact LL (same theta)\", kv.logLikelihood(method=\"vecchia\"),\n", - " ke.logLikelihood(), rtol=1e-6)\n", - " suite.check_close(\"vecchia(n-1) predict == exact predict\", pred(kv, Xt, method=\"vecchia\")[0], pred(ke, Xt)[0],\n", - " rtol=1e-5, atol=1e-6 * np.std(y), severity=\"warn\")\n", - " kn = lk.Kriging(y, X, \"matern5_2\", optim=\"none\", parameters=fixed, method=f\"nystrom({len(y)})\")\n", - " suite.check_close(\"nystrom(n) LL == exact LL (same theta)\", kn.logLikelihood(method=\"nystrom\"), ke.logLikelihood(),\n", - " rtol=1e-3, severity=\"warn\")\n", - " # the method must be explicit after an approximate fit\n", - " suite.check_raises(\"nystrom fit: predict(method='exact') raises\", lambda: kn.predict(Xt, return_stdev=True,\n", - " method=\"exact\"))\n", - " suite.check_true(\"method() reports the fit method\", kv.method().startswith(\"vecchia\") and\n", - " kn.method().startswith(\"nystrom\"), detail=f\"{kv.method()}, {kn.method()}\")" - ] - }, - { - "cell_type": "markdown", - "id": "c005", - "metadata": {}, - "source": [ - "## Convergence to the exact model, at fixed θ\n", - "\n", - "Hartmann 3, n = 300, θ of the exact fit kept fixed: the reference is the exact\n", - "model with the same hyperparameters, so the gap measures the approximation\n", - "alone (no optimizer effect). Relative gap of the log-likelihood, and RMS gap\n", - "of the predicted mean on 500 test points relative to its standard deviation." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c006", - "metadata": {}, - "outputs": [], - "source": [ - "if HAS_METHOD:\n", - " tf = TESTS[\"hartmann3\"]\n", - " X, y, Xt, yt = design.dataset(tf, 300, n_test=500, seed=54)\n", - " n = len(y)\n", - " theta = np.asarray(lk.Kriging(y, X, \"matern5_2\").theta()).reshape(1, -1)\n", - " fixed = {\"theta\": theta, \"is_theta_estim\": False}\n", - " ke = lk.Kriging(y, X, \"matern5_2\", optim=\"none\", parameters=fixed)\n", - " ll_e, m_e = ke.logLikelihood(), pred(ke, Xt)[0]\n", - " rows = []\n", - " for meth, sizes in [(\"vecchia\", [2, 5, 10, 20, 40, 80, 160, n - 1]), (\"nystrom\", [10, 25, 50, 100, 200, n])]:\n", - " for size in sizes:\n", - " with suite.check_runs(f\"{meth}({size}) at fixed theta: fit + predict\"):\n", - " ka = lk.Kriging(y, X, \"matern5_2\", optim=\"none\", parameters=fixed, method=f\"{meth}({size})\")\n", - " rows.append(dict(method=meth, size=size,\n", - " ll_gap=abs(ka.logLikelihood(method=meth) - ll_e) / abs(ll_e),\n", - " mean_gap=np.sqrt(np.mean((pred(ka, Xt, method=meth)[0] - m_e) ** 2)) / np.std(m_e)))\n", - " conv = pd.DataFrame(rows)\n", - " for meth, g in conv.groupby(\"method\"):\n", - " suite.metric(f\"hartmann3/n={n}/fixed theta\", meth, **{f\"ll_gap_{s}\": v for s, v in zip(g[\"size\"], g[\"ll_gap\"])})\n", - " for col, lim in [(\"ll_gap\", 1e-6), (\"mean_gap\", 1e-2)]:\n", - " suite.check_less(f\"{meth}: {col} at the limit size ({g['size'].iloc[-1]})\", g[col].iloc[-1], lim)\n", - " suite.check_true(f\"{meth}: {col} decreases with the size\", bool(np.all(np.diff(g[col].to_numpy()) < 0)),\n", - " detail=\", \".join(f\"{v:.2g}\" for v in g[col]), severity=\"warn\")\n", - " fig, axes = plt.subplots(1, 2, figsize=(10, 3.8))\n", - " for ax, col, label in [(axes[0], \"ll_gap\", \"|LL − LL(exact)| / |LL(exact)|\"),\n", - " (axes[1], \"mean_gap\", \"RMS(mean − mean(exact)) / sd\")]:\n", - " for meth, g in conv.groupby(\"method\"):\n", - " ax.loglog(g[\"size\"], g[col], \"o-\", label=f\"{meth}(m)\" if meth == \"vecchia\" else f\"{meth}(k)\")\n", - " ax.set_xlabel(\"m (neighbours) or k (landmarks)\"); ax.set_ylabel(label); ax.grid(True, which=\"both\", alpha=.3)\n", - " axes[0].legend(); fig.suptitle(f\"Hartmann 3, n = {n}, θ fixed: approximation → exact model\"); plt.show()\n", - " display(conv.pivot(index=\"size\", columns=\"method\", values=[\"ll_gap\", \"mean_gap\"]).round(6))" - ] - }, - { - "cell_type": "markdown", - "id": "c007", - "metadata": {}, - "source": [ - "## Accuracy and cost" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c008", - "metadata": {}, - "outputs": [], - "source": [ - "CASES = {\"hartmann3\": (1500, 5000), \"otl_circuit\": (1500, 5000)}\n", - "FLOORS = {\"hartmann3\": 0.98, \"otl_circuit\": 0.99}\n", - "for fname, (n_mid, n_big) in CASES.items():\n", - " tf = TESTS[fname]\n", - " X, y, Xt, yt = design.dataset(tf, n_big, n_test=2000, seed=52)\n", - " Xm, ym = X[:n_mid], y[:n_mid]\n", - " ke, t_exact = timed(lambda: lk.Kriging(ym, Xm, \"matern5_2\"))\n", - " res_e = metrics.all_metrics(yt, *pred(ke, Xt))\n", - " suite.metric(f\"{fname}/n={n_mid}\", \"exact\", fit_s=t_exact, **res_e)\n", - " suite.check_greater(f\"{fname}/n={n_mid}: Q2 exact\", res_e[\"q2\"], FLOORS[fname])\n", - " variants = {}\n", - " if HAS_METHOD:\n", - " variants.update({\n", - " \"vecchia(30)\": lambda Xa, ya: (lk.Kriging(ya, Xa, \"matern5_2\", method=\"vecchia(30)\"), \"vecchia\"),\n", - " \"nystrom(100)\": lambda Xa, ya: (lk.Kriging(ya, Xa, \"matern5_2\", method=\"nystrom(100)\"), \"nystrom\"),\n", - " })\n", - " if compat.has(\"subsetOfData\"):\n", - " def sod(Xa, ya):\n", - " idx = np.ravel(lk.Kriging.subsetOfData(Xa, n_max=500, method=\"kmeans\", seed=1)).astype(int)\n", - " return lk.Kriging(ya[idx], Xa[idx], \"matern5_2\"), None\n", - " variants[\"subset of data (500)\"] = sod\n", - " for label, make in variants.items():\n", - " for n in (n_mid, n_big):\n", - " case = f\"{fname}/n={n}\"\n", - " with suite.check_runs(f\"{case}/{label}: fit + predict\"):\n", - " (k, meth), t = timed(lambda: make(X[:n], y[:n]))\n", - " m, s = pred(k, Xt, method=meth) if meth else (\n", - " lambda o: (np.ravel(o[0]), np.ravel(o[1])))(k.predict(Xt, return_stdev=True))\n", - " suite.check_finite(f\"{case}/{label}: finite predictions\", m)\n", - " res = metrics.all_metrics(yt, m, s)\n", - " suite.metric(case, label, fit_s=t, **res)\n", - " suite.check_greater(f\"{case}/{label}: Q2\", res[\"q2\"], FLOORS[fname] - 0.05)\n", - " if n == n_mid:\n", - " suite.check_greater(f\"{case}/{label}: Q2 - Q2(exact)\", res[\"q2\"] - res_e[\"q2\"], -0.02,\n", - " severity=\"warn\")\n", - " suite.check_less(f\"{case}/{label}: fit time / exact fit time\", t / t_exact, 1.0, severity=\"warn\")\n", - "suite.table().round(4)" - ] - }, - { - "cell_type": "markdown", - "id": "c009", - "metadata": {}, - "source": [ - "## Summary\n", - "\n", - "The metrics of this notebook, the number of checks per status, and every check that\n", - "did not pass (FAIL: error-level, fails the run; WARN: warn-level; KNOWN: listed in\n", - "`tools/known_issues.csv`, with its issue; SKIP: feature absent from the installed version).\n", - "All of them are also written to `results/large-n_Kriging-vecchia-nystrom_python.checks.csv` and `.metrics.csv`." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c010", - "metadata": {}, - "outputs": [], - "source": [ - "suite.summary()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c011", - "metadata": {}, - "outputs": [], - "source": [ - "suite.finalize()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/noisy-data_Kriging-nugget-noise_python.ipynb b/noisy-data_Kriging-nugget-noise_python.ipynb deleted file mode 100644 index 68ad63d..0000000 --- a/noisy-data_Kriging-nugget-noise_python.ipynb +++ /dev/null @@ -1,305 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "c000", - "metadata": {}, - "source": [ - "# Python · Kriging with noise (`noise=\"nugget\"`, known variances) — noisy observations\n", - "\n", - "**Idea.** Kriging with noisy observations: an estimated homogeneous nugget, or known\n", - "(heteroscedastic) noise variances, must recover the noise level and predict the\n", - "noise-free function with calibrated intervals.\n", - "\n", - "**Reference.** The true noise variance, replicate averaging, and the interpolating\n", - "(noise-free) fit on the same data.\n", - "\n", - "`Kriging` handles noisy responses through its `noise` argument: `\"nugget\"`\n", - "estimates one homogeneous noise variance, a vector gives a known variance per\n", - "observation (heteroscedastic case).\n", - "\n", - "| case | function | noise |\n", - "|---|---|---|\n", - "| 1D homoscedastic | Forrester | N(0, 0.5²) |\n", - "| 5D homoscedastic | Friedman | N(0, 1) (the classical \"Friedman 1\" setup) |\n", - "| 1D heteroscedastic | Gramacy & Lee (2012) | sd growing linearly with x |\n", - "| 2D with replicates | Branin | 4 replicates of 15 points, N(0, 2²) |\n", - "\n", - "Checks:\n", - "* the estimated nugget is close to the true noise variance (factor 3);\n", - "* the noisy fit predicts the noise-free function well (Q² floor), and is\n", - " **smooth**: it does not chase the noisy data between the observations. (At the\n", - " design points themselves the `\"nugget\"` predictor may return y exactly, as\n", - " DiceKriging's nugget does: the nugget is then a micro-scale variation, and\n", - " the predictor is discontinuous there. This is reported, not checked.);\n", - "* prediction intervals are calibrated: 95% coverage of the *noise-free* truth;\n", - "* a zero noise vector gives the noise-free model;\n", - "* with heteroscedastic noise, giving the true variances beats a single nugget\n", - " (NLPD, warn-level);\n", - "* replicates: averaging the replicates with variance σ²/r is equivalent to\n", - " fitting all the replicates (same predictor, up to the optimizer)." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c001", - "metadata": {}, - "outputs": [], - "source": [ - "# Setup for running this notebook outside the repository (Colab, Binder, a lone copy):\n", - "# nothing to do in the repository itself (CI, local runs). Otherwise the repository\n", - "# (datasets of data/, helpers of tools/python/gallery.py) is cloned and the released pylibkriging installed.\n", - "import importlib\n", - "import os\n", - "import subprocess\n", - "import sys\n", - "from pathlib import Path\n", - "\n", - "if not Path(\"tools/known_issues.csv\").is_file():\n", - " if not Path(\"gallery/tools/known_issues.csv\").is_file():\n", - " subprocess.run([\"git\", \"clone\", \"--depth\", \"1\", \"https://github.com/libKriging/gallery\"], check=True)\n", - " os.chdir(\"gallery\")\n", - " try:\n", - " import pylibkriging # noqa: F401\n", - " except ImportError:\n", - " r = subprocess.run([sys.executable, \"-m\", \"pip\", \"install\", \"--quiet\", \"pylibkriging\"],\n", - " capture_output=True, text=True)\n", - " if r.returncode != 0:\n", - " raise RuntimeError(\n", - " f\"pylibkriging could not be installed for Python {sys.version_info.major}.{sys.version_info.minor} \"\n", - " \"(PyPI has wheels for CPython 3.10 to 3.12 only). On Colab: Runtime > Change runtime version > \"\n", - " \"2026.07 (Python 3.12), then run again.\\n\" + r.stderr[-500:])\n", - " importlib.invalidate_caches()\n", - "sys.path.insert(0, str(Path(\"tools/python\").resolve())) # for \"import gallery\" (tools/python/gallery.py)\n", - "print(\"working directory:\", os.getcwd())" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c002", - "metadata": {}, - "outputs": [], - "source": [ - "import numpy as np\n", - "import pandas as pd\n", - "import matplotlib.pyplot as plt\n", - "import pylibkriging as lk\n", - "\n", - "import gallery\n", - "from gallery import TESTS, design, metrics, compat\n", - "\n", - "suite = gallery.Suite(\"noisy-data_Kriging-nugget-noise_python\")\n", - "print(\"pylibkriging\", compat.version())\n", - "\n", - "\n", - "def pred(k, X):\n", - " m, s, *_ = k.predict(X, return_stdev=True)\n", - " return np.ravel(m), np.ravel(s)" - ] - }, - { - "cell_type": "markdown", - "id": "c003", - "metadata": {}, - "source": [ - "## Homoscedastic noise, estimated nugget" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c004", - "metadata": {}, - "outputs": [], - "source": [ - "for fname, n, sd, q2_floor in [(\"forrester\", 40, 0.5, 0.95), (\"friedman\", 200, 1.0, 0.90)]:\n", - " tf = TESTS[fname]\n", - " X, y, Xt, yt = design.dataset(tf, n, n_test=2000, seed=11, noise_sd=sd)\n", - " k = lk.Kriging(y, X, \"matern5_2\", noise=\"nugget\", optim=\"BFGS10\")\n", - " nugget = float(np.ravel(k.nugget())[0])\n", - " m, s = pred(k, Xt)\n", - " res = metrics.all_metrics(yt, m, s)\n", - " suite.metric(fname, \"nugget\", nugget=nugget, true_var=sd ** 2, **res)\n", - " suite.check_between(f\"{fname}: nugget / true noise variance\", nugget / sd ** 2, 1 / 3, 3)\n", - " k1 = lk.Kriging(y, X, \"matern5_2\", noise=\"nugget\") # default single start\n", - " suite.metric(fname, \"nugget, single start\", nugget=float(np.ravel(k1.nugget())[0]), ll=float(k1.logLikelihood()),\n", - " **metrics.all_metrics(yt, *pred(k1, Xt)))\n", - " suite.check_greater(f\"{fname}: LL(single start) - LL(10 restarts)\", k1.logLikelihood() - k.logLikelihood(), -1,\n", - " severity=\"warn\")\n", - " suite.check_greater(f\"{fname}: Q2 vs noise-free truth\", res[\"q2\"], q2_floor)\n", - " mX, _ = pred(k, X)\n", - " mX_near, _ = pred(k, np.clip(X + 1e-7, 0, 1))\n", - " suite.metric(fname, \"nugget\", rms_resid_at_X=metrics.rmse(y, mX) / sd,\n", - " rms_resid_near_X=metrics.rmse(y, mX_near) / sd)\n", - " suite.check_greater(f\"{fname}: smooths the noisy data next to the design points (rms residual / sd)\",\n", - " metrics.rmse(y, mX_near) / sd, 0.3)\n", - " suite.check_between(f\"{fname}: coverage of the 95% intervals (noise-free truth)\", res[\"coverage95\"], 0.80, 1.0,\n", - " severity=\"warn\")\n", - " # the noise-free model on the same noisy data interpolates, and generalizes worse\n", - " k0 = lk.Kriging(y, X, \"matern5_2\")\n", - " m0, s0 = pred(k0, Xt)\n", - " suite.metric(fname, \"noise-free fit\", **metrics.all_metrics(yt, m0, s0))\n", - " suite.check_less(f\"{fname}: RMSE(nugget) < RMSE(interpolating fit)\", res[\"rmse\"], metrics.rmse(yt, m0),\n", - " severity=\"warn\")\n", - "suite.table()" - ] - }, - { - "cell_type": "markdown", - "id": "c005", - "metadata": {}, - "source": [ - "## A zero noise vector is the noise-free model" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c006", - "metadata": {}, - "outputs": [], - "source": [ - "tf = TESTS[\"branin\"]\n", - "X, y, Xt, yt = design.dataset(tf, 25, n_test=200, seed=12)\n", - "k0 = lk.Kriging(y, X, \"matern5_2\")\n", - "theta = np.asarray(k0.theta()).reshape(1, -1)\n", - "fixed = {\"theta\": theta, \"is_theta_estim\": False, \"sigma2\": float(k0.sigma2()), \"is_sigma2_estim\": False}\n", - "kf = lk.Kriging(y, X, \"matern5_2\", optim=\"none\", parameters=fixed)\n", - "with suite.check_runs(\"noise=zeros, theta and sigma2 fixed: fit\"):\n", - " kz = lk.Kriging(y, X, \"matern5_2\", noise=np.zeros(len(y)), optim=\"none\", parameters=fixed)\n", - " suite.check_close(\"noise=zeros: same mean as the noise-free model\", pred(kz, Xt)[0], pred(kf, Xt)[0], rtol=1e-6,\n", - " atol=1e-8 * np.std(y))\n", - " suite.check_close(\"noise=zeros: same stdev as the noise-free model\", pred(kz, Xt)[1], pred(kf, Xt)[1], rtol=1e-4,\n", - " atol=1e-6 * np.std(y))\n", - "with suite.check_runs(\"noise=zeros, sigma2 estimated: fit\", severity=\"warn\"):\n", - " lk.Kriging(y, X, \"matern5_2\", noise=np.zeros(len(y)), optim=\"none\",\n", - " parameters={\"theta\": theta, \"is_theta_estim\": False})" - ] - }, - { - "cell_type": "markdown", - "id": "c007", - "metadata": {}, - "source": [ - "## Heteroscedastic noise: known variances vs a single nugget" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c008", - "metadata": {}, - "outputs": [], - "source": [ - "tf = TESTS[\"gramacy_lee_2012\"]\n", - "n = 60\n", - "X, _, Xt, yt = design.dataset(tf, n, n_test=2000, seed=13)\n", - "sd_x = 0.02 + 0.4 * X[:, 0] # noise sd grows with x (in unit coordinates)\n", - "y = tf.unit(X) + np.random.default_rng(13).normal(0, sd_x)\n", - "k_known = lk.Kriging(y, X, \"matern5_2\", noise=sd_x ** 2, optim=\"BFGS10\")\n", - "k_nug = lk.Kriging(y, X, \"matern5_2\", noise=\"nugget\", optim=\"BFGS10\")\n", - "res_known = metrics.all_metrics(yt, *pred(k_known, Xt))\n", - "res_nug = metrics.all_metrics(yt, *pred(k_nug, Xt))\n", - "suite.metric(\"gramacy_lee_2012/hetero\", \"known variances\", **res_known)\n", - "suite.metric(\"gramacy_lee_2012/hetero\", \"nugget\", **res_nug)\n", - "suite.check_finite(\"hetero: predictions finite\", pred(k_known, Xt)[0])\n", - "suite.check_less(\"hetero: NLPD(known variances) <= NLPD(nugget)\", res_known[\"nlpd\"] - res_nug[\"nlpd\"], 0.05,\n", - " severity=\"warn\")\n", - "suite.check_close(\"noise(): returns the given variances\", np.ravel(k_known.noise()), sd_x ** 2, rtol=1e-12)\n", - "\n", - "u = np.linspace(0, 1, 400)[:, None]\n", - "fig, ax = plt.subplots(figsize=(10, 4))\n", - "ax.plot(u, tf.unit(u), \"k--\", label=\"truth\")\n", - "ax.errorbar(X[:, 0], y, yerr=2 * sd_x, fmt=\"k.\", alpha=0.4)\n", - "for k, lab in [(k_known, \"known variances\"), (k_nug, \"nugget\")]:\n", - " m, s = pred(k, u)\n", - " ax.plot(u, m, label=lab)\n", - " ax.fill_between(u[:, 0], m - 2 * s, m + 2 * s, alpha=0.15)\n", - "ax.legend();" - ] - }, - { - "cell_type": "markdown", - "id": "c009", - "metadata": {}, - "source": [ - "## Replicates\n", - "\n", - "r noisy replicates at the same point carry the same information as their mean\n", - "with variance σ²/r: fitting the replicates with `noise=σ²` or their means with\n", - "`noise=σ²/r` must give the same predictor at a fixed θ." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c010", - "metadata": {}, - "outputs": [], - "source": [ - "tf = TESTS[\"branin\"]\n", - "r, sd = 4, 2.0\n", - "X1, _, Xt, yt = design.dataset(tf, 15, n_test=300, seed=14)\n", - "Xr = np.repeat(X1, r, axis=0)\n", - "yr = tf.unit(Xr) + np.random.default_rng(14).normal(0, sd, len(Xr))\n", - "ybar = yr.reshape(-1, r).mean(axis=1)\n", - "k_rep = lk.Kriging(yr, Xr, \"matern5_2\", noise=np.full(len(yr), sd ** 2))\n", - "fixed = {\"theta\": np.asarray(k_rep.theta()).reshape(1, -1), \"is_theta_estim\": False,\n", - " \"sigma2\": float(k_rep.sigma2()), \"is_sigma2_estim\": False}\n", - "k_rep = lk.Kriging(yr, Xr, \"matern5_2\", noise=np.full(len(yr), sd ** 2), optim=\"none\", parameters=fixed)\n", - "k_avg = lk.Kriging(ybar, X1, \"matern5_2\", noise=np.full(len(ybar), sd ** 2 / r), optim=\"none\", parameters=fixed)\n", - "suite.check_close(\"replicates == averages with variance/r (mean)\", pred(k_rep, Xt)[0], pred(k_avg, Xt)[0],\n", - " rtol=1e-6, atol=1e-6 * np.std(yr))\n", - "suite.check_close(\"replicates == averages with variance/r (stdev)\", pred(k_rep, Xt)[1], pred(k_avg, Xt)[1],\n", - " rtol=1e-4, atol=1e-6 * np.std(yr))" - ] - }, - { - "cell_type": "markdown", - "id": "c011", - "metadata": {}, - "source": [ - "## Summary\n", - "\n", - "The metrics of this notebook, the number of checks per status, and every check that\n", - "did not pass (FAIL: error-level, fails the run; WARN: warn-level; KNOWN: listed in\n", - "`tools/known_issues.csv`, with its issue; SKIP: feature absent from the installed version).\n", - "All of them are also written to `results/noisy-data_Kriging-nugget-noise_python.checks.csv` and `.metrics.csv`." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c012", - "metadata": {}, - "outputs": [], - "source": [ - "suite.summary()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c013", - "metadata": {}, - "outputs": [], - "source": [ - "suite.finalize()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/objectives-LL-LOO-LMP-gradients_Kriging_python.ipynb b/objectives-LL-LOO-LMP-gradients_Kriging_python.ipynb deleted file mode 100644 index 74a140c..0000000 --- a/objectives-LL-LOO-LMP-gradients_Kriging_python.ipynb +++ /dev/null @@ -1,379 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "c000", - "metadata": {}, - "source": [ - "# Python · Kriging — objectives (LL, LOO, LMP), their gradients and optima\n", - "\n", - "**Idea.** The three estimation criteria (likelihood, leave-one-out, marginal\n", - "posterior) must be computed and differentiated correctly, and the fitted θ̂ must be\n", - "an optimum of the criterion it maximises.\n", - "\n", - "**Reference.** Finite differences of each criterion, explicit leave-one-out refits,\n", - "and the criterion profile around θ̂.\n", - "\n", - "The hyperparameters θ of `Kriging` are estimated by maximizing the\n", - "log-likelihood (`LL`), maximizing the log marginal posterior (`LMP`), or\n", - "minimizing the leave-one-out error (`LOO`). This notebook checks the machinery\n", - "behind those fits rather than the predictions:\n", - "\n", - "* **analytic gradients** of `logLikelihoodFun`, `leaveOneOutFun`,\n", - " `logMargPostFun` against central finite differences, at random θ, for\n", - " every kernel;\n", - "* **optimality** of the fitted θ: the objective at θ̂ is not improved by small\n", - " perturbations of θ̂ (a local optimum was reached). A component θ_j > 5 on\n", - " the unit cube is a nearly linear direction whose optimum sits at the upper\n", - " bound of θ: moving it further out is allowed to improve the objective;\n", - "* **global vs local optimum**: the gap between the default single-start fit\n", - " and a 20-start fit (warn-level: it measures how often the default fit ends in\n", - " a poor local optimum);\n", - "* **consistency** between `logLikelihood()` and `logLikelihoodFun(theta())`;\n", - "* the closed-form **leave-one-out** predictions of `leaveOneOutVec` against an\n", - " explicit refit without each point (same θ, σ²);\n", - "* multistart (`BFGS10`) never ends below a single `BFGS` run (warn-level: a\n", - " multistart that does worse points to a starting-point issue).\n", - "\n", - "Functions: Branin (2D), Hartmann 3 (3D), Ishigami (3D), Friedman (5D)." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c001", - "metadata": {}, - "outputs": [], - "source": [ - "# Setup for running this notebook outside the repository (Colab, Binder, a lone copy):\n", - "# nothing to do in the repository itself (CI, local runs). Otherwise the repository\n", - "# (datasets of data/, helpers of tools/python/gallery.py) is cloned and the released pylibkriging installed.\n", - "import importlib\n", - "import os\n", - "import subprocess\n", - "import sys\n", - "from pathlib import Path\n", - "\n", - "if not Path(\"tools/known_issues.csv\").is_file():\n", - " if not Path(\"gallery/tools/known_issues.csv\").is_file():\n", - " subprocess.run([\"git\", \"clone\", \"--depth\", \"1\", \"https://github.com/libKriging/gallery\"], check=True)\n", - " os.chdir(\"gallery\")\n", - " try:\n", - " import pylibkriging # noqa: F401\n", - " except ImportError:\n", - " r = subprocess.run([sys.executable, \"-m\", \"pip\", \"install\", \"--quiet\", \"pylibkriging\"],\n", - " capture_output=True, text=True)\n", - " if r.returncode != 0:\n", - " raise RuntimeError(\n", - " f\"pylibkriging could not be installed for Python {sys.version_info.major}.{sys.version_info.minor} \"\n", - " \"(PyPI has wheels for CPython 3.10 to 3.12 only). On Colab: Runtime > Change runtime version > \"\n", - " \"2026.07 (Python 3.12), then run again.\\n\" + r.stderr[-500:])\n", - " importlib.invalidate_caches()\n", - "sys.path.insert(0, str(Path(\"tools/python\").resolve())) # for \"import gallery\" (tools/python/gallery.py)\n", - "print(\"working directory:\", os.getcwd())" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c002", - "metadata": {}, - "outputs": [], - "source": [ - "import matplotlib.pyplot as plt\n", - "import numpy as np\n", - "import pandas as pd\n", - "import pylibkriging as lk\n", - "\n", - "import gallery\n", - "from gallery import TESTS, design, metrics, compat\n", - "\n", - "suite = gallery.Suite(\"objectives-LL-LOO-LMP-gradients_Kriging_python\")\n", - "print(\"pylibkriging\", compat.version())\n", - "CASES = {\"branin\": 25, \"hartmann3\": 40, \"ishigami\": 60, \"friedman\": 50}\n", - "KERNELS = [\"gauss\", \"exp\", \"matern3_2\", \"matern5_2\"]\n", - "rng = np.random.default_rng(0)\n", - "\n", - "\n", - "def objective_fun(k, objective):\n", - " \"\"\"(value, gradient) of the objective as a function of theta.\"\"\"\n", - " if objective == \"LL\":\n", - " return lambda t, g=True: k.logLikelihoodFun(np.asarray(t, float), g)[:2]\n", - " if objective == \"LOO\":\n", - " return lambda t, g=True: k.leaveOneOutFun(np.asarray(t, float), g)\n", - " return lambda t, g=True: k.logMargPostFun(np.asarray(t, float), g)" - ] - }, - { - "cell_type": "markdown", - "id": "c003", - "metadata": {}, - "source": [ - "## Analytic gradients vs finite differences" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c004", - "metadata": {}, - "outputs": [], - "source": [ - "rows = []\n", - "for fname, n in CASES.items():\n", - " tf = TESTS[fname]\n", - " X, y, _, _ = design.dataset(tf, n, seed=2)\n", - " for kernel in KERNELS:\n", - " k = lk.Kriging(y, X, kernel)\n", - " for objective in [\"LL\", \"LOO\", \"LMP\"]:\n", - " f = objective_fun(k, objective)\n", - " worst = 0.0\n", - " for _ in range(3):\n", - " theta = np.exp(rng.uniform(np.log(0.05), np.log(1.0), tf.d))\n", - " val, grad = f(theta)\n", - " fd = metrics.finite_diff_grad(lambda t: f(t, False)[0], theta, h=1e-5)\n", - " grad = np.asarray(grad).ravel()\n", - " err = np.max(np.abs(grad - fd)) / max(np.max(np.abs(fd)), 1e-8)\n", - " worst = max(worst, err)\n", - " rows.append((fname, kernel, objective, worst))\n", - " suite.check_less(f\"{fname}/{kernel}/{objective}: gradient vs finite diff (rel. err)\", worst, 1e-3)\n", - "pd.DataFrame(rows, columns=[\"function\", \"kernel\", \"objective\", \"max rel. error\"]).pivot_table(\n", - " index=[\"function\", \"kernel\"], columns=\"objective\", values=\"max rel. error\").style.format(\"{:.1e}\")" - ] - }, - { - "cell_type": "markdown", - "id": "c005", - "metadata": {}, - "source": [ - "## Optimality of the fitted θ, objective by objective" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c006", - "metadata": {}, - "outputs": [], - "source": [ - "SIGN = {\"LL\": 1, \"LMP\": 1, \"LOO\": -1} # +1: maximized, -1: minimized\n", - "for fname, n in CASES.items():\n", - " tf = TESTS[fname]\n", - " X, y, Xt, yt = design.dataset(tf, n, n_test=1000, seed=2)\n", - " for objective in [\"LL\", \"LOO\", \"LMP\"]:\n", - " k = lk.Kriging(y, X, \"matern5_2\", objective=objective)\n", - " theta = np.asarray(k.theta()).ravel()\n", - " f = objective_fun(k, objective)\n", - " best = SIGN[objective] * f(theta, False)[0]\n", - " improved = []\n", - " for j in range(tf.d):\n", - " for fac in (0.9, 1.1):\n", - " t = theta.copy(); t[j] *= fac\n", - " at_bound = fac > 1 and theta[j] > 5\n", - " # tolerance: the optimizer's stopping criterion (0.01 on a log-density is immaterial)\n", - " tol = 1e-4 * abs(best) + (0.0 if objective == \"LOO\" else 1e-2)\n", - " if SIGN[objective] * f(t, False)[0] > best + tol and not at_bound:\n", - " improved.append(f\"theta{j + 1}*{fac}\")\n", - " suite.check_true(f\"{fname}/{objective}: theta_hat not improved by +/-10% moves\", not improved,\n", - " detail=f\"theta={np.round(theta, 4)}; improving moves: {improved}\")\n", - " m, s, *_ = k.predict(Xt, return_stdev=True)\n", - " suite.metric(f\"{fname}/{objective}\", \"pylibkriging\", **metrics.all_metrics(yt, m, s))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c007", - "metadata": {}, - "outputs": [], - "source": [ - "tab = suite.table().reset_index()\n", - "tab[[\"function\", \"objective\"]] = tab[\"case\"].str.split(\"/\", expand=True)\n", - "tab.pivot_table(index=\"function\", columns=\"objective\", values=[\"q2\", \"coverage95\"]).round(3)" - ] - }, - { - "cell_type": "markdown", - "id": "c008", - "metadata": {}, - "source": [ - "## In pictures: each criterion along θ = t · θ̂ (Branin)\n", - "\n", - "Each criterion fitted on its own, then evaluated along the ray t · θ̂ through its\n", - "optimum (t = 1, dashed line): a maximum for LL and LMP, a minimum for LOO.\n", - "The three optima are not the same θ, which is why the fitted models differ." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c009", - "metadata": {}, - "outputs": [], - "source": [ - "tf = TESTS[\"branin\"]\n", - "X, y, _, _ = design.dataset(tf, CASES[\"branin\"], seed=2)\n", - "ts = np.exp(np.linspace(np.log(0.2), np.log(5), 61))\n", - "fig, axes = plt.subplots(1, 3, figsize=(13, 3.6))\n", - "for ax, objective in zip(axes, [\"LL\", \"LOO\", \"LMP\"]):\n", - " k = lk.Kriging(y, X, \"matern5_2\", objective=objective)\n", - " theta = np.asarray(k.theta()).ravel()\n", - " f = objective_fun(k, objective)\n", - " vals = [f(t * theta, False)[0] for t in ts]\n", - " ax.semilogx(ts, vals); ax.axvline(1, color=\"k\", ls=\"--\", lw=.8)\n", - " ax.set_title(f\"{objective}, θ̂ = {np.round(theta, 3)}\"); ax.set_xlabel(\"t (θ = t · θ̂)\")\n", - " i = int(np.argmax(vals) if SIGN[objective] > 0 else np.argmin(vals))\n", - " suite.check_between(f\"branin/{objective}: optimum of the profile along t * theta_hat (t)\", ts[i], 0.8, 1.25,\n", - " severity=\"warn\")\n", - "axes[0].set_ylabel(\"criterion\"); plt.tight_layout(); plt.show()" - ] - }, - { - "cell_type": "markdown", - "id": "c010", - "metadata": {}, - "source": [ - "## `logLikelihood()` == `logLikelihoodFun(theta())` (and the same for LOO, LMP)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c011", - "metadata": {}, - "outputs": [], - "source": [ - "tf = TESTS[\"hartmann3\"]\n", - "X, y, _, _ = design.dataset(tf, 40, seed=5)\n", - "for objective, getter in [(\"LL\", \"logLikelihood\"), (\"LOO\", \"leaveOneOut\"), (\"LMP\", \"logMargPost\")]:\n", - " k = lk.Kriging(y, X, \"matern5_2\", objective=objective)\n", - " v_fun = objective_fun(k, objective)(np.asarray(k.theta()).ravel(), False)[0]\n", - " suite.check_close(f\"{getter}() == {objective} function at theta()\", getattr(k, getter)(), v_fun, rtol=1e-8)" - ] - }, - { - "cell_type": "markdown", - "id": "c012", - "metadata": {}, - "source": [ - "## Closed-form leave-one-out vs explicit refits\n", - "\n", - "`leaveOneOutVec(θ)` gives the mean and stdev of the prediction of each `y_i`\n", - "from the n−1 others, by the closed-form (Dubrule) formulas. An explicit refit\n", - "without point i, with θ and σ² fixed (β re-estimated, as in the formula), must\n", - "give the same mean." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c013", - "metadata": {}, - "outputs": [], - "source": [ - "for fname in [\"branin\", \"hartmann3\"]:\n", - " tf = TESTS[fname]\n", - " X, y, _, _ = design.dataset(tf, 20, seed=6)\n", - " for trend in [\"constant\", \"linear\"]:\n", - " k = lk.Kriging(y, X, \"matern5_2\", regmodel=trend)\n", - " theta, sigma2 = np.asarray(k.theta()).ravel(), float(k.sigma2())\n", - " m_loo, s_loo = (np.asarray(a).ravel() for a in k.leaveOneOutVec(theta))\n", - " m_ref, s_ref = np.zeros(len(y)), np.zeros(len(y))\n", - " for i in range(len(y)):\n", - " keep = np.arange(len(y)) != i\n", - " ki = lk.Kriging(y[keep], X[keep], \"matern5_2\", regmodel=trend, optim=\"none\",\n", - " parameters={\"theta\": theta.reshape(1, -1), \"is_theta_estim\": False,\n", - " \"sigma2\": sigma2, \"is_sigma2_estim\": False})\n", - " mi, si, *_ = ki.predict(X[i:i + 1], return_stdev=True)\n", - " m_ref[i], s_ref[i] = np.ravel(mi)[0], np.ravel(si)[0]\n", - " suite.check_close(f\"{fname}/{trend}: leaveOneOutVec mean == explicit refits\", m_loo, m_ref,\n", - " rtol=1e-6, atol=1e-8 * np.std(y))\n", - " suite.check_close(f\"{fname}/{trend}: leaveOneOutVec stdev == explicit refits\", s_loo, s_ref,\n", - " rtol=1e-4, atol=1e-8 * np.std(y), severity=\"warn\")\n", - " # leaveOneOut() is the mean squared LOO error at the fitted theta\n", - " k_loo = lk.Kriging(y, X, \"matern5_2\", regmodel=trend, objective=\"LOO\")\n", - " m2, _ = k_loo.leaveOneOutVec(np.asarray(k_loo.theta()).ravel())\n", - " suite.check_close(f\"{fname}/{trend}: leaveOneOut() == mean squared LOO error\", k_loo.leaveOneOut(),\n", - " np.mean((np.ravel(m2) - y) ** 2), rtol=1e-6, severity=\"warn\")" - ] - }, - { - "cell_type": "markdown", - "id": "c014", - "metadata": {}, - "source": [ - "## Multistart\n", - "\n", - "Gap between the default single start and 10 / 20 starts. A large gap means the\n", - "default fit is unreliable on this kind of problem." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c015", - "metadata": {}, - "outputs": [], - "source": [ - "for fname, n in CASES.items():\n", - " tf = TESTS[fname]\n", - " X, y, _, _ = design.dataset(tf, n, seed=7)\n", - " ll1 = lk.Kriging(y, X, \"matern5_2\", optim=\"BFGS\").logLikelihood()\n", - " ll10 = lk.Kriging(y, X, \"matern5_2\", optim=\"BFGS10\").logLikelihood()\n", - " ll20 = lk.Kriging(y, X, \"matern5_2\", optim=\"BFGS20\").logLikelihood()\n", - " suite.metric(fname, \"BFGS\", ll=ll1)\n", - " suite.metric(fname, \"BFGS10\", ll=ll10)\n", - " suite.metric(fname, \"BFGS20\", ll=ll20)\n", - " suite.check_greater(f\"{fname}: LL(BFGS10) - LL(BFGS)\", ll10 - ll1, -1e-3 * abs(ll1), severity=\"warn\")\n", - " suite.check_less(f\"{fname}: LL(BFGS20) - LL(BFGS) (default fit far from the best optimum)\", ll20 - ll1, 1.0,\n", - " severity=\"warn\")\n", - "suite.table().loc[list(CASES)].unstack(\"package\")" - ] - }, - { - "cell_type": "markdown", - "id": "c016", - "metadata": {}, - "source": [ - "## Summary\n", - "\n", - "The metrics of this notebook, the number of checks per status, and every check that\n", - "did not pass (FAIL: error-level, fails the run; WARN: warn-level; KNOWN: listed in\n", - "`tools/known_issues.csv`, with its issue; SKIP: feature absent from the installed version).\n", - "All of them are also written to `results/objectives-LL-LOO-LMP-gradients_Kriging_python.checks.csv` and `.metrics.csv`." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c017", - "metadata": {}, - "outputs": [], - "source": [ - "suite.summary()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c018", - "metadata": {}, - "outputs": [], - "source": [ - "suite.finalize()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/robustness-invalid-inputs_Kriging_python.ipynb b/robustness-invalid-inputs_Kriging_python.ipynb deleted file mode 100644 index 214d97c..0000000 --- a/robustness-invalid-inputs_Kriging_python.ipynb +++ /dev/null @@ -1,333 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "c000", - "metadata": {}, - "source": [ - "# Python · Kriging — robustness: degenerate and adversarial inputs\n", - "\n", - "**Idea.** What happens off the happy path: invalid, degenerate or ill-conditioned\n", - "inputs must be rejected or handled, results must not depend on the units, and the\n", - "prediction cost must grow linearly with the number of points.\n", - "\n", - "**Reference.** Equivariance identities (rescaled inputs / outputs give rescaled\n", - "results) and the O(q) cost of `predict`.\n", - "\n", - "What a user eventually feeds a surrogate model, and what must happen:\n", - "\n", - "| input | expected behaviour |\n", - "|---|---|\n", - "| NaN / Inf in X or y | rejected with an exception (never a silent NaN model) |\n", - "| X and y with inconsistent sizes | rejected |\n", - "| a wrong number of warping specs | rejected |\n", - "| duplicated design points, noise-free | either rejected or a finite model (no NaN, no crash) |\n", - "| duplicated design points with `noise=\"nugget\"` | a finite model |\n", - "| constant response y | a finite model predicting the constant |\n", - "| very few points (n = 2, 3) | a finite model |\n", - "| very clustered points (1e-8 apart) with the `gauss` kernel | finite predictions (ill-conditioning) |\n", - "| inputs scaled by 1e6 or 1e-6 with `normalize=True` | the same predictions as on [0,1] |\n", - "| y scaled by 1e8 / shifted by 1e6 | predictions scaled / shifted accordingly (equivariance) |\n", - "| predicting far outside the design | the mean reverts to the trend, the stdev to (slightly above) √σ² |\n", - "| many prediction points | cost linear in their number (mean and stdev only need O(n·q), not O(q²)) |" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c001", - "metadata": {}, - "outputs": [], - "source": [ - "# Setup for running this notebook outside the repository (Colab, Binder, a lone copy):\n", - "# nothing to do in the repository itself (CI, local runs). Otherwise the repository\n", - "# (datasets of data/, helpers of tools/python/gallery.py) is cloned and the released pylibkriging installed.\n", - "import importlib\n", - "import os\n", - "import subprocess\n", - "import sys\n", - "from pathlib import Path\n", - "\n", - "if not Path(\"tools/known_issues.csv\").is_file():\n", - " if not Path(\"gallery/tools/known_issues.csv\").is_file():\n", - " subprocess.run([\"git\", \"clone\", \"--depth\", \"1\", \"https://github.com/libKriging/gallery\"], check=True)\n", - " os.chdir(\"gallery\")\n", - " try:\n", - " import pylibkriging # noqa: F401\n", - " except ImportError:\n", - " r = subprocess.run([sys.executable, \"-m\", \"pip\", \"install\", \"--quiet\", \"pylibkriging\"],\n", - " capture_output=True, text=True)\n", - " if r.returncode != 0:\n", - " raise RuntimeError(\n", - " f\"pylibkriging could not be installed for Python {sys.version_info.major}.{sys.version_info.minor} \"\n", - " \"(PyPI has wheels for CPython 3.10 to 3.12 only). On Colab: Runtime > Change runtime version > \"\n", - " \"2026.07 (Python 3.12), then run again.\\n\" + r.stderr[-500:])\n", - " importlib.invalidate_caches()\n", - "sys.path.insert(0, str(Path(\"tools/python\").resolve())) # for \"import gallery\" (tools/python/gallery.py)\n", - "print(\"working directory:\", os.getcwd())" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c002", - "metadata": {}, - "outputs": [], - "source": [ - "import time\n", - "\n", - "import matplotlib.pyplot as plt\n", - "import numpy as np\n", - "import pylibkriging as lk\n", - "\n", - "import gallery\n", - "from gallery import TESTS, design, compat\n", - "\n", - "suite = gallery.Suite(\"robustness-invalid-inputs_Kriging_python\")\n", - "print(\"pylibkriging\", compat.version())\n", - "tf = TESTS[\"branin\"]\n", - "X, y, Xt, yt = design.dataset(tf, 20, n_test=200, seed=71)\n", - "\n", - "\n", - "def pred(k, Xn):\n", - " m, s, *_ = k.predict(Xn, return_stdev=True)\n", - " return np.ravel(m), np.ravel(s)\n", - "\n", - "\n", - "def finite_model(check, make, Xn=Xt, severity=\"error\"):\n", - " \"\"\"The model either raises (acceptable) or predicts finite values.\"\"\"\n", - " try:\n", - " k = make()\n", - " except Exception as exc: # noqa: BLE001\n", - " suite.check_true(f\"{check}: rejected or finite\", True, detail=f\"rejected: {type(exc).__name__}\")\n", - " return None\n", - " m, s = pred(k, Xn)\n", - " suite.check_true(f\"{check}: rejected or finite\", np.all(np.isfinite(m)) and np.all(np.isfinite(s)),\n", - " detail=f\"non-finite: {np.sum(~np.isfinite(m))} mean, {np.sum(~np.isfinite(s))} stdev\",\n", - " severity=severity)\n", - " return k" - ] - }, - { - "cell_type": "markdown", - "id": "c003", - "metadata": {}, - "source": [ - "## Invalid inputs are rejected" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c004", - "metadata": {}, - "outputs": [], - "source": [ - "Xn = X.copy(); Xn[3, 1] = np.nan\n", - "yn = y.copy(); yn[5] = np.nan\n", - "yi = y.copy(); yi[5] = np.inf\n", - "suite.check_raises(\"NaN in X is rejected\", lambda: lk.Kriging(y, Xn, \"matern5_2\"))\n", - "suite.check_raises(\"NaN in y is rejected\", lambda: lk.Kriging(yn, X, \"matern5_2\"))\n", - "suite.check_raises(\"Inf in y is rejected\", lambda: lk.Kriging(yi, X, \"matern5_2\"))\n", - "suite.check_raises(\"len(y) != nrow(X) is rejected\", lambda: lk.Kriging(y[:-1], X, \"matern5_2\"))\n", - "suite.check_raises(\"unknown kernel is rejected\", lambda: lk.Kriging(y, X, \"matern7_2\"))\n", - "suite.check_raises(\"unknown trend is rejected\", lambda: lk.Kriging(y, X, \"matern5_2\", regmodel=\"cubic\"))\n", - "suite.check_raises(\"unknown objective is rejected\", lambda: lk.Kriging(y, X, \"matern5_2\", objective=\"XYZ\"))\n", - "suite.check_raises(\"noise vector of the wrong length is rejected\",\n", - " lambda: lk.Kriging(y, X, \"matern5_2\", noise=np.ones(len(y) - 1)))\n", - "suite.check_raises(\"negative noise variance is rejected\",\n", - " lambda: lk.Kriging(y, X, \"matern5_2\", noise=-np.ones(len(y))), severity=\"warn\")\n", - "k = lk.Kriging(y, X, \"matern5_2\")\n", - "suite.check_raises(\"predict with the wrong number of columns is rejected\", lambda: k.predict(np.ones((3, 3)), True))\n", - "if compat.has(\"WarpKriging\"):\n", - " suite.check_raises(\"WarpKriging with 1 spec for 2 columns is rejected\", lambda: lk.WarpKriging(y, X, [\"none\"], \"gauss\"))\n", - " suite.check_raises(\"WarpKriging with an unknown spec is rejected\",\n", - " lambda: lk.WarpKriging(y, X, [\"none\", \"spline\"], \"gauss\"))" - ] - }, - { - "cell_type": "markdown", - "id": "c005", - "metadata": {}, - "source": [ - "## Degenerate designs and responses" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c006", - "metadata": {}, - "outputs": [], - "source": [ - "Xd = np.vstack([X, X[:3]]); yd = np.concatenate([y, y[:3]])\n", - "finite_model(\"duplicated points, noise-free\", lambda: lk.Kriging(yd, Xd, \"matern5_2\"))\n", - "if compat.has(\"noise\"):\n", - " k = finite_model(\"duplicated points, nugget\", lambda: lk.Kriging(yd, Xd, \"matern5_2\", noise=\"nugget\"))\n", - " suite.check_true(\"duplicated points, nugget: a model is built\", k is not None)\n", - "kc = finite_model(\"constant y\", lambda: lk.Kriging(np.full(20, 3.7), X, \"matern5_2\"))\n", - "if kc is not None:\n", - " suite.check_close(\"constant y: predicts the constant\", pred(kc, Xt)[0], np.full(len(Xt), 3.7), rtol=1e-6, atol=1e-6)\n", - "for n in (2, 3):\n", - " finite_model(f\"n = {n}\", lambda: lk.Kriging(y[:n], X[:n], \"matern5_2\"))\n", - "Xclose = np.vstack([X[:10], X[:10] + 1e-8])\n", - "yclose = tf.unit(Xclose)\n", - "for kernel in (\"gauss\", \"matern5_2\"):\n", - " finite_model(f\"points 1e-8 apart, {kernel}\", lambda: lk.Kriging(yclose, Xclose, kernel))\n", - "# a single input dimension constant over the design\n", - "Xconst = np.column_stack([X[:, 0], np.full(20, 0.5)])\n", - "finite_model(\"one input constant over the design\", lambda: lk.Kriging(tf.unit(Xconst), Xconst, \"matern5_2\"))" - ] - }, - { - "cell_type": "markdown", - "id": "c007", - "metadata": {}, - "source": [ - "## Scale invariance and equivariance" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c008", - "metadata": {}, - "outputs": [], - "source": [ - "k0 = lk.Kriging(y, X, \"matern5_2\", normalize=True)\n", - "m0, s0 = pred(k0, Xt)\n", - "for scale in (1e-6, 1e6):\n", - " with suite.check_runs(f\"X * {scale:g} with normalize=True: fit\"):\n", - " ks = lk.Kriging(y, X * scale, \"matern5_2\", normalize=True)\n", - " suite.check_close(f\"X * {scale:g}, normalize=True: same mean as on [0,1]\", pred(ks, Xt * scale)[0], m0,\n", - " rtol=1e-3, atol=1e-3 * np.std(y))\n", - "for a, b in [(1e8, 0.0), (1.0, 1e6), (1e-8, 0.0)]:\n", - " with suite.check_runs(f\"y * {a:g} + {b:g}: fit\"):\n", - " ka = lk.Kriging(a * y + b, X, \"matern5_2\", normalize=True)\n", - " ma, sa = pred(ka, Xt)\n", - " suite.check_close(f\"y * {a:g} + {b:g}: mean equivariant\", ma, a * m0 + b, rtol=1e-3, atol=1e-3 * a * np.std(y))\n", - " suite.check_close(f\"y * {a:g} + {b:g}: stdev scales by |a|\", sa, abs(a) * s0, rtol=1e-2,\n", - " atol=1e-3 * a * np.std(y))\n", - "# the same without normalize: the optimizer starts from a scale-dependent point, warn-level only\n", - "for a, b in [(1e6, 0.0), (1.0, 1e6)]:\n", - " ka = lk.Kriging(a * y + b, X, \"matern5_2\")\n", - " suite.check_close(f\"y * {a:g} + {b:g} (normalize=False): mean equivariant\", pred(ka, Xt)[0],\n", - " a * pred(lk.Kriging(y, X, \"matern5_2\"), Xt)[0] + b, rtol=1e-3, atol=1e-3 * a * np.std(y),\n", - " severity=\"warn\")" - ] - }, - { - "cell_type": "markdown", - "id": "c009", - "metadata": {}, - "source": [ - "## Far from the data" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c010", - "metadata": {}, - "outputs": [], - "source": [ - "k = lk.Kriging(y, X, \"matern5_2\")\n", - "far = np.array([[50.0, 50.0], [-30.0, 80.0]])\n", - "m, s = pred(k, far)\n", - "suite.check_close(\"far away: mean -> beta (constant trend)\", m, np.full(2, np.ravel(k.beta())[0]), rtol=1e-6,\n", - " atol=1e-8)\n", - "# universal kriging: sigma2 * (1 + 1 / (1' R^-1 1)) with the trend uncertainty, slightly above sigma2\n", - "suite.check_between(\"far away: stdev / sqrt(sigma2)\", np.max(s) / np.sqrt(k.sigma2()), 1.0 - 1e-6, 1.5)\n", - "suite.check_close(\"far away: same stdev at both points\", s[0], s[1], rtol=1e-6)" - ] - }, - { - "cell_type": "markdown", - "id": "c011", - "metadata": {}, - "source": [ - "## Many prediction points\n", - "\n", - "Mean and stdev at q points need O(n·q) operations and memory; only the full\n", - "covariance (`return_cov=True`) is O(q²). The time for q = 1000, 2000, 4000,\n", - "8000 points must grow linearly (log2 of the ratio per doubling ~ 1). An\n", - "exponent close to 2 means a q × q matrix is formed: a million points then needs\n", - "terabytes. (Not run at 10⁶ points directly: it would exhaust the runner.)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c012", - "metadata": {}, - "outputs": [], - "source": [ - "times = {}\n", - "for q in (1000, 2000, 4000, 8000):\n", - " Xq = design.uniform(q, 2, seed=72)\n", - " best = np.inf\n", - " for _ in range(3):\n", - " t0 = time.perf_counter()\n", - " pred(k, Xq)\n", - " best = min(best, time.perf_counter() - t0)\n", - " times[q] = best\n", - " suite.metric(f\"predict_q={q}\", \"pylibkriging\", seconds=best)\n", - "expo = np.log2(times[8000] / times[4000])\n", - "suite.check_less(\"predict(stdev) time growth exponent in q (8000 vs 4000)\", expo, 1.5,\n", - " detail=\", \".join(f\"q={q}: {t:.3f}s\" for q, t in times.items()))\n", - "qs = np.array(list(times)); ts = np.array(list(times.values()))\n", - "plt.figure(figsize=(5, 3.6))\n", - "plt.loglog(qs, ts, \"o-\", label=\"predict(return_stdev=True)\")\n", - "plt.loglog(qs, ts[0] * qs / qs[0], \"k--\", lw=.8, label=\"O(q) reference\")\n", - "plt.loglog(qs, ts[0] * (qs / qs[0]) ** 2, \"k:\", lw=.8, label=\"O(q²) reference\")\n", - "plt.xlabel(\"q (prediction points)\"); plt.ylabel(\"time (s)\"); plt.legend(); plt.grid(True, which=\"both\", alpha=.3)\n", - "plt.show()" - ] - }, - { - "cell_type": "markdown", - "id": "c013", - "metadata": {}, - "source": [ - "## Summary\n", - "\n", - "The metrics of this notebook, the number of checks per status, and every check that\n", - "did not pass (FAIL: error-level, fails the run; WARN: warn-level; KNOWN: listed in\n", - "`tools/known_issues.csv`, with its issue; SKIP: feature absent from the installed version).\n", - "All of them are also written to `results/robustness-invalid-inputs_Kriging_python.checks.csv` and `.metrics.csv`." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c014", - "metadata": {}, - "outputs": [], - "source": [ - "suite.summary()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c015", - "metadata": {}, - "outputs": [], - "source": [ - "suite.finalize()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/save-load-pickle-copy-accessors_Kriging_python.ipynb b/save-load-pickle-copy-accessors_Kriging_python.ipynb deleted file mode 100644 index e1f51b3..0000000 --- a/save-load-pickle-copy-accessors_Kriging_python.ipynb +++ /dev/null @@ -1,239 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "c000", - "metadata": {}, - "source": [ - "# Python · Kriging — save/load, pickle, copy, accessors\n", - "\n", - "**Idea.** A model must survive a round trip (file, pickle, copy) unchanged, and\n", - "expose its settings and hyperparameters: for every noise model, objective and fit\n", - "method of `Kriging`.\n", - "\n", - "**Reference.** The original model: predictions and hyperparameters must be identical\n", - "to the last bit (1e-12).\n", - "\n", - "Persistence and introspection of `Kriging`, for each of its variants (noise\n", - "models, objectives, approximate fit methods):\n", - "\n", - "* `save` → `load` (class-specific and generic `pylibkriging.load`) gives a model\n", - " with **identical** predictions and hyperparameters;\n", - "* `pickle` / `copy` round trips, and a copy is independent of the original\n", - " (updating one leaves the other untouched);\n", - "* accessors mirror the constructor arguments (`kernel()`, `regmodel()`,\n", - " `optim()`, `objective()`, `X()`, `y()`, ...).\n", - "\n", - "The same persistence checks for the other classes are in their own notebooks\n", - "(`*_WarpKriging_python`, `*_MLPKriging_python`, `*_NestedKriging_python`); the\n", - "scikit-learn estimators in `sklearn-estimators_Kriging_python.ipynb`.\n", - "\n", - "An API notebook: its results are identities (to 1e-12), so they are given as\n", - "the table of checks of the Summary rather than as plots." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c001", - "metadata": {}, - "outputs": [], - "source": [ - "# Setup for running this notebook outside the repository (Colab, Binder, a lone copy):\n", - "# nothing to do in the repository itself (CI, local runs). Otherwise the repository\n", - "# (datasets of data/, helpers of tools/python/gallery.py) is cloned and the released pylibkriging installed.\n", - "import importlib\n", - "import os\n", - "import subprocess\n", - "import sys\n", - "from pathlib import Path\n", - "\n", - "if not Path(\"tools/known_issues.csv\").is_file():\n", - " if not Path(\"gallery/tools/known_issues.csv\").is_file():\n", - " subprocess.run([\"git\", \"clone\", \"--depth\", \"1\", \"https://github.com/libKriging/gallery\"], check=True)\n", - " os.chdir(\"gallery\")\n", - " try:\n", - " import pylibkriging # noqa: F401\n", - " except ImportError:\n", - " r = subprocess.run([sys.executable, \"-m\", \"pip\", \"install\", \"--quiet\", \"pylibkriging\"],\n", - " capture_output=True, text=True)\n", - " if r.returncode != 0:\n", - " raise RuntimeError(\n", - " f\"pylibkriging could not be installed for Python {sys.version_info.major}.{sys.version_info.minor} \"\n", - " \"(PyPI has wheels for CPython 3.10 to 3.12 only). On Colab: Runtime > Change runtime version > \"\n", - " \"2026.07 (Python 3.12), then run again.\\n\" + r.stderr[-500:])\n", - " importlib.invalidate_caches()\n", - "sys.path.insert(0, str(Path(\"tools/python\").resolve())) # for \"import gallery\" (tools/python/gallery.py)\n", - "print(\"working directory:\", os.getcwd())" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c002", - "metadata": {}, - "outputs": [], - "source": [ - "import copy\n", - "\n", - "import numpy as np\n", - "import pylibkriging as lk\n", - "\n", - "import gallery\n", - "from gallery import TESTS, design, compat\n", - "\n", - "suite = gallery.Suite(\"save-load-pickle-copy-accessors_Kriging_python\")\n", - "print(\"pylibkriging\", compat.version())\n", - "\n", - "tf = TESTS[\"hartmann3\"]\n", - "X, y, Xt, yt = design.dataset(tf, 40, n_test=100, seed=61)\n", - "\n", - "\n", - "def mean_sd(k, Xn):\n", - " out = k.predict(Xn, return_stdev=True)\n", - " return np.ravel(out[0]), np.ravel(out[1])\n", - "\n", - "\n", - "MODELS = {\n", - " \"Kriging\": lambda: lk.Kriging(y, X, \"matern5_2\", regmodel=\"linear\"),\n", - " \"Kriging nugget\": lambda: lk.Kriging(y, X, \"matern3_2\", noise=\"nugget\"),\n", - " \"Kriging noise vector\": lambda: lk.Kriging(y, X, \"gauss\", noise=np.full(len(y), 1e-4)),\n", - " \"Kriging LOO\": lambda: lk.Kriging(y, X, \"matern5_2\", objective=\"LOO\"),\n", - " \"Kriging LMP\": lambda: lk.Kriging(y, X, \"matern5_2\", objective=\"LMP\"),\n", - "}\n", - "if compat.has(\"method\"):\n", - " MODELS[\"Kriging vecchia\"] = lambda: lk.Kriging(y, X, \"matern5_2\", method=\"vecchia(10)\")\n", - " MODELS[\"Kriging nystrom\"] = lambda: lk.Kriging(y, X, \"matern5_2\", method=\"nystrom(20)\")\n", - "\n", - "\n", - "def predict_any(k, Xn):\n", - " meth = k.method() if hasattr(k, \"method\") and isinstance(k, lk.Kriging) else None\n", - " if meth and meth != \"exact\":\n", - " out = k.predict(Xn, return_stdev=True, method=meth.split(\"(\")[0])\n", - " return np.ravel(out[0]), np.ravel(out[1])\n", - " return mean_sd(k, Xn)" - ] - }, - { - "cell_type": "markdown", - "id": "c003", - "metadata": {}, - "source": [ - "## save / load, pickle, copy" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c004", - "metadata": {}, - "outputs": [], - "source": [ - "for name, make in MODELS.items():\n", - " with suite.check_runs(f\"{name}: fit\"):\n", - " gallery.persistence.check(suite, name, make(), Xt, predict_any)" - ] - }, - { - "cell_type": "markdown", - "id": "c005", - "metadata": {}, - "source": [ - "A copy is independent: updating it leaves the original model unchanged." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c006", - "metadata": {}, - "outputs": [], - "source": [ - "k = lk.Kriging(y[:30], X[:30], \"matern5_2\")\n", - "m_before = mean_sd(k, Xt)[0]\n", - "kc = k.copy() if hasattr(k, \"copy\") else copy.deepcopy(k)\n", - "kc.update(y[30:], X[30:], refit=False)\n", - "suite.check_close(\"update on a copy leaves the original unchanged\", mean_sd(k, Xt)[0], m_before, rtol=0, atol=0)\n", - "suite.check_true(\"the updated copy did change\", not np.allclose(mean_sd(kc, Xt)[0], m_before))" - ] - }, - { - "cell_type": "markdown", - "id": "c007", - "metadata": {}, - "source": [ - "## Accessors mirror the constructor" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c008", - "metadata": {}, - "outputs": [], - "source": [ - "k = lk.Kriging(y, X, \"matern3_2\", regmodel=\"linear\", normalize=False, optim=\"BFGS\", objective=\"LMP\")\n", - "for acc, expected in [(\"kernel\", \"matern3_2\"), (\"regmodel\", \"linear\"), (\"optim\", \"BFGS\"), (\"objective\", \"LMP\"),\n", - " (\"normalize\", False)]:\n", - " suite.check_true(f\"Kriging.{acc}() == {expected!r}\", getattr(k, acc)() == expected, detail=str(getattr(k, acc)()))\n", - "suite.check_close(\"Kriging.X() is the design\", np.asarray(k.X()), X, rtol=0, atol=0)\n", - "suite.check_close(\"Kriging.y() is the response\", np.ravel(k.y()), y, rtol=0, atol=0)\n", - "# with normalize=True: do the accessors return the data as given, or as rescaled internally?\n", - "kn = lk.Kriging(y, X * 10, \"matern3_2\", normalize=True)\n", - "suite.check_true(\"Kriging.normalize() == True\", kn.normalize() is True or kn.normalize() == 1)\n", - "suite.check_close(\"normalize=True: X() is the design as given\", np.asarray(kn.X()), X * 10, rtol=1e-12, atol=0,\n", - " severity=\"warn\")\n", - "suite.check_close(\"normalize=True: y() is the response as given\", np.ravel(kn.y()), y, rtol=1e-12, atol=0,\n", - " severity=\"warn\")\n", - "suite.check_true(\"Kriging.beta() has 1 + d coefficients (linear trend)\", np.ravel(k.beta()).size == 1 + X.shape[1])\n", - "suite.check_true(\"summary() is a non-empty string\", isinstance(k.summary(), str) and len(k.summary()) > 20)\n", - "suite.check_true(\"is_theta_estim() after an optimized fit\", bool(k.is_theta_estim()))" - ] - }, - { - "cell_type": "markdown", - "id": "c009", - "metadata": {}, - "source": [ - "## Summary\n", - "\n", - "The metrics of this notebook, the number of checks per status, and every check that\n", - "did not pass (FAIL: error-level, fails the run; WARN: warn-level; KNOWN: listed in\n", - "`tools/known_issues.csv`, with its issue; SKIP: feature absent from the installed version).\n", - "All of them are also written to `results/save-load-pickle-copy-accessors_Kriging_python.checks.csv` and `.metrics.csv`." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c010", - "metadata": {}, - "outputs": [], - "source": [ - "suite.summary()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c011", - "metadata": {}, - "outputs": [], - "source": [ - "suite.finalize()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/sklearn-estimators_Kriging_python.ipynb b/sklearn-estimators_Kriging_python.ipynb deleted file mode 100644 index 0c2752e..0000000 --- a/sklearn-estimators_Kriging_python.ipynb +++ /dev/null @@ -1,222 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "c000", - "metadata": {}, - "source": [ - "# Python · Kriging — scikit-learn estimators (`pylibkriging.sklearn`)\n", - "\n", - "**Idea.** `pylibkriging.sklearn` makes `Kriging` a scikit-learn estimator, usable in\n", - "pipelines, grid searches and cross-validation.\n", - "\n", - "**Reference.** The native `Kriging` class on the same data (same predictions), and\n", - "scikit-learn's estimator conventions.\n", - "\n", - "`KrigingRegressor` (the scikit-learn face of `Kriging`) on Hartmann 3:\n", - "`fit`/`predict` (scikit-learn conventions), the same predictions as the native\n", - "class, `get_params`/`set_params`/`clone`, use inside `Pipeline`,\n", - "`GridSearchCV`, `cross_val_score`, and `sample_y`." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c001", - "metadata": {}, - "outputs": [], - "source": [ - "# Setup for running this notebook outside the repository (Colab, Binder, a lone copy):\n", - "# nothing to do in the repository itself (CI, local runs). Otherwise the repository\n", - "# (datasets of data/, helpers of tools/python/gallery.py) is cloned and the released pylibkriging installed.\n", - "import importlib\n", - "import os\n", - "import subprocess\n", - "import sys\n", - "from pathlib import Path\n", - "\n", - "if not Path(\"tools/known_issues.csv\").is_file():\n", - " if not Path(\"gallery/tools/known_issues.csv\").is_file():\n", - " subprocess.run([\"git\", \"clone\", \"--depth\", \"1\", \"https://github.com/libKriging/gallery\"], check=True)\n", - " os.chdir(\"gallery\")\n", - " try:\n", - " import pylibkriging # noqa: F401\n", - " except ImportError:\n", - " r = subprocess.run([sys.executable, \"-m\", \"pip\", \"install\", \"--quiet\", \"pylibkriging\"],\n", - " capture_output=True, text=True)\n", - " if r.returncode != 0:\n", - " raise RuntimeError(\n", - " f\"pylibkriging could not be installed for Python {sys.version_info.major}.{sys.version_info.minor} \"\n", - " \"(PyPI has wheels for CPython 3.10 to 3.12 only). On Colab: Runtime > Change runtime version > \"\n", - " \"2026.07 (Python 3.12), then run again.\\n\" + r.stderr[-500:])\n", - " importlib.invalidate_caches()\n", - "sys.path.insert(0, str(Path(\"tools/python\").resolve())) # for \"import gallery\" (tools/python/gallery.py)\n", - "print(\"working directory:\", os.getcwd())" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c002", - "metadata": {}, - "outputs": [], - "source": [ - "import matplotlib.pyplot as plt\n", - "import numpy as np\n", - "import pandas as pd\n", - "import pylibkriging as lk\n", - "\n", - "import gallery\n", - "from gallery import TESTS, design, compat\n", - "\n", - "suite = gallery.Suite(\"sklearn-estimators_Kriging_python\")\n", - "print(\"pylibkriging\", compat.version())\n", - "\n", - "tf = TESTS[\"hartmann3\"]\n", - "X, y, Xt, yt = design.dataset(tf, 40, n_test=100, seed=61)\n", - "\n", - "\n", - "def mean_sd(k, Xn):\n", - " out = k.predict(Xn, return_stdev=True)\n", - " return np.ravel(out[0]), np.ravel(out[1])" - ] - }, - { - "cell_type": "markdown", - "id": "c003", - "metadata": {}, - "source": [ - "## Installing scikit-learn\n", - "\n", - "With pip, into the kernel's environment, numpy pinned to its installed version\n", - "(a package that would downgrade it is not installed: the libKriging extension\n", - "is built against it). A package that cannot be installed is skipped, and\n", - "reported." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c004", - "metadata": {}, - "outputs": [], - "source": [ - "import importlib\n", - "import subprocess\n", - "import sys\n", - "\n", - "import numpy\n", - "\n", - "for package in [\"scikit-learn\"]:\n", - " r = subprocess.run([sys.executable, \"-m\", \"pip\", \"install\", \"--quiet\", package, f\"numpy=={numpy.__version__}\"],\n", - " capture_output=True, text=True)\n", - " print(f\"{package}: \" + (\"installed\" if r.returncode == 0 else \"NOT installed\\n\" + r.stderr.strip()[-800:]))\n", - "importlib.invalidate_caches()" - ] - }, - { - "cell_type": "markdown", - "id": "c005", - "metadata": {}, - "source": [ - "## scikit-learn estimators" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c006", - "metadata": {}, - "outputs": [], - "source": [ - "if compat.has(\"sklearn\"):\n", - " from sklearn.base import clone\n", - " from sklearn.model_selection import GridSearchCV, cross_val_score\n", - " from sklearn.pipeline import make_pipeline\n", - " from sklearn.preprocessing import MinMaxScaler\n", - " from pylibkriging.sklearn import KrigingRegressor\n", - "\n", - " est = KrigingRegressor(kernel=\"matern5_2\").fit(X, y)\n", - " m_sk, s_sk = est.predict(Xt, return_std=True)\n", - " suite.check_true(\"KrigingRegressor.predict returns 1-D arrays (scikit-learn convention)\",\n", - " np.ndim(m_sk) == 1 and np.ndim(s_sk) == 1, detail=f\"shapes {np.shape(m_sk)}, {np.shape(s_sk)}\",\n", - " severity=\"warn\")\n", - " m_sk, s_sk = np.ravel(m_sk), np.ravel(s_sk)\n", - " m_nat, s_nat = mean_sd(lk.Kriging(y, X, \"matern5_2\"), Xt)\n", - " suite.check_close(\"KrigingRegressor == Kriging (mean)\", m_sk, m_nat, rtol=1e-8, atol=1e-10)\n", - " suite.check_close(\"KrigingRegressor == Kriging (stdev)\", s_sk, s_nat, rtol=1e-6, atol=1e-10)\n", - " est2 = clone(est).set_params(kernel=\"gauss\")\n", - " suite.check_true(\"clone + set_params\", est2.get_params()[\"kernel\"] == \"gauss\" and est.get_params()[\"kernel\"] ==\n", - " \"matern5_2\")\n", - " with suite.check_runs(\"Pipeline(MinMaxScaler, KrigingRegressor)\"):\n", - " pipe = make_pipeline(MinMaxScaler(), KrigingRegressor(kernel=\"matern5_2\")).fit(tf.scale(X), y)\n", - " suite.check_greater(\"pipeline Q2 on native inputs\", pipe.score(tf.scale(Xt), yt), 0.8)\n", - " with suite.check_runs(\"GridSearchCV over kernels\"):\n", - " gs = GridSearchCV(KrigingRegressor(), {\"kernel\": [\"gauss\", \"matern3_2\", \"matern5_2\"]}, cv=3).fit(X, y)\n", - " suite.check_true(\"GridSearchCV picked a kernel\", gs.best_params_[\"kernel\"] in (\"gauss\", \"matern3_2\", \"matern5_2\"),\n", - " detail=str(gs.best_params_))\n", - " cv = pd.DataFrame(gs.cv_results_)[[\"param_kernel\", \"mean_test_score\", \"std_test_score\", \"mean_fit_time\"]]\n", - " for _, r in cv.iterrows():\n", - " suite.metric(\"hartmann3/GridSearchCV\", str(r[\"param_kernel\"]), cv_r2=r[\"mean_test_score\"],\n", - " cv_r2_sd=r[\"std_test_score\"], fit_s=r[\"mean_fit_time\"])\n", - " plt.figure(figsize=(5, 3.2))\n", - " plt.bar(cv[\"param_kernel\"].astype(str), cv[\"mean_test_score\"], yerr=cv[\"std_test_score\"], capsize=4)\n", - " plt.ylabel(\"3-fold CV R²\"); plt.title(f\"GridSearchCV(KrigingRegressor): best = {gs.best_params_['kernel']}\")\n", - " plt.ylim(min(0, cv[\"mean_test_score\"].min() - .05), 1.02); plt.show()\n", - " with suite.check_runs(\"cross_val_score\"):\n", - " sc = cross_val_score(KrigingRegressor(kernel=\"matern5_2\"), X, y, cv=4)\n", - " suite.check_finite(\"cross_val_score finite\", sc)\n", - " with suite.check_runs(\"sample_y\"):\n", - " S = est.sample_y(Xt[:10], n_samples=5, random_state=1)\n", - " suite.check_true(\"sample_y shape (q, n_samples)\", np.shape(S) == (10, 5), detail=str(np.shape(S)))\n", - "else:\n", - " suite.check_skip(\"sklearn estimators\", \"pylibkriging.sklearn or scikit-learn not available\")" - ] - }, - { - "cell_type": "markdown", - "id": "c007", - "metadata": {}, - "source": [ - "## Summary\n", - "\n", - "The metrics of this notebook, the number of checks per status, and every check that\n", - "did not pass (FAIL: error-level, fails the run; WARN: warn-level; KNOWN: listed in\n", - "`tools/known_issues.csv`, with its issue; SKIP: feature absent from the installed version).\n", - "All of them are also written to `results/sklearn-estimators_Kriging_python.checks.csv` and `.metrics.csv`." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c008", - "metadata": {}, - "outputs": [], - "source": [ - "suite.summary()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c009", - "metadata": {}, - "outputs": [], - "source": [ - "suite.finalize()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/tools/src/aggregations-large-n_NestedKriging_python.py b/src/aggregations-large-n_NestedKriging_python.py similarity index 98% rename from tools/src/aggregations-large-n_NestedKriging_python.py rename to src/aggregations-large-n_NestedKriging_python.py index fad25de..abdaa93 100644 --- a/tools/src/aggregations-large-n_NestedKriging_python.py +++ b/src/aggregations-large-n_NestedKriging_python.py @@ -8,6 +8,9 @@ # **Reference.** The exact predictor at the same θ (limit: 1 group), and the exact # fit. # +# **Tags.** class: NestedKriging · aggregation: NK, PoE, gPoE, BCM, rBCM · +# topic: large n +# # `NestedKriging` partitions the design into groups, fits one `Kriging` # sub-model per group (common hyperparameters), and aggregates the sub-model # predictions: `NK` (nested kriging: kriging of the sub-predictors, a proper diff --git a/tools/src/cross-binding-reference_all-classes_python.py b/src/cross-binding-reference_all-classes_python.py similarity index 97% rename from tools/src/cross-binding-reference_all-classes_python.py rename to src/cross-binding-reference_all-classes_python.py index bc8187d..c9363f0 100644 --- a/tools/src/cross-binding-reference_all-classes_python.py +++ b/src/cross-binding-reference_all-classes_python.py @@ -8,6 +8,10 @@ # **Reference.** The R, Octave and Julia notebooks `cross-binding_all-classes_*`, # compared by `tools/report.py`. # +# **Tags.** class: Kriging, NestedKriging, WarpKriging, MLPKriging · kernel: matern5_2 · +# objective: LL · method: exact, vecchia, nystrom · noise: none, nugget, variances · +# warping: kumaraswamy · aggregation: NK · topic: cross-binding +# # All bindings (Python, R, Octave, Julia) wrap the same C++ library: on the # same data with the same options they must return the same numbers. Each # language runs the protocol below on the shared datasets of `data/` and writes diff --git a/tools/src/cross-binding_all-classes_R.R b/src/cross-binding_all-classes_R.R similarity index 95% rename from tools/src/cross-binding_all-classes_R.R rename to src/cross-binding_all-classes_R.R index 15b7532..a02300d 100644 --- a/tools/src/cross-binding_all-classes_R.R +++ b/src/cross-binding_all-classes_R.R @@ -7,6 +7,10 @@ # **Reference.** The Python values (`cross-binding-reference_all- # classes_python.ipynb`), compared by `tools/report.py`. # +# **Tags.** class: Kriging, NestedKriging, WarpKriging, MLPKriging · kernel: matern5_2 · +# objective: LL · method: exact, vecchia, nystrom · noise: none, nugget, variances · +# warping: kumaraswamy · aggregation: NK · topic: cross-binding +# # Same protocol as `cross-binding-reference_all-classes_python.ipynb` (see there), on the # shared datasets of `data/`; writes `results/xbind_R.csv`, compared with the # Python values by `tools/report.py`. diff --git a/tools/src/cross-binding_all-classes_julia.jl b/src/cross-binding_all-classes_julia.jl similarity index 95% rename from tools/src/cross-binding_all-classes_julia.jl rename to src/cross-binding_all-classes_julia.jl index 94a8d41..7a38904 100644 --- a/tools/src/cross-binding_all-classes_julia.jl +++ b/src/cross-binding_all-classes_julia.jl @@ -7,6 +7,10 @@ # **Reference.** The Python values (`cross-binding-reference_all- # classes_python.ipynb`), compared by `tools/report.py`. # +# **Tags.** class: Kriging, NestedKriging, WarpKriging, MLPKriging · kernel: matern5_2 · +# objective: LL · method: exact, vecchia, nystrom · noise: none, nugget, variances · +# warping: kumaraswamy · aggregation: NK · topic: cross-binding +# # Same protocol as `cross-binding-reference_all-classes_python.ipynb` (see there), on # the shared datasets of `data/`; writes `results/xbind_julia.csv`, compared with # the Python values by `tools/report.py`. diff --git a/tools/src/cross-binding_all-classes_octave.m b/src/cross-binding_all-classes_octave.m similarity index 95% rename from tools/src/cross-binding_all-classes_octave.m rename to src/cross-binding_all-classes_octave.m index cc57e2d..8cd587b 100644 --- a/tools/src/cross-binding_all-classes_octave.m +++ b/src/cross-binding_all-classes_octave.m @@ -7,6 +7,10 @@ % **Reference.** The Python values (`cross-binding-reference_all- % classes_python.ipynb`), compared by `tools/report.py`. % +% **Tags.** class: Kriging, NestedKriging, WarpKriging, MLPKriging · kernel: matern5_2 · +% objective: LL · method: exact, vecchia, nystrom · noise: none, nugget, variances · +% warping: kumaraswamy · aggregation: NK · topic: cross-binding +% % Same protocol as `cross-binding-reference_all-classes_python.ipynb` (see there), on the % shared datasets of `data/`; writes `results/xbind_octave.csv`, compared with % the Python values by `tools/report.py`. mLibKriging must be on the path diff --git a/tools/src/deep-kernel-projected-branin_MLPKriging_python.py b/src/deep-kernel-projected-branin_MLPKriging_python.py similarity index 97% rename from tools/src/deep-kernel-projected-branin_MLPKriging_python.py rename to src/deep-kernel-projected-branin_MLPKriging_python.py index 79e7093..a0bfa89 100644 --- a/tools/src/deep-kernel-projected-branin_MLPKriging_python.py +++ b/src/deep-kernel-projected-branin_MLPKriging_python.py @@ -7,6 +7,9 @@ # # **Reference.** Plain `Kriging` on the same data. # +# **Tags.** class: MLPKriging, Kriging · kernel: matern5_2 · +# topic: deep kernel, dimension reduction +# # f(x) = Branin(Wx) for x ∈ [0,1]¹⁰ and a fixed 2 × 10 projection W: the # function only varies in a 2D subspace, which a joint feature map (an MLP from # ℝ¹⁰ to ℝ², then a stationary kernel) can find, whereas a per-axis kernel diff --git a/tools/src/high-dimension-normalize-screening_Kriging_python.py b/src/high-dimension-normalize-screening_Kriging_python.py similarity index 97% rename from tools/src/high-dimension-normalize-screening_Kriging_python.py rename to src/high-dimension-normalize-screening_Kriging_python.py index 22713f2..94b25fb 100644 --- a/tools/src/high-dimension-normalize-screening_Kriging_python.py +++ b/src/high-dimension-normalize-screening_Kriging_python.py @@ -8,6 +8,9 @@ # **Reference.** The same model on inputs rescaled by hand, and the known active # inputs of the Welch function. # +# **Tags.** class: Kriging · kernel: matern5_2 · +# topic: high dimension, normalize, screening +# # | function | d | character | # |---|---|---| # | OTL circuit | 6 | smooth, inputs on very different scales | diff --git a/tools/src/kernels-trends-interpolation_Kriging_python.py b/src/kernels-trends-interpolation_Kriging_python.py similarity index 98% rename from tools/src/kernels-trends-interpolation_Kriging_python.py rename to src/kernels-trends-interpolation_Kriging_python.py index 94a55c7..ccb9784 100644 --- a/tools/src/kernels-trends-interpolation_Kriging_python.py +++ b/src/kernels-trends-interpolation_Kriging_python.py @@ -9,6 +9,9 @@ # **Reference.** Finite differences of the prediction, the data themselves, and per- # function Q² floors. # +# **Tags.** class: Kriging · kernel: gauss, exp, matern3_2, matern5_2 · +# trend: constant, linear, interactive, quadratic · topic: interpolation, derivatives +# # Plain noise-free `Kriging`, every kernel (`gauss`, `exp`, `matern3_2`, # `matern5_2`) crossed with every trend (`constant`, `linear`, `interactive`, # `quadratic`) on functions of different regularity: diff --git a/tools/src/large-n_Kriging-vecchia-nystrom_python.py b/src/large-n_Kriging-vecchia-nystrom_python.py similarity index 99% rename from tools/src/large-n_Kriging-vecchia-nystrom_python.py rename to src/large-n_Kriging-vecchia-nystrom_python.py index d60b9e9..0098dd2 100644 --- a/tools/src/large-n_Kriging-vecchia-nystrom_python.py +++ b/src/large-n_Kriging-vecchia-nystrom_python.py @@ -8,6 +8,8 @@ # **Reference.** The exact model at the same θ (limits m = n − 1 and k = n), and the # exact fit. # +# **Tags.** class: Kriging · method: exact, vecchia, nystrom · topic: large n +# # The exact fit costs O(n³). `Kriging` offers: # # * `method="vecchia(m)"`: Vecchia likelihood with m conditioning neighbours diff --git a/tools/src/noisy-data_Kriging-nugget-noise_python.py b/src/noisy-data_Kriging-nugget-noise_python.py similarity index 98% rename from tools/src/noisy-data_Kriging-nugget-noise_python.py rename to src/noisy-data_Kriging-nugget-noise_python.py index 54419f3..8c30625 100644 --- a/tools/src/noisy-data_Kriging-nugget-noise_python.py +++ b/src/noisy-data_Kriging-nugget-noise_python.py @@ -8,6 +8,9 @@ # **Reference.** The true noise variance, replicate averaging, and the interpolating # (noise-free) fit on the same data. # +# **Tags.** class: Kriging · noise: none, nugget, variances · +# topic: noisy data, calibration +# # `Kriging` handles noisy responses through its `noise` argument: `"nugget"` # estimates one homogeneous noise variance, a vector gives a known variance per # observation (heteroscedastic case). diff --git a/tools/src/objectives-LL-LOO-LMP-gradients_Kriging_python.py b/src/objectives-LL-LOO-LMP-gradients_Kriging_python.py similarity index 99% rename from tools/src/objectives-LL-LOO-LMP-gradients_Kriging_python.py rename to src/objectives-LL-LOO-LMP-gradients_Kriging_python.py index 50cf1e8..626ef6e 100644 --- a/tools/src/objectives-LL-LOO-LMP-gradients_Kriging_python.py +++ b/src/objectives-LL-LOO-LMP-gradients_Kriging_python.py @@ -8,6 +8,8 @@ # **Reference.** Finite differences of each criterion, explicit leave-one-out refits, # and the criterion profile around θ̂. # +# **Tags.** class: Kriging · objective: LL, LOO, LMP · topic: gradients, optimization +# # The hyperparameters θ of `Kriging` are estimated by maximizing the # log-likelihood (`LL`), maximizing the log marginal posterior (`LMP`), or # minimizing the leave-one-out error (`LOO`). This notebook checks the machinery diff --git a/tools/src/robustness-invalid-inputs_Kriging_python.py b/src/robustness-invalid-inputs_Kriging_python.py similarity index 99% rename from tools/src/robustness-invalid-inputs_Kriging_python.py rename to src/robustness-invalid-inputs_Kriging_python.py index 50102ff..fee0a9e 100644 --- a/tools/src/robustness-invalid-inputs_Kriging_python.py +++ b/src/robustness-invalid-inputs_Kriging_python.py @@ -8,6 +8,8 @@ # **Reference.** Equivariance identities (rescaled inputs / outputs give rescaled # results) and the O(q) cost of `predict`. # +# **Tags.** class: Kriging · topic: robustness, invariance, cost +# # What a user eventually feeds a surrogate model, and what must happen: # # | input | expected behaviour | diff --git a/tools/src/save-load-pickle-copy-accessors_Kriging_python.py b/src/save-load-pickle-copy-accessors_Kriging_python.py similarity index 97% rename from tools/src/save-load-pickle-copy-accessors_Kriging_python.py rename to src/save-load-pickle-copy-accessors_Kriging_python.py index b1c1a22..e6d6d04 100644 --- a/tools/src/save-load-pickle-copy-accessors_Kriging_python.py +++ b/src/save-load-pickle-copy-accessors_Kriging_python.py @@ -8,6 +8,9 @@ # **Reference.** The original model: predictions and hyperparameters must be identical # to the last bit (1e-12). # +# **Tags.** class: Kriging · objective: LL, LOO, LMP · method: exact, vecchia, nystrom · +# noise: none, nugget, variances · topic: persistence +# # Persistence and introspection of `Kriging`, for each of its variants (noise # models, objectives, approximate fit methods): # diff --git a/tools/src/sklearn-estimators_Kriging_python.py b/src/sklearn-estimators_Kriging_python.py similarity index 98% rename from tools/src/sklearn-estimators_Kriging_python.py rename to src/sklearn-estimators_Kriging_python.py index 98978c5..2e58c37 100644 --- a/tools/src/sklearn-estimators_Kriging_python.py +++ b/src/sklearn-estimators_Kriging_python.py @@ -7,6 +7,8 @@ # **Reference.** The native `Kriging` class on the same data (same predictions), and # scikit-learn's estimator conventions. # +# **Tags.** class: Kriging · compared to: scikit-learn · topic: scikit-learn API +# # `KrigingRegressor` (the scikit-learn face of `Kriging`) on Hartmann 3: # `fit`/`predict` (scikit-learn conventions), the same predictions as the native # class, `get_params`/`set_params`/`clone`, use inside `Pipeline`, diff --git a/tools/src/test-functions-reference-values_no-model_python.py b/src/test-functions-reference-values_no-model_python.py similarity index 98% rename from tools/src/test-functions-reference-values_no-model_python.py rename to src/test-functions-reference-values_no-model_python.py index 5651a88..3515b58 100644 --- a/tools/src/test-functions-reference-values_no-model_python.py +++ b/src/test-functions-reference-values_no-model_python.py @@ -8,6 +8,8 @@ # **Reference.** Published optima and nominal values (Virtual Library of Simulation # Experiments, Surjanovic & Bingham). # +# **Tags.** class: none · topic: test functions +# # Every notebook of this gallery relies on the test functions of # `gallery.functions` (after the [Virtual Library of Simulation Experiments](https://www.sfu.ca/~ssurjano/)). # A typo in one of them would silently bias every result built on it, so they are diff --git a/tools/src/update-simulate_Kriging_python.py b/src/update-simulate_Kriging_python.py similarity index 99% rename from tools/src/update-simulate_Kriging_python.py rename to src/update-simulate_Kriging_python.py index 2221c0b..80d244a 100644 --- a/tools/src/update-simulate_Kriging_python.py +++ b/src/update-simulate_Kriging_python.py @@ -7,6 +7,8 @@ # **Reference.** A model fitted from scratch on all the data, and Monte Carlo moments # against the predictive mean and covariance. # +# **Tags.** class: Kriging · topic: update, conditional simulation +# # Sequential use of a fitted model (sequential design, Bayesian optimization): # # * `update(y_u, X_u, refit=False)` adds observations through a Cholesky diff --git a/tools/src/vs-DiceKriging-RobustGaSP_Kriging_R.R b/src/vs-DiceKriging-RobustGaSP_Kriging_R.R similarity index 98% rename from tools/src/vs-DiceKriging-RobustGaSP_Kriging_R.R rename to src/vs-DiceKriging-RobustGaSP_Kriging_R.R index 4270d91..8e2a700 100644 --- a/tools/src/vs-DiceKriging-RobustGaSP_Kriging_R.R +++ b/src/vs-DiceKriging-RobustGaSP_Kriging_R.R @@ -7,6 +7,9 @@ # # **Reference.** DiceKriging (exact equality) and RobustGaSP (accuracy, cost). # +# **Tags.** class: Kriging · kernel: gauss, exp, matern3_2, matern5_2 · +# compared to: DiceKriging, RobustGaSP +# # DiceKriging is the reference implementation libKriging's `Kriging` follows # (same product-form kernels and range parametrization, same concentrated # likelihood). Hence two levels of comparison: diff --git a/tools/src/vs-GaussianProcesses_Kriging_julia.jl b/src/vs-GaussianProcesses_Kriging_julia.jl similarity index 97% rename from tools/src/vs-GaussianProcesses_Kriging_julia.jl rename to src/vs-GaussianProcesses_Kriging_julia.jl index 288fdf6..46c8288 100644 --- a/tools/src/vs-GaussianProcesses_Kriging_julia.jl +++ b/src/vs-GaussianProcesses_Kriging_julia.jl @@ -6,6 +6,8 @@ # # **Reference.** GaussianProcesses.jl (accuracy and cost). # +# **Tags.** class: Kriging · kernel: matern5_2 · compared to: GaussianProcesses.jl +# # Default fits on the shared datasets of `data/` (Branin, Hartmann 3, Borehole): # JLibKriging (`matern5_2`, constant trend, single start and 10 restarts) against # [GaussianProcesses.jl](https://github.com/STOR-i/GaussianProcesses.jl) diff --git a/tools/src/vs-STK_Kriging_octave.m b/src/vs-STK_Kriging_octave.m similarity index 98% rename from tools/src/vs-STK_Kriging_octave.m rename to src/vs-STK_Kriging_octave.m index 0f30ba5..ae87951 100644 --- a/tools/src/vs-STK_Kriging_octave.m +++ b/src/vs-STK_Kriging_octave.m @@ -7,6 +7,8 @@ % % **Reference.** STK (exact equality, then accuracy and cost). % +% **Tags.** class: Kriging · kernel: gauss, matern5_2 · compared to: STK +% % [STK](https://github.com/stk-kriging/stk) (Small Toolbox for Kriging) is the % reference kriging toolbox for Octave/MATLAB. % diff --git a/tools/src/vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python.py b/src/vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python.py similarity index 99% rename from tools/src/vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python.py rename to src/vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python.py index f879eb1..eca82a5 100644 --- a/tools/src/vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python.py +++ b/src/vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python.py @@ -7,6 +7,9 @@ # # **Reference.** Scikit-learn, SMT, OpenTURNS and GPy. # +# **Tags.** class: Kriging · kernel: gauss, matern5_2 · +# compared to: scikit-learn, SMT, OpenTURNS, GPy +# # Two kinds of comparisons, which should not be confused: # # 1. **Same model, same hyperparameters → same numbers.** With θ, σ², β fixed, diff --git a/tools/src/warpings-categorical-ordinal_WarpKriging_python.py b/src/warpings-categorical-ordinal_WarpKriging_python.py similarity index 98% rename from tools/src/warpings-categorical-ordinal_WarpKriging_python.py rename to src/warpings-categorical-ordinal_WarpKriging_python.py index c954d90..8c1080c 100644 --- a/tools/src/warpings-categorical-ordinal_WarpKriging_python.py +++ b/src/warpings-categorical-ordinal_WarpKriging_python.py @@ -8,6 +8,10 @@ # **Reference.** Plain `Kriging` on the same data (accuracy and NLPD), and `none` # warping = `Kriging`. # +# **Tags.** class: WarpKriging, Kriging · +# warping: none, affine, boxcox, kumaraswamy, neural_mono, categorical, ordinal · +# topic: non-stationarity, categorical inputs +# # `WarpKriging` learns a transform per input variable jointly with the GP # hyperparameters. Problems where a plain stationary GP is the wrong model: # diff --git a/test-functions-reference-values_no-model_python.ipynb b/test-functions-reference-values_no-model_python.ipynb deleted file mode 100644 index 99997be..0000000 --- a/test-functions-reference-values_no-model_python.ipynb +++ /dev/null @@ -1,233 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "c000", - "metadata": {}, - "source": [ - "# Python · no model — the test functions against their published reference values\n", - "\n", - "**Idea.** Every example of this gallery is only as good as its test functions:\n", - "before fitting anything, each of the 21 functions is evaluated at points where its\n", - "value is published, and its range and shape are plotted.\n", - "\n", - "**Reference.** Published optima and nominal values (Virtual Library of Simulation\n", - "Experiments, Surjanovic & Bingham).\n", - "\n", - "Every notebook of this gallery relies on the test functions of\n", - "`gallery.functions` (after the [Virtual Library of Simulation Experiments](https://www.sfu.ca/~ssurjano/)).\n", - "A typo in one of them would silently bias every result built on it, so they are\n", - "checked first, against the reference values published on the library pages\n", - "(global optima, nominal points) and against the known output ranges." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c001", - "metadata": {}, - "outputs": [], - "source": [ - "# Setup for running this notebook outside the repository (Colab, Binder, a lone copy):\n", - "# nothing to do in the repository itself (CI, local runs). Otherwise the repository\n", - "# (datasets of data/, helpers of tools/python/gallery.py) is cloned and the released pylibkriging installed.\n", - "import importlib\n", - "import os\n", - "import subprocess\n", - "import sys\n", - "from pathlib import Path\n", - "\n", - "if not Path(\"tools/known_issues.csv\").is_file():\n", - " if not Path(\"gallery/tools/known_issues.csv\").is_file():\n", - " subprocess.run([\"git\", \"clone\", \"--depth\", \"1\", \"https://github.com/libKriging/gallery\"], check=True)\n", - " os.chdir(\"gallery\")\n", - " try:\n", - " import pylibkriging # noqa: F401\n", - " except ImportError:\n", - " r = subprocess.run([sys.executable, \"-m\", \"pip\", \"install\", \"--quiet\", \"pylibkriging\"],\n", - " capture_output=True, text=True)\n", - " if r.returncode != 0:\n", - " raise RuntimeError(\n", - " f\"pylibkriging could not be installed for Python {sys.version_info.major}.{sys.version_info.minor} \"\n", - " \"(PyPI has wheels for CPython 3.10 to 3.12 only). On Colab: Runtime > Change runtime version > \"\n", - " \"2026.07 (Python 3.12), then run again.\\n\" + r.stderr[-500:])\n", - " importlib.invalidate_caches()\n", - "sys.path.insert(0, str(Path(\"tools/python\").resolve())) # for \"import gallery\" (tools/python/gallery.py)\n", - "print(\"working directory:\", os.getcwd())" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c002", - "metadata": {}, - "outputs": [], - "source": [ - "import numpy as np\n", - "import pandas as pd\n", - "import matplotlib.pyplot as plt\n", - "\n", - "import gallery\n", - "from gallery import TESTS, design\n", - "\n", - "suite = gallery.Suite(\"test-functions-reference-values_no-model_python\")" - ] - }, - { - "cell_type": "markdown", - "id": "c003", - "metadata": {}, - "source": [ - "## Reference values" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c004", - "metadata": {}, - "outputs": [], - "source": [ - "rows = []\n", - "for name, tf in TESTS.items():\n", - " for x, v in tf.known:\n", - " fx = float(tf(x)[0])\n", - " suite.check_close(f\"{name}: f({np.round(x, 4).tolist()})\", fx, v, rtol=1e-4, atol=1e-4)\n", - " rows.append((name, tf.d, np.round(x, 4).tolist(), v, fx))\n", - "pd.DataFrame(rows, columns=[\"function\", \"d\", \"x\", \"reference\", \"computed\"])" - ] - }, - { - "cell_type": "markdown", - "id": "c005", - "metadata": {}, - "source": [ - "## Output ranges\n", - "\n", - "For the engineering functions without a closed-form optimum, the range over the\n", - "domain is compared with the one reported in the literature (loose bounds: a\n", - "random sample does not reach the extremes)." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c006", - "metadata": {}, - "outputs": [], - "source": [ - "EXPECTED_RANGES = { # (lower bound of min, upper bound of max) over the domain\n", - " \"otl_circuit\": (2.5, 10.0),\n", - " \"piston\": (0.1, 1.3),\n", - " \"borehole\": (2.0, 320.0),\n", - " \"wing_weight\": (120.0, 480.0),\n", - " \"currin_exp\": (1.0, 14.0),\n", - " \"dette_pepelyshev\": (0.0, 100.0),\n", - "}\n", - "summary = []\n", - "for name, tf in TESTS.items():\n", - " U = design.uniform(20_000, tf.d, seed=1)\n", - " y = tf.unit(U)\n", - " suite.check_finite(f\"{name}: finite on the domain\", y)\n", - " if name in EXPECTED_RANGES:\n", - " lo, hi = EXPECTED_RANGES[name]\n", - " suite.check_true(f\"{name}: range within [{lo}, {hi}]\", lo <= y.min() and y.max() <= hi,\n", - " detail=f\"sampled range [{y.min():.4g}, {y.max():.4g}]\")\n", - " summary.append((name, tf.d, \",\".join(tf.tags), y.min(), y.max(), y.std()))\n", - "pd.DataFrame(summary, columns=[\"function\", \"d\", \"tags\", \"min\", \"max\", \"std\"])" - ] - }, - { - "cell_type": "markdown", - "id": "c007", - "metadata": {}, - "source": [ - "## Designs\n", - "\n", - "The designs are seeded: the same seed gives the same points (reproducibility of\n", - "every notebook), and the optimized LHS is a Latin hypercube (one point per\n", - "stratum in each dimension)." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c008", - "metadata": {}, - "outputs": [], - "source": [ - "A, B = design.lhs(20, 3, seed=7), design.lhs(20, 3, seed=7)\n", - "suite.check_close(\"lhs: reproducible with a seed\", A, B, rtol=0, atol=0)\n", - "strata = np.floor(A * 20).astype(int)\n", - "suite.check_true(\"lhs: one point per stratum\", all(len(set(strata[:, j])) == 20 for j in range(3)))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c009", - "metadata": {}, - "outputs": [], - "source": [ - "fig, axes = plt.subplots(1, 4, figsize=(16, 3.5))\n", - "for ax, name in zip(axes, [\"forrester\", \"gramacy_lee_2012\"]):\n", - " tf = TESTS[name]\n", - " u = np.linspace(0, 1, 500)[:, None]\n", - " ax.plot(tf.scale(u)[:, 0], tf.unit(u))\n", - " ax.set_title(name)\n", - "for ax, name in zip(axes[2:], [\"branin\", \"gramacy_lee_2008\"]):\n", - " tf = TESTS[name]\n", - " g = np.linspace(0, 1, 100)\n", - " G = np.array(np.meshgrid(g, g)).reshape(2, -1).T\n", - " ax.contourf(g, g, tf.unit(G).reshape(100, 100), 30)\n", - " ax.set_title(name)\n", - "plt.tight_layout()" - ] - }, - { - "cell_type": "markdown", - "id": "c010", - "metadata": {}, - "source": [ - "## Summary\n", - "\n", - "The metrics of this notebook, the number of checks per status, and every check that\n", - "did not pass (FAIL: error-level, fails the run; WARN: warn-level; KNOWN: listed in\n", - "`tools/known_issues.csv`, with its issue; SKIP: feature absent from the installed version).\n", - "All of them are also written to `results/test-functions-reference-values_no-model_python.checks.csv` and `.metrics.csv`." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c011", - "metadata": {}, - "outputs": [], - "source": [ - "suite.summary()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c012", - "metadata": {}, - "outputs": [], - "source": [ - "suite.finalize()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/tools/py2nb.py b/tools/py2nb.py index 6c27499..b414ade 100644 --- a/tools/py2nb.py +++ b/tools/py2nb.py @@ -1,13 +1,20 @@ #!/usr/bin/env python3 """Convert a "percent" script (cells separated by ``# %%`` / ``# %% [markdown]``, or -``% %%`` in Octave) into a notebook without outputs. The notebooks of this repository -are written this way (sources in ``tools/src/``) so that they review as plain diffs; -the ``.ipynb`` files, at the repository root, are generated, and CI checks that they -are in sync (and that no notebook lacks its source). Every Python notebook gets the -cell of ``tools/notebook_setup.py`` after its title, to run on Colab or Binder too. - -Usage: python tools/py2nb.py [--check] [SRC...] (default: every tools/src/* file) - each ``tools/src/.`` gives ``.ipynb`` at the repository root. +``% %%`` in Octave) into a notebook without outputs. The notebooks of this gallery are +written this way, in ``src/``, so that they review as plain diffs; the ``.ipynb`` files +are not committed: CI generates and executes them, and publishes the executed ones (see +``tools/site.py``). Every Python notebook gets the cell of ``tools/notebook_setup.py`` +after its title, to run on Colab or Binder too. + +Every source is named ``__.`` and starts with a +markdown cell holding its title (``# · — ``), an ``**Idea.**`` +paragraph, a ``**Reference.**`` paragraph and a ``**Tags.**`` paragraph +(``key: value, value · key: value``, keys from ``TAG_KEYS``); the suite it writes +(``Suite("")``) is named after the file. ``--check`` validates all that; the title, +idea and tags also go to the notebook metadata (``metadata.gallery``), for the site. + +Usage: python tools/py2nb.py [--check] [--out DIR] [SRC...] (default: every src/* file) + each ``src/.`` gives ``/.ipynb`` (default DIR: the repository root). """ import argparse import re @@ -17,7 +24,7 @@ import nbformat ROOT = Path(__file__).resolve().parents[1] -SRC = ROOT / "tools" / "src" +SRC = ROOT / "src" # code cell inserted after the title cell of every Python notebook, so that it also runs # outside the repository (Colab, Binder): a no-op in the repository itself SETUP = {".py": ROOT / "tools" / "notebook_setup.py"} @@ -32,6 +39,82 @@ ".m": ({"name": "octave", "display_name": "Octave", "language": "octave"}, {"name": "octave"}, "%"), } +BINDINGS = {".py": "python", ".R": "R", ".jl": "julia", ".m": "octave"} +LANGUAGES = {".py": "Python", ".R": "R", ".jl": "Julia", ".m": "Octave"} +# keys of the **Tags.** paragraph (all optional but class), in display order +TAG_KEYS = ("class", "kernel", "trend", "objective", "method", "noise", "warping", + "aggregation", "compared to", "topic") +SUITE = re.compile(r'\b(?:Suite|suite)\("([^"]+)"\)') + + +def sources(): + return sorted(f for f in SRC.iterdir() if f.suffix in KERNELS) + + +def paragraph(text, label): + """The paragraph starting with ``**