diff --git a/.binder/requirements.txt b/.binder/requirements.txt new file mode 100644 index 0000000..9969ea4 --- /dev/null +++ b/.binder/requirements.txt @@ -0,0 +1,10 @@ +# Binder (mybinder.org) environment for the Python notebooks: the released pylibkriging +# (PyPI wheels exist for CPython 3.10 to 3.12, hence runtime.txt) and the dependencies of +# tools/python/gallery.py, which the notebooks load from the repository; the +# comparison packages are installed by the notebooks that use them. +pylibkriging +numpy +scipy +pandas +matplotlib +jinja2 diff --git a/.binder/runtime.txt b/.binder/runtime.txt new file mode 100644 index 0000000..d2aca3a --- /dev/null +++ b/.binder/runtime.txt @@ -0,0 +1 @@ +python-3.12 diff --git a/.github/workflows/notebooks.yml b/.github/workflows/notebooks.yml new file mode 100644 index 0000000..7055c40 --- /dev/null +++ b/.github/workflows/notebooks.yml @@ -0,0 +1,541 @@ +name: notebooks + +# Generates every notebook from its source (src/, tools/py2nb.py) and executes it +# against libKriging, one job per notebook, each job uploading its executed notebook +# (outputs saved, failures included) and its results as the artifact +# `executed--`; then aggregates the checks of all languages into +# one report per channel (tools/report.py), published as the job summary and as +# artifacts. Finally builds the gallery site from the executed notebooks of the +# master channel (tools/site.py), published on the gh-pages branch from main. +# +# Channels: +# master libKriging and its bindings built from source at `libkriging_ref` +# (default: master) of `libkriging_repo`: Python wheel, R package +# (bindings/R), Octave mex, JLibKriging.jl, all from the same commit. +# The default for every event except the weekly schedule. +# release the last published packages: pylibkriging from PyPI, rlibkriging from +# CRAN, mLibKriging built from the last release tag (no Julia). A +# feature missing from the release is recorded as SKIP. Weekly, or on +# demand (`channels` input). +# +# A notebook job fails when one of its error-level checks fails; the report job +# fails when any notebook failed or crashed, or when the bindings disagree +# (cross-binding protocol, cross-binding-reference_all-classes_python.ipynb). +# The binding installations are the scripts of tools/install/, usable locally too. + +on: + push: + branches: [main] + pull_request: + schedule: + - cron: '17 3 * * 1' # weekly, Monday 03:17 UTC: master and release + workflow_dispatch: + inputs: + libkriging_ref: + description: 'libKriging branch, tag or commit for the master channel' + default: 'master' + libkriging_repo: + description: 'libKriging repository (owner/name), e.g. a fork' + default: 'libKriging/libKriging' + channels: + description: 'channels to run: "master", "release" or "master,release"' + default: 'master' + repository_dispatch: + # libKriging can trigger a run after a merge: + # POST /repos//dispatches {"event_type": "libkriging-updated", "client_payload": {"ref": ""}} + types: [libkriging-updated] + +permissions: + contents: read + +concurrency: + # per event too: a repository_dispatch on main does not cancel a push run on main + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +env: + LIBKRIGING_REPO: ${{ inputs.libkriging_repo || 'libKriging/libKriging' }} + LIBKRIGING_REF: ${{ inputs.libkriging_ref || github.event.client_payload.ref || 'master' }} + # cmake >= 4 rejects dependencies declaring cmake_minimum_required < 3.5 + CMAKE_POLICY_VERSION_MINIMUM: "3.5" + # NestedKriging's OpenMP loop can hang on 2-core runners (same workaround as libKriging's CI) + OMP_NUM_THREADS: 1 + # Same OpenBLAS kernels on every runner, whatever its CPU: the cross-binding checks + # compare jobs at 1e-8, and nystrom(10) moves by ~1e-8 (LL) / ~1e-4 (mean) between kernels. + # Haswell (AVX2) runs on all GitHub x64 runners. + OPENBLAS_CORETYPE: Haswell + PYTHON_VERSION: '3.12' + JULIA_VERSION: '1.11' + +jobs: + # --------------------------------------------------------------------------- + setup: + runs-on: ubuntu-24.04 + outputs: + sha: ${{ steps.refs.outputs.sha }} + release_tag: ${{ steps.refs.outputs.release_tag }} + channels: ${{ steps.refs.outputs.channels }} + python: ${{ steps.matrix.outputs.python }} + r: ${{ steps.matrix.outputs.r }} + octave: ${{ steps.matrix.outputs.octave }} + julia: ${{ steps.matrix.outputs.julia }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + - name: Notebook sources valid (name, title, idea, reference, tags), and listed in the README + run: | + pip install nbformat + python tools/py2nb.py --check + missing=$(for f in src/*; do grep -qF "(src/${f#src/})" README.md || echo "$f"; done) + if [ -n "$missing" ]; then echo "not linked from README.md: $missing"; exit 1; fi + - name: Resolve libKriging refs and channels + id: refs + env: + GH_TOKEN: ${{ github.token }} + CHANNELS: ${{ github.event_name == 'schedule' && 'master,release' || inputs.channels || 'master' }} + run: | + sha=$(git ls-remote "https://github.com/${LIBKRIGING_REPO}" "${LIBKRIGING_REF}" | head -1 | cut -f1) + if [ -z "$sha" ]; then # a commit, possibly abbreviated + sha=$(gh api "repos/${LIBKRIGING_REPO}/commits/${LIBKRIGING_REF}" --jq .sha) + fi + channels="$CHANNELS" + tag=$(gh api "repos/${LIBKRIGING_REPO}/releases/latest" --jq .tag_name 2>/dev/null || true) + if [ -z "$tag" ] && [[ "$channels" == *release* ]]; then + echo "::warning::${LIBKRIGING_REPO} has no release: release channel skipped" + channels=$(echo "$channels" | tr ',' '\n' | grep -vx release | paste -sd, -) + fi + { echo "sha=$sha"; echo "release_tag=$tag"; echo "channels=$channels"; } >> "$GITHUB_OUTPUT" + echo "libKriging ${LIBKRIGING_REPO}@${LIBKRIGING_REF} = $sha; last release ${tag:-none}; channels $channels" >> "$GITHUB_STEP_SUMMARY" + - name: One job per notebook and channel + id: matrix + env: + CHANNELS: ${{ steps.refs.outputs.channels }} + run: | + python - <<'EOF' >> "$GITHUB_OUTPUT" + import json, os + from pathlib import Path + channels = os.environ["CHANNELS"].split(",") + for lang, binding in [("python", "python"), ("r", "R"), ("octave", "octave"), ("julia", "julia")]: + chans = [c for c in channels if c == "master" or lang != "julia"] # no Julia release + include = [{"channel": c, "notebook": src.stem, "source": src.name} for c in chans + for src in sorted(Path("src").glob(f"*_{binding}.*"))] + print(f"{lang}={json.dumps({'include': include or [{'channel': 'none', 'notebook': 'none'}]})}") + EOF + cat "$GITHUB_OUTPUT" + + # --------------------------------------------------------------------------- builds + build-python: + needs: setup + if: contains(needs.setup.outputs.channels, 'master') + runs-on: ubuntu-24.04 + timeout-minutes: 120 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + - id: cache + uses: actions/cache@v4 + with: + path: dist + key: pylibkriging-${{ needs.setup.outputs.sha }}-py${{ env.PYTHON_VERSION }} + - if: steps.cache.outputs.cache-hit != 'true' + uses: actions/checkout@v4 + with: + repository: ${{ env.LIBKRIGING_REPO }} + ref: ${{ needs.setup.outputs.sha }} + submodules: recursive + path: libKriging + - name: Build the pylibkriging wheel + if: steps.cache.outputs.cache-hit != 'true' + run: | + sudo apt-get update && sudo apt-get install -y liblapack-dev libopenblas-dev + tools/install/python.sh wheel libKriging dist + - uses: actions/upload-artifact@v4 + with: + name: build-python-master + path: dist/*.whl + + build-r: + needs: setup + if: contains(needs.setup.outputs.channels, 'master') + runs-on: ubuntu-24.04 + timeout-minutes: 120 + steps: + - uses: actions/checkout@v4 + - uses: actions/checkout@v4 + with: + repository: ${{ env.LIBKRIGING_REPO }} + ref: ${{ needs.setup.outputs.sha }} + submodules: recursive + path: libKriging + - uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + - uses: r-lib/actions/setup-r@v2 + with: + use-public-rspm: true + - name: Build rlibkriging from libKriging's bindings/R + run: | + sudo apt-get update && sudo apt-get install -y liblapack-dev libopenblas-dev gfortran + tools/install/r.sh master "$RUNNER_TEMP/Rlib" libKriging + tar -C "$RUNNER_TEMP" -cf Rlib.tar Rlib + - uses: actions/upload-artifact@v4 + with: + name: build-r-master + path: Rlib.tar + + build-octave: + needs: setup + runs-on: ubuntu-24.04 + timeout-minutes: 120 + strategy: + matrix: + channel: [master, release] + steps: + - name: Channel enabled? + id: on + run: echo 'enabled=${{ contains(needs.setup.outputs.channels, matrix.channel) }}' >> "$GITHUB_OUTPUT" + - if: steps.on.outputs.enabled == 'true' + uses: actions/checkout@v4 + - if: steps.on.outputs.enabled == 'true' + id: cache + uses: actions/cache@v4 + with: + path: mlibkriging + key: mlibkriging-${{ matrix.channel == 'master' && needs.setup.outputs.sha || needs.setup.outputs.release_tag }}-ubuntu-24.04-octave + - if: steps.on.outputs.enabled == 'true' && steps.cache.outputs.cache-hit != 'true' + uses: actions/checkout@v4 + with: + repository: ${{ env.LIBKRIGING_REPO }} + ref: ${{ matrix.channel == 'master' && needs.setup.outputs.sha || needs.setup.outputs.release_tag }} + submodules: recursive + path: libKriging + - if: steps.on.outputs.enabled == 'true' + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + - name: Build mLibKriging + if: steps.on.outputs.enabled == 'true' + run: | + sudo apt-get update && sudo apt-get install -y octave octave-dev liblapack-dev libopenblas-dev + tools/install/octave.sh "${{ steps.cache.outputs.cache-hit == 'true' && 'mlibkriging' || 'libKriging' }}" mlibkriging + - if: steps.on.outputs.enabled == 'true' + uses: actions/upload-artifact@v4 + with: + name: build-octave-${{ matrix.channel }} + path: mlibkriging/ + + build-julia: + needs: setup + if: contains(needs.setup.outputs.channels, 'master') + runs-on: ubuntu-24.04 + timeout-minutes: 120 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + - uses: julia-actions/setup-julia@v2 + with: + version: ${{ env.JULIA_VERSION }} + - id: cache + uses: actions/cache@v4 + with: + path: ~/.julia + key: julia-depot-${{ needs.setup.outputs.sha }}-julia${{ env.JULIA_VERSION }} + - if: steps.cache.outputs.cache-hit != 'true' + uses: actions/checkout@v4 + with: + repository: ${{ env.LIBKRIGING_REPO }} + ref: ${{ needs.setup.outputs.sha }} + submodules: recursive + path: libKriging + - name: Build JLibKriging.jl against the libKriging checkout + if: steps.cache.outputs.cache-hit != 'true' + run: | + sudo apt-get update && sudo apt-get install -y liblapack-dev libopenblas-dev + tools/install/julia.sh libKriging + + # --------------------------------------------------------------------------- notebooks + python: + needs: [setup, build-python] + if: always() && needs.setup.result == 'success' && !contains(needs.setup.outputs.python, '"none"') + runs-on: ubuntu-24.04 + timeout-minutes: 120 + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.setup.outputs.python) }} + name: ${{ matrix.notebook }} (${{ matrix.channel }}) + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + - if: matrix.channel == 'master' + uses: actions/download-artifact@v4 + with: + name: build-python-master + path: dist + - name: Install pylibkriging (${{ matrix.channel }}) + run: | + # the master wheel is linked against the system BLAS/LAPACK (libopenblas.so.0) + sudo apt-get update && sudo apt-get install -y liblapack-dev libopenblas-dev + tools/install/python.sh ${{ matrix.channel }} dist + - name: Generate the notebook from src/${{ matrix.source }} + run: | + python -m pip install --quiet nbformat + python tools/py2nb.py "src/${{ matrix.source }}" + - name: Execute and save the notebook + run: python tools/run_notebooks.py --inplace --timeout 3600 ${{ matrix.notebook }}.ipynb + - if: always() + uses: actions/upload-artifact@v4 + with: + name: executed-${{ matrix.channel }}-${{ matrix.notebook }} + path: | + ${{ matrix.notebook }}.ipynb + results/ + retention-days: 90 + + r: + needs: [setup, build-r] + if: always() && needs.setup.result == 'success' && !contains(needs.setup.outputs.r, '"none"') + runs-on: ubuntu-24.04 + timeout-minutes: 90 + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.setup.outputs.r) }} + name: ${{ matrix.notebook }} (${{ matrix.channel }}) + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + - uses: r-lib/actions/setup-r@v2 + with: + use-public-rspm: true + - if: matrix.channel == 'master' + uses: actions/download-artifact@v4 + with: + name: build-r-master + - name: Install rlibkriging (${{ matrix.channel }}) + run: | + sudo apt-get update && sudo apt-get install -y liblapack-dev libopenblas-dev + if [ "${{ matrix.channel }}" = master ]; then + tar -C "$RUNNER_TEMP" -xf Rlib.tar + echo "R_LIBS_USER=$RUNNER_TEMP/Rlib" >> "$GITHUB_ENV" + tools/install/r.sh kernel "$RUNNER_TEMP/Rlib" + else + tools/install/r.sh release "$RUNNER_TEMP/Rlib" + echo "R_LIBS_USER=$RUNNER_TEMP/Rlib" >> "$GITHUB_ENV" + fi + - name: Generate the notebook from src/${{ matrix.source }} + run: | + python -m pip install --quiet nbformat + python tools/py2nb.py "src/${{ matrix.source }}" + - name: Execute and save the notebook + run: python tools/run_notebooks.py --inplace --timeout 3600 ${{ matrix.notebook }}.ipynb + - if: always() + uses: actions/upload-artifact@v4 + with: + name: executed-${{ matrix.channel }}-${{ matrix.notebook }} + path: | + ${{ matrix.notebook }}.ipynb + results/ + retention-days: 90 + + octave: + needs: [setup, build-octave] + if: always() && needs.setup.result == 'success' && !contains(needs.setup.outputs.octave, '"none"') + runs-on: ubuntu-24.04 + timeout-minutes: 90 + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.setup.outputs.octave) }} + name: ${{ matrix.notebook }} (${{ matrix.channel }}) + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + - uses: actions/download-artifact@v4 + with: + name: build-octave-${{ matrix.channel }} + path: mlibkriging + - name: Install Octave and mLibKriging (${{ matrix.channel }}) + run: | + # octave-dev: mkoctfile, which STK's stk_init needs to compile its MEX files + sudo apt-get update && sudo apt-get install -y octave octave-dev liblapack-dev libopenblas-dev + tools/install/octave.sh mlibkriging mlibkriging # already built: only the kernel and a smoke test + echo "OCTAVE_PATH=$PWD/mlibkriging" >> "$GITHUB_ENV" + - name: Generate the notebook from src/${{ matrix.source }} + run: | + python -m pip install --quiet nbformat + python tools/py2nb.py "src/${{ matrix.source }}" + - name: Execute and save the notebook + run: python tools/run_notebooks.py --inplace --timeout 3600 ${{ matrix.notebook }}.ipynb + - if: always() + uses: actions/upload-artifact@v4 + with: + name: executed-${{ matrix.channel }}-${{ matrix.notebook }} + path: | + ${{ matrix.notebook }}.ipynb + results/ + retention-days: 90 + + julia: + needs: [setup, build-julia] + if: always() && needs.setup.result == 'success' && !contains(needs.setup.outputs.julia, '"none"') + runs-on: ubuntu-24.04 + timeout-minutes: 120 + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.setup.outputs.julia) }} + name: ${{ matrix.notebook }} (${{ matrix.channel }}) + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + - uses: julia-actions/setup-julia@v2 + with: + version: ${{ env.JULIA_VERSION }} + - id: cache + uses: actions/cache/restore@v4 + with: + path: ~/.julia + key: julia-depot-${{ needs.setup.outputs.sha }}-julia${{ env.JULIA_VERSION }} + - if: steps.cache.outputs.cache-hit != 'true' + uses: actions/checkout@v4 + with: + repository: ${{ env.LIBKRIGING_REPO }} + ref: ${{ needs.setup.outputs.sha }} + submodules: recursive + path: libKriging + - name: Install JLibKriging.jl (built by build-julia, or here on a cache miss) + run: | + sudo apt-get update && sudo apt-get install -y liblapack-dev libopenblas-dev + tools/install/julia.sh libKriging + - name: Generate the notebook from src/${{ matrix.source }} + run: | + python -m pip install --quiet nbformat + python tools/py2nb.py "src/${{ matrix.source }}" + - name: Execute and save the notebook + run: python tools/run_notebooks.py --inplace --timeout 3600 --kernel julia-${{ env.JULIA_VERSION }} ${{ matrix.notebook }}.ipynb + - if: always() + uses: actions/upload-artifact@v4 + with: + name: executed-${{ matrix.channel }}-${{ matrix.notebook }} + path: | + ${{ matrix.notebook }}.ipynb + results/ + retention-days: 90 + + # --------------------------------------------------------------------------- report + report: + needs: [setup, python, r, octave, julia] + if: always() && needs.setup.result == 'success' + runs-on: ubuntu-24.04 + strategy: + fail-fast: false + matrix: + channel: [master, release] + name: report ${{ matrix.channel }} + steps: + - name: Channel enabled? + id: on + run: echo 'enabled=${{ contains(needs.setup.outputs.channels, matrix.channel) }}' >> "$GITHUB_OUTPUT" + - if: steps.on.outputs.enabled == 'true' + uses: actions/checkout@v4 + - if: steps.on.outputs.enabled == 'true' + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + - if: steps.on.outputs.enabled == 'true' + uses: actions/download-artifact@v4 + with: + pattern: executed-${{ matrix.channel }}-* + path: artifacts + - name: Report + if: steps.on.outputs.enabled == 'true' + run: | + pip install numpy pandas + mkdir -p results executed + find artifacts -path '*/results/*' -type f -exec cp {} results/ \; + find artifacts -maxdepth 2 -name '*.ipynb' -exec cp {} executed/ \; + expect=python,R,octave + if [ "${{ matrix.channel }}" = master ]; then expect="$expect,julia"; fi + echo "## Channel: ${{ matrix.channel }}" >> "$GITHUB_STEP_SUMMARY" + python tools/report.py --results results --expect "$expect" \ + --baseline tools/baseline/metrics-${{ matrix.channel }}.csv \ + --write-baseline new-baseline/metrics-${{ matrix.channel }}.csv \ + --summary "$GITHUB_STEP_SUMMARY" > report.md + - if: always() && steps.on.outputs.enabled == 'true' + uses: actions/upload-artifact@v4 + with: + name: report-${{ matrix.channel }} + path: | + report.md + results/ + new-baseline/ + retention-days: 90 + - if: always() && steps.on.outputs.enabled == 'true' + uses: actions/upload-artifact@v4 + with: + name: executed-notebooks-${{ matrix.channel }} + path: executed/ + retention-days: 90 + + # --------------------------------------------------------------------------- site + # The gallery site (tools/site.py): one card per notebook, from the executed notebooks + # of the master channel (failed ones included: their card shows it). Built on every + # run (artifact `site`, for a preview), published as the gh-pages branch (a single + # commit, replaced each time) only from main and from libKriging/libKriging itself. + site: + needs: [setup, python, r, octave, julia] + if: always() && needs.setup.result == 'success' && contains(needs.setup.outputs.channels, 'master') + runs-on: ubuntu-24.04 + permissions: + contents: write + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + - uses: actions/download-artifact@v4 + with: + pattern: executed-master-* + path: artifacts + - name: Build the site + env: + SHA: ${{ needs.setup.outputs.sha }} + run: | + pip install nbformat + mkdir -p results executed + find artifacts -path '*/results/*' -name '*.checks.csv' -exec cp {} results/ \; + find artifacts -maxdepth 2 -name '*.ipynb' -exec cp {} executed/ \; + python tools/site.py --notebooks executed --results results --out _site \ + --repo "${{ github.repository }}" --branch gh-pages \ + --libkriging "${LIBKRIGING_REPO}@${LIBKRIGING_REF} (${SHA:0:10})" + - uses: actions/upload-artifact@v4 + with: + name: site + path: _site/ + include-hidden-files: true + retention-days: 30 + - name: Publish on gh-pages + if: >- + github.ref == 'refs/heads/main' && github.event_name != 'pull_request' + && env.LIBKRIGING_REPO == 'libKriging/libKriging' + env: + GH_TOKEN: ${{ github.token }} + run: | + cd _site + git init -q -b gh-pages + git add -A + git -c user.name='github-actions[bot]' -c user.email='41898282+github-actions[bot]@users.noreply.github.com' \ + commit -q -m "Gallery site, ${{ github.sha }}, libKriging ${LIBKRIGING_REF} (${{ needs.setup.outputs.sha }})" + git push -f "https://x-access-token:${GH_TOKEN}@github.com/${{ github.repository }}.git" gh-pages + echo "Site published: https://${{ github.repository_owner }}.github.io/${{ github.event.repository.name }}/" >> "$GITHUB_STEP_SUMMARY" diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c0e3251 --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +results/ +__pycache__/ +*.egg-info/ +.ipynb_checkpoints/ +new-baseline/ +executed/ +report.md +/*.ipynb +_site/ diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md new file mode 100644 index 0000000..b786930 --- /dev/null +++ b/README.md @@ -0,0 +1,243 @@ +# libKriging gallery + +Worked examples of [libKriging](https://github.com/libKriging/libKriging), one per +use case: every model class (`Kriging` with its noise models and approximate fit +methods, `NestedKriging`, `WarpKriging`, `MLPKriging`) on classical test problems, +explained, plotted and tabulated, next to the packages it is usually compared with. + +Each example is also a **validation**: what it shows is checked against a reference +(finite differences, a fit from scratch, the exact model an approximation converges +to, another kriging package at the same hyperparameters, the other bindings). +CI runs every notebook against the **development head of libKriging and of its +bindings** (and weekly against the releases), saves it with its outputs, and turns +the checks into a report: the gallery is libKriging's end-to-end acceptance test. + +Every notebook starts with its **idea**, its **reference** and its **tags**, and ends +with a **summary** of its checks; error-level checks fail the run, warn-level ones are +reported (see [Checks and statuses](#checks-and-statuses)). + +**Gallery site: **: one card per notebook +(first figure, title, idea, tags: language, Kriging class, kernel, objective, method, +noise/nugget, warping, aggregation, compared packages; status of its checks), tag +filters; a card opens its notebook, executed by the last CI run on `main`, in Colab +(Python) or nbviewer (R, Octave, Julia). Built by [`tools/site.py`](tools/site.py), +published on the `gh-pages` branch. + +## The examples + +**executed**: the notebook as run by the last CI run on `main` against libKriging +master, with its outputs (a zip of the `executed-master-` artifact through +[nightly.link](https://nightly.link), which serves the last artifact of a branch at a +stable URL; [all runs on `main`](https://github.com/libKriging/gallery/actions/workflows/notebooks.yml?query=branch%3Amain) +keep them 90 days; all of a run at once: +[executed-notebooks-master](https://nightly.link/libKriging/gallery/workflows/notebooks/main/executed-notebooks-master.zip), +and the aggregated report: [report-master](https://nightly.link/libKriging/gallery/workflows/notebooks/main/report-master.zip)). +**open in**: the executed notebook of the site, run in a free hosted service (see +[below](#running-a-notebook-in-colab-or-binder)), or read in nbviewer. The +**notebook** column links to its source in [`src/`](src): the notebooks themselves +are generated and executed by CI only, never committed. + +One binding per use case: Python, the most complete binding, unless the reference +belongs to another language (DiceKriging and RobustGaSP in R, STK in Octave, +GaussianProcesses.jl in Julia); the cross-binding notebooks check that every +binding gives the same numbers as Python, for every class and fit method. + +### Kriging: the basics + +| notebook | binding | kriging type | what it shows | validated against | executed | open in | +|---|---|---|---|---|---|---| +| [update-simulate_Kriging_python](src/update-simulate_Kriging_python.py) | python | `Kriging` | the workflow on 1D to 6D functions: fit, `predict`, conditional paths, `update` with new data, `update_simulate`, in pictures | fit from scratch, Monte Carlo moments | [zip](https://nightly.link/libKriging/gallery/workflows/notebooks/main/executed-master-update-simulate_Kriging_python.zip) | [Colab](https://colab.research.google.com/github/libKriging/gallery/blob/gh-pages/notebooks/update-simulate_Kriging_python.ipynb) · [Binder](https://mybinder.org/v2/gh/libKriging/gallery/gh-pages?labpath=notebooks%2Fupdate-simulate_Kriging_python.ipynb) | +| [kernels-trends-interpolation_Kriging_python](src/kernels-trends-interpolation_Kriging_python.py) | python | `Kriging` | choosing a kernel and a trend: 4 × 4 combinations on 8 functions, interpolation, predicted covariance and derivatives | finite differences, Q² floors | [zip](https://nightly.link/libKriging/gallery/workflows/notebooks/main/executed-master-kernels-trends-interpolation_Kriging_python.zip) | [Colab](https://colab.research.google.com/github/libKriging/gallery/blob/gh-pages/notebooks/kernels-trends-interpolation_Kriging_python.ipynb) · [Binder](https://mybinder.org/v2/gh/libKriging/gallery/gh-pages?labpath=notebooks%2Fkernels-trends-interpolation_Kriging_python.ipynb) | +| [objectives-LL-LOO-LMP-gradients_Kriging_python](src/objectives-LL-LOO-LMP-gradients_Kriging_python.py) | python | `Kriging` | choosing the estimation criterion (`LL`, `LOO`, `LMP`), its profile around θ̂, multistart | finite differences, explicit leave-one-out refits | [zip](https://nightly.link/libKriging/gallery/workflows/notebooks/main/executed-master-objectives-LL-LOO-LMP-gradients_Kriging_python.zip) | [Colab](https://colab.research.google.com/github/libKriging/gallery/blob/gh-pages/notebooks/objectives-LL-LOO-LMP-gradients_Kriging_python.ipynb) · [Binder](https://mybinder.org/v2/gh/libKriging/gallery/gh-pages?labpath=notebooks%2Fobjectives-LL-LOO-LMP-gradients_Kriging_python.ipynb) | +| [high-dimension-normalize-screening_Kriging_python](src/high-dimension-normalize-screening_Kriging_python.py) | python | `Kriging` | 6 to 20 inputs in physical units: `normalize`, reading the fitted θ to screen inputs (Welch) | explicit rescaling, known active inputs | [zip](https://nightly.link/libKriging/gallery/workflows/notebooks/main/executed-master-high-dimension-normalize-screening_Kriging_python.zip) | [Colab](https://colab.research.google.com/github/libKriging/gallery/blob/gh-pages/notebooks/high-dimension-normalize-screening_Kriging_python.ipynb) · [Binder](https://mybinder.org/v2/gh/libKriging/gallery/gh-pages?labpath=notebooks%2Fhigh-dimension-normalize-screening_Kriging_python.ipynb) | +| [save-load-pickle-copy-accessors_Kriging_python](src/save-load-pickle-copy-accessors_Kriging_python.py) | python | `Kriging` | saving, loading, pickling, copying a model; reading its settings back | the original model (to 1e-12) | [zip](https://nightly.link/libKriging/gallery/workflows/notebooks/main/executed-master-save-load-pickle-copy-accessors_Kriging_python.zip) | [Colab](https://colab.research.google.com/github/libKriging/gallery/blob/gh-pages/notebooks/save-load-pickle-copy-accessors_Kriging_python.ipynb) · [Binder](https://mybinder.org/v2/gh/libKriging/gallery/gh-pages?labpath=notebooks%2Fsave-load-pickle-copy-accessors_Kriging_python.ipynb) | +| [sklearn-estimators_Kriging_python](src/sklearn-estimators_Kriging_python.py) | python | `Kriging` | libKriging inside scikit-learn: `Pipeline`, `GridSearchCV` over kernels, `cross_val_score`, `sample_y` | the native class | [zip](https://nightly.link/libKriging/gallery/workflows/notebooks/main/executed-master-sklearn-estimators_Kriging_python.zip) | [Colab](https://colab.research.google.com/github/libKriging/gallery/blob/gh-pages/notebooks/sklearn-estimators_Kriging_python.ipynb) · [Binder](https://mybinder.org/v2/gh/libKriging/gallery/gh-pages?labpath=notebooks%2Fsklearn-estimators_Kriging_python.ipynb) | + +### Noisy observations + +| notebook | binding | kriging type | what it shows | validated against | executed | open in | +|---|---|---|---|---|---|---| +| [noisy-data_Kriging-nugget-noise_python](src/noisy-data_Kriging-nugget-noise_python.py) | python | `Kriging` with `noise="nugget"` or variances | estimated nugget vs known noise variances, replicates, calibrated intervals on the noise-free function | true noise, replicate averaging, interpolating fit | [zip](https://nightly.link/libKriging/gallery/workflows/notebooks/main/executed-master-noisy-data_Kriging-nugget-noise_python.zip) | [Colab](https://colab.research.google.com/github/libKriging/gallery/blob/gh-pages/notebooks/noisy-data_Kriging-nugget-noise_python.ipynb) · [Binder](https://mybinder.org/v2/gh/libKriging/gallery/gh-pages?labpath=notebooks%2Fnoisy-data_Kriging-nugget-noise_python.ipynb) | + +### Large datasets + +| notebook | binding | kriging type | what it shows | validated against | executed | open in | +|---|---|---|---|---|---|---| +| [large-n_Kriging-vecchia-nystrom_python](src/large-n_Kriging-vecchia-nystrom_python.py) | python | `Kriging` with `method="vecchia(m)"` / `"nystrom(k)"` | n up to 5000: Vecchia and Nyström fits, their convergence to the exact model as m and k grow, `subsetOfData` | exact model (limits m = n − 1, k = n), exact fit | [zip](https://nightly.link/libKriging/gallery/workflows/notebooks/main/executed-master-large-n_Kriging-vecchia-nystrom_python.zip) | [Colab](https://colab.research.google.com/github/libKriging/gallery/blob/gh-pages/notebooks/large-n_Kriging-vecchia-nystrom_python.ipynb) · [Binder](https://mybinder.org/v2/gh/libKriging/gallery/gh-pages?labpath=notebooks%2Flarge-n_Kriging-vecchia-nystrom_python.ipynb) | +| [aggregations-large-n_NestedKriging_python](src/aggregations-large-n_NestedKriging_python.py) | python | `NestedKriging` | splitting a design into groups: `NK`, `PoE`, `gPoE`, `BCM`, `rBCM`, from 1 group to 32 | exact predictor (limit: 1 group), exact fit | [zip](https://nightly.link/libKriging/gallery/workflows/notebooks/main/executed-master-aggregations-large-n_NestedKriging_python.zip) | [Colab](https://colab.research.google.com/github/libKriging/gallery/blob/gh-pages/notebooks/aggregations-large-n_NestedKriging_python.ipynb) · [Binder](https://mybinder.org/v2/gh/libKriging/gallery/gh-pages?labpath=notebooks%2Faggregations-large-n_NestedKriging_python.ipynb) | + +### Non-stationary, categorical and ordinal inputs + +| notebook | binding | kriging type | what it shows | validated against | executed | open in | +|---|---|---|---|---|---|---| +| [warpings-categorical-ordinal_WarpKriging_python](src/warpings-categorical-ordinal_WarpKriging_python.py) | python | `WarpKriging` | input warpings (affine, boxcox, kumaraswamy, knots, neural_mono), categorical and ordinal inputs | plain `Kriging` (accuracy, NLPD), `none` warping = `Kriging` | [zip](https://nightly.link/libKriging/gallery/workflows/notebooks/main/executed-master-warpings-categorical-ordinal_WarpKriging_python.zip) | [Colab](https://colab.research.google.com/github/libKriging/gallery/blob/gh-pages/notebooks/warpings-categorical-ordinal_WarpKriging_python.ipynb) · [Binder](https://mybinder.org/v2/gh/libKriging/gallery/gh-pages?labpath=notebooks%2Fwarpings-categorical-ordinal_WarpKriging_python.ipynb) | + +### Deep kernels + +| notebook | binding | kriging type | what it shows | validated against | executed | open in | +|---|---|---|---|---|---|---| +| [deep-kernel-projected-branin_MLPKriging_python](src/deep-kernel-projected-branin_MLPKriging_python.py) | python | `MLPKriging` | an MLP feature map finding the 2D structure of a 10D function; activations, multistart | plain `Kriging` | [zip](https://nightly.link/libKriging/gallery/workflows/notebooks/main/executed-master-deep-kernel-projected-branin_MLPKriging_python.zip) | [Colab](https://colab.research.google.com/github/libKriging/gallery/blob/gh-pages/notebooks/deep-kernel-projected-branin_MLPKriging_python.ipynb) · [Binder](https://mybinder.org/v2/gh/libKriging/gallery/gh-pages?labpath=notebooks%2Fdeep-kernel-projected-branin_MLPKriging_python.ipynb) | + +### libKriging and other kriging packages + +| notebook | binding | kriging type | what it shows | validated against | executed | open in | +|---|---|---|---|---|---|---| +| [vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python](src/vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python.py) | python | `Kriging` | same hyperparameters → same numbers, then default fits on 10 functions (accuracy, calibration, cost) | scikit-learn, SMT, OpenTURNS, GPy | [zip](https://nightly.link/libKriging/gallery/workflows/notebooks/main/executed-master-vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python.zip) | [Colab](https://colab.research.google.com/github/libKriging/gallery/blob/gh-pages/notebooks/vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python.ipynb) · [Binder](https://mybinder.org/v2/gh/libKriging/gallery/gh-pages?labpath=notebooks%2Fvs-sklearn-SMT-OpenTURNS-GPy_Kriging_python.ipynb) | +| [vs-DiceKriging-RobustGaSP_Kriging_R](src/vs-DiceKriging-RobustGaSP_Kriging_R.R) | R | `Kriging` | rlibkriging next to the R references: equality at fixed hyperparameters, default fits | DiceKriging, RobustGaSP | [zip](https://nightly.link/libKriging/gallery/workflows/notebooks/main/executed-master-vs-DiceKriging-RobustGaSP_Kriging_R.zip) | [nbviewer](https://nbviewer.org/github/libKriging/gallery/blob/gh-pages/notebooks/vs-DiceKriging-RobustGaSP_Kriging_R.ipynb) | +| [vs-STK_Kriging_octave](src/vs-STK_Kriging_octave.m) | octave | `Kriging` | mLibKriging next to STK: equality at fixed hyperparameters (Gaussian kernel), default fits | STK | [zip](https://nightly.link/libKriging/gallery/workflows/notebooks/main/executed-master-vs-STK_Kriging_octave.zip) | [nbviewer](https://nbviewer.org/github/libKriging/gallery/blob/gh-pages/notebooks/vs-STK_Kriging_octave.ipynb) | +| [vs-GaussianProcesses_Kriging_julia](src/vs-GaussianProcesses_Kriging_julia.jl) | julia | `Kriging` | JLibKriging next to GaussianProcesses.jl: default fits | GaussianProcesses.jl | [zip](https://nightly.link/libKriging/gallery/workflows/notebooks/main/executed-master-vs-GaussianProcesses_Kriging_julia.zip) | [nbviewer](https://nbviewer.org/github/libKriging/gallery/blob/gh-pages/notebooks/vs-GaussianProcesses_Kriging_julia.ipynb) | + +### Validation only + +| notebook | binding | kriging type | what it shows | validated against | executed | open in | +|---|---|---|---|---|---|---| +| [test-functions-reference-values_no-model_python](src/test-functions-reference-values_no-model_python.py) | python | — | the 21 test functions used throughout, plotted and checked | published optima / values | [zip](https://nightly.link/libKriging/gallery/workflows/notebooks/main/executed-master-test-functions-reference-values_no-model_python.zip) | [Colab](https://colab.research.google.com/github/libKriging/gallery/blob/gh-pages/notebooks/test-functions-reference-values_no-model_python.ipynb) · [Binder](https://mybinder.org/v2/gh/libKriging/gallery/gh-pages?labpath=notebooks%2Ftest-functions-reference-values_no-model_python.ipynb) | +| [robustness-invalid-inputs_Kriging_python](src/robustness-invalid-inputs_Kriging_python.py) | python | `Kriging` | invalid and degenerate inputs, ill-conditioning, units, extrapolation, prediction cost in q | equivariance identities, O(q) cost | [zip](https://nightly.link/libKriging/gallery/workflows/notebooks/main/executed-master-robustness-invalid-inputs_Kriging_python.zip) | [Colab](https://colab.research.google.com/github/libKriging/gallery/blob/gh-pages/notebooks/robustness-invalid-inputs_Kriging_python.ipynb) · [Binder](https://mybinder.org/v2/gh/libKriging/gallery/gh-pages?labpath=notebooks%2Frobustness-invalid-inputs_Kriging_python.ipynb) | +| [cross-binding-reference_all-classes_python](src/cross-binding-reference_all-classes_python.py) | python | all classes | the same data and options in every binding, every class and fit method: the Python values | R, Octave, Julia (report) | [zip](https://nightly.link/libKriging/gallery/workflows/notebooks/main/executed-master-cross-binding-reference_all-classes_python.zip) | [Colab](https://colab.research.google.com/github/libKriging/gallery/blob/gh-pages/notebooks/cross-binding-reference_all-classes_python.ipynb) · [Binder](https://mybinder.org/v2/gh/libKriging/gallery/gh-pages?labpath=notebooks%2Fcross-binding-reference_all-classes_python.ipynb) | +| [cross-binding_all-classes_R](src/cross-binding_all-classes_R.R) | R | all classes | the R side of the cross-binding protocol | Python (report) | [zip](https://nightly.link/libKriging/gallery/workflows/notebooks/main/executed-master-cross-binding_all-classes_R.zip) | [nbviewer](https://nbviewer.org/github/libKriging/gallery/blob/gh-pages/notebooks/cross-binding_all-classes_R.ipynb) | +| [cross-binding_all-classes_octave](src/cross-binding_all-classes_octave.m) | octave | all classes | the Octave side | Python (report) | [zip](https://nightly.link/libKriging/gallery/workflows/notebooks/main/executed-master-cross-binding_all-classes_octave.zip) | [nbviewer](https://nbviewer.org/github/libKriging/gallery/blob/gh-pages/notebooks/cross-binding_all-classes_octave.ipynb) | +| [cross-binding_all-classes_julia](src/cross-binding_all-classes_julia.jl) | julia | all classes | the Julia side | Python (report) | [zip](https://nightly.link/libKriging/gallery/workflows/notebooks/main/executed-master-cross-binding_all-classes_julia.zip) | [nbviewer](https://nbviewer.org/github/libKriging/gallery/blob/gh-pages/notebooks/cross-binding_all-classes_julia.ipynb) | + +Notebooks are named `__`. Those comparing +with another package install it themselves, in their first cells (pip, CRAN, +`Pkg.add`, or a clone of STK); a package that cannot be installed is reported +(warn-level) and its comparison skipped. Test functions: +[`tools/python/gallery.py`](tools/python/gallery.py) (section 1), after the +[Virtual Library of Simulation Experiments](https://www.sfu.ca/~ssurjano/) +(Surjanovic & Bingham). The notebook sources are in [`src/`](src); everything else is under +[`tools/`](tools) (helpers, notebook generation, installation of the bindings, +CI scripts, site generator, known issues, baselines), the shared datasets under +[`data/`](data), the Binder environment under [`.binder/`](.binder). + +## Running a notebook in Colab or Binder + +The **open in** links (and the cards of the site) open the executed notebook of the +`gh-pages` branch, outputs of the last CI run included; re-running it in a free +hosted service uses the **released** pylibkriging (from PyPI), not libKriging master +as CI does: a feature not released yet is reported as SKIP. The first code cell of each Python +notebook ([`tools/notebook_setup.py`](tools/notebook_setup.py), a no-op in this +repository) clones the repository for its datasets and helpers and installs +pylibkriging when missing; the notebooks that compare with other packages install +them themselves. + +* **Colab**: PyPI has pylibkriging wheels for CPython 3.10 to 3.12 only, and Colab's + default runtime has used Python 3.13 since August-September 2026: choose + *Runtime > Change runtime version > 2026.07* (Python 3.12) before running, + otherwise the setup cell stops with this message. +* **Binder**: the environment of [`.binder/`](.binder) (Python 3.12, pylibkriging), + copied to `gh-pages` by the site generator; the first start builds the image, which + takes a few minutes. + +Not offered for the R, Octave and Julia notebooks (nbviewer instead, read only): +Octave is not available on Colab, +and the installation of rlibkriging (compiled from CRAN sources) and of JLibKriging.jl +(which compiles libKriging) on these services has not been checked. + +## Checks and statuses + +Each notebook records its checks in a suite named after the notebook, and writes +`results/.checks.csv` and `results/.metrics.csv`, with the helpers +of one file per language, loaded by the notebook (not installed): + +| | Python | R | Octave | Julia | +|---|---|---|---|---| +| file | [`tools/python/gallery.py`](tools/python/gallery.py) | [`tools/R/gallery.R`](tools/R/gallery.R) | [`tools/octave/gallery.m`](tools/octave/gallery.m) | [`tools/julia/gallery.jl`](tools/julia/gallery.jl) | +| loaded by | `import gallery` (path set by the setup cell) | `source("tools/R/gallery.R")` | `source("tools/octave/gallery.m")` | `include("tools/julia/gallery.jl")` | +| a suite | `suite = gallery.Suite(name)` | `s <- suite(name)` | `s = suite(name)` | `s = Suite(name)` | +| a check | `suite.check(name, ok, ...)` | `check(s, name, ok, ...)` | `s = check(s, name, ok, ...)` | `check!(s, name, ok; ...)` | +| usual checks | `suite.check_close`, `check_less`, `check_greater`, `check_between`, `check_true`, `check_finite`, `check_runs`, `check_skip`, `check_raises` | `check_close(s, ...)`, ... | `s = check_close(s, ...)`, ... (no `check_runs`: `try`) | `check_close!(s, ...)`, ..., `check_runs!(s, name) do ... end` | +| metrics | `suite.metric(case, package, q2=...)` | `metric(s, case, package, q2 = ...)` | `s = metric(s, case, package, "q2", v)` | `metric!(s, case, package; q2=...)` | +| end of notebook | `suite.summary()`, `suite.finalize()` | `suite_summary(s)`, `suite_finalize(s)` | `suite_summary(s)`, `suite_finalize(s)` | `suite_summary(s)`, `suite_finalize!(s)` | +| data, errors | `gallery.read_data`, `metrics.rmse`, `metrics.q2`, `metrics.nlpd`, ... | `read_data`, `rmse`, `q2`, `nlpd`, `coverage` | `read_data`, `rmse`, `q2` | `read_data`, `rmse`, `q2` | + +The Python file also holds the test functions (`TESTS`), the designs (`design.lhs`, +`design.dataset`), what the installed pylibkriging offers (`compat.has`) and the +persistence checks (`persistence.check`), used by the Python notebooks only. + +| status | meaning | fails CI | +|---|---|---| +| PASS | | | +| FAIL | an error-level check failed: a correctness property (gradient vs finite differences, `update` vs fit from scratch, equality with another package at fixed hyperparameters, ...) or an accuracy floor | yes | +| WARN | a warn-level check failed: performance comparison, optimizer behaviour, judgement thresholds | no | +| KNOWN | a failing check (of either level) listed in [`tools/known_issues.csv`](tools/known_issues.csv), with the issue | no | +| SKIP | feature absent from the installed version (release channel) | no | + +`tools/report.py` aggregates all results: status per notebook, failures, +warnings, known issues still present, known issues that now pass (to be removed +from the list), cross-binding consistency against Python, and metric drifts +against [`tools/baseline/`](tools/baseline). + +## Running locally + +From the repository root (the notebooks' paths are relative to it; generated +notebooks land there, ignored by git): + +```sh +git clone --recurse-submodules https://github.com/libKriging/libKriging ../libKriging # for a master build +tools/install/python.sh master ../libKriging # or: tools/install/python.sh release +tools/install/r.sh master ~/Rlib ../libKriging # then R_LIBS_USER=~/Rlib +tools/install/octave.sh ../libKriging ~/mlib # then OCTAVE_PATH=~/mlib +tools/install/julia.sh ../libKriging +python tools/py2nb.py src/update-simulate_Kriging_python.py # or no argument: every source +python tools/run_notebooks.py --inplace update-simulate_Kriging_python.ipynb +python tools/report.py --expect python +mkdir -p executed && cp *.ipynb executed/ +python tools/site.py --notebooks executed --results results --out _site # then open _site/index.html +``` + +The scripts of [`tools/install/`](tools/install) are the ones CI runs; each says +what it needs (BLAS/LAPACK headers, Octave, R, Julia). + +## Editing notebooks + +A notebook exists only as a plain-text source in [`src/`](src) (cells separated by +`# %%` / `# %% [markdown]`, or `% %%` in Octave), so that reviews are plain diffs. +[`tools/py2nb.py`](tools/py2nb.py) turns it into a notebook (Python ones get the setup +cell of `tools/notebook_setup.py`); CI does so in each notebook job, executes it, and +archives and publishes the executed notebook (see [The examples](#the-examples)). + +```sh +python tools/py2nb.py # every notebook of src/, at the repository root +python tools/py2nb.py --check # CI: every source valid (name, title, idea, reference, tags, suite) +``` + +A new example is a new source in `src/` named `__`, +with its suite named the same, a first markdown cell holding its title +(`# · — `) and an **Idea**, a **Reference** and a **Tags** +paragraph, then plots and tables of its results, its checks, a final `## Summary` +cell, and a line in the tables above (CI checks that every source is linked from this +README); CI picks it up automatically. Tags are `key: value, value` groups separated +by ` · `, with the keys `class` (required), `kernel`, `trend`, `objective`, `method`, +`noise`, `warping`, `aggregation`, `compared to`, `topic`; the language comes from the +file extension. They become the tags and filters of the site. + +Shared datasets (`data/`, used by the cross-binding notebooks of every language) +are written once by `tools/make_data.py` and committed: the files are the +reference. + +## CI + +[`.github/workflows/notebooks.yml`](.github/workflows/notebooks.yml), on push, +pull request, weekly, on demand (`libkriging_ref`, `libkriging_repo`, +`channels` inputs) and on a `repository_dispatch` of type `libkriging-updated` +(sent by libKriging after a merge, with the commit in `client_payload.ref`): + +* **master channel** (every event): libKriging built from source at the + requested ref, with every binding from the same checkout: Python wheel, R + package (`bindings/R`), Octave mex, JLibKriging.jl; +* **release channel** (weekly, or on demand): pylibkriging from PyPI, + rlibkriging from CRAN, mLibKriging built from the last release tag (the + published binaries target another Octave version than Ubuntu's); no Julia. + +One build job per binding and channel, then **one job per notebook**, each +uploading its executed notebook and its results as the artifact +`executed--`; then one report per channel (job summary, +`report-` artifact with the report, all CSV results and a refreshed +baseline, and `executed-notebooks-` with all the executed notebooks); then +the **site** job builds the gallery site from the executed notebooks of the master +channel (artifact `site`, for a preview, on every run) and, on `main` with +libKriging/libKriging as source, force-pushes it as the single commit of the +`gh-pages` branch (`contents: write` for this job only). +The initial `tools/baseline/` files come from a local run (Python notebooks only, +4-core container); to refresh them, commit the `new-baseline/` files of a +trusted CI run. diff --git a/data/borehole_X.csv b/data/borehole_X.csv new file mode 100644 index 0000000..fcc35f3 --- /dev/null +++ b/data/borehole_X.csv @@ -0,0 +1,80 @@ +0.017315140363590921,0.34810328286366077,0.52113184961397885,0.61230499400028382,0.85197029620993292,0.98524761505608716,0.50795743396967485,0.74508070757782807 +0.72375790748518831,0.13498865872561727,0.067640508555753376,0.90478990607720289,0.3509872369953323,0.22198429007544349,0.49612143134603137,0.94418601000738978 +0.88780472252964326,0.66206822291695433,0.5647102873597587,0.86083142574266047,0.26421480623741617,0.38224624798877649,0.15931847750843336,0.3140630269984222 +0.10817302203271725,0.41037975937866189,0.11282305953725118,0.14664543342531278,0.67164921413040468,0.41161397769712299,0.19972692960858918,0.29669743200074417 +0.16032788691691899,0.22335243395614746,0.39032791335376826,0.36953254195018853,0.10431792864846676,0.31851947828835819,0.6186289979763232,0.7663854127961589 +0.41124298711267582,0.047690161516542137,0.98383549478148047,0.45169279988580929,0.69535426273966849,0.41457628711338812,0.57391279424403707,0.70894843334845015 +0.2348355331783254,0.24616853589685378,0.54129851270646989,0.96893654709667698,0.43424104934328522,0.041233051929850946,0.076488887176965031,0.79552733368392126 +0.5930523805714718,0.57965241741162798,0.96889240235602014,0.8821375514324703,0.80117533160238064,0.30542836186217709,0.21561038053279469,0.37216837961072829 +0.31189734526692592,0.38043609100891229,0.83837009737996593,0.63001932302394958,0.21461161412026622,0.20883319928083077,0.18682179570624344,0.98576554844266528 +0.49597377766898781,0.28241251611859897,0.70234200643336475,0.38733154432716405,0.92974270697264316,0.5567880812381546,0.96763656234055195,0.24658639537774177 +0.064975512467892946,0.48275871362259692,0.055039720416198513,0.56792723611643703,0.38322509252848502,0.50408538717721307,0.38445424851046173,0.076641340705933481 +0.91620087931188743,0.0094235343778137255,0.14712474180255045,0.52335466699141364,0.63706891969883439,0.58503803554473699,0.17420422454429518,0.60900159112080732 +0.37780180281451586,0.41500714451935627,0.2956767785683751,0.010910645708621257,0.42447131117220033,0.89875280178053085,0.90620775664595743,0.91787368088969035 +0.092334379815928985,0.031181689554096604,0.48725476235481846,0.78854082811545201,0.49691468521311233,0.47131819474334102,0.95567913089135492,0.42718203070284738 +0.13985358558640684,0.45957708183050394,0.85194312060665178,0.20482400873236234,0.89951930124143398,0.58868656777521289,0.36690379250660271,0.63865908765049972 +0.6858045118220214,0.76265832003875977,0.0899991480318518,0.47659943901433577,0.70988725923463103,0.26310790187534899,0.84296491175789634,0.30070071479132809 +0.80466527314897041,0.1951074522677953,0.68642867761080528,0.64966491923805247,0.44612116635057381,0.49925285099216177,0.2358147642654071,0.89024538977743206 +0.54295954513342892,0.25116845898761292,0.11186463525065531,0.87138250204055612,0.76671595277135784,0.8753450857205326,0.048958669702908894,0.46482920908820269 +0.7881480927158121,0.076579813552882625,0.42468046958732242,0.94038558412428819,0.13992375614022695,0.61670065808888042,0.39676464171473147,0.23651207103628061 +0.026215372709436423,0.87185424944376566,0.89158427545650398,0.32546888645260003,0.31255247376418166,0.75610977614511665,0.79590092080290953,0.40995055185222345 +0.72638363963149766,0.8929737116931602,0.57651794726171701,0.50203764035800913,0.038656413649791686,0.86072210230899926,0.53314034276965483,0.9651465325132399 +0.11603230918849794,0.11755880800456098,0.86368156369948823,0.65274765019812364,0.17283376587182298,0.021122067881225292,0.86881086322049494,0.14247197419743751 +0.22485950108088937,0.80637747750968336,0.62083764391833396,0.89593312931146607,0.91604705493664207,0.69850254707470527,0.60207451438622928,0.1296947947077933 +0.81303046766391096,0.71656624174953165,0.13418695218651699,0.049545847740734485,0.24288264686734617,0.54645121744317993,0.62667342280984828,0.032819313593866556 +0.3278619219259033,0.15900602814099674,0.95925578073167694,0.49988995386364959,0.79392555769042761,0.64743331911239399,0.52255467307210957,0.012066486277763754 +0.47772089756032854,0.72740135148113239,0.42947537827416749,0.13019059098004981,0.9859632941081854,0.44656445512819209,0.14749111986341717,0.80860263198982896 +0.75943494661677169,0.81485062048505463,0.2573773040165776,0.19784410455598023,0.64094238916356683,0.036862876412423209,0.1052271152542805,0.20371386052095955 +0.64395842556086758,0.70536233434179407,0.69066446825420935,0.71903213249890796,0.72332350875277185,0.63658193696816734,0.76687473114040405,0.8789793243501961 +0.36563386396908093,0.51945250087204764,0.80558363480593731,0.080341071150435692,0.40327041581682488,0.60385042802350031,0.47823817640469057,0.49761196394919488 +0.50846454180598866,0.82582591792621618,0.15440968762329768,0.95321639506435329,0.0086356079417305911,0.009051989149911923,0.33556299541805351,0.59766676852197542 +0.27577820392913982,0.74092827918847493,0.78573951924637331,0.83323173255705019,0.23528104236557151,0.84370665490954555,0.98673607024758403,0.32750151991771226 +0.62050441385388311,0.55940738267715662,0.90711742017660657,0.75640943923684645,0.073299130552726077,0.77171727707623217,0.068331598549948816,0.11842100352269769 +0.13498454521277389,0.85858161746734607,0.014060634647304471,0.67883441218652729,0.53793845354240322,0.6785912507081967,0.12507561256257951,0.75931207998105088 +0.63464561787805329,0.088382639034930266,0.66087076181679927,0.39475858189804047,0.61299228478454793,0.82027815515891267,0.26792835717342139,0.35663318168516311 +0.92729463827795799,0.46861879080169783,0.50231816391238149,0.59456893851947634,0.90484224936121005,0.096582255511394072,0.70588273836123228,0.16589823488845926 +0.87949401115075942,0.17157193744680899,0.83261904303031908,0.10230655884878179,0.94068874404422309,0.86892768697160283,0.023431550876115419,0.52416022560128817 +0.58320835134121962,0.6338049337976942,0.08375310086519043,0.23591822219502728,0.83919118933918413,0.83238771582210069,0.55279904876283614,0.92875984526388522 +0.20131157874126826,0.44642102109321735,0.60961612932474352,0.18294361931978523,0.39222109107018371,0.11237032114642301,0.42678663838891079,0.41450849178136534 +0.18468040177841011,0.17520897738997676,0.22390029914387891,0.44891877991182938,0.028672284314457952,0.80033273768136259,0.738450447424564,0.55644465546459032 +0.66067494686822159,0.92005477777928424,0.2419326686243132,0.01724710504477877,0.1529124510656836,0.70325090574149685,0.23982677220927875,0.65386883345667957 +0.31388822452460635,0.69403222442407575,0.63280820194251197,0.46731237165918965,0.50563636073246487,0.94729514988833086,0.29652448250322483,0.54537531822691554 +0.74582679723704515,0.49990290096737422,0.38509883720409238,0.82313394829738606,0.58802724448933874,0.13444784816128244,0.25384698396637473,0.71318367547419137 +0.26481875459343623,0.77541884328173405,0.94039104371044502,0.062841618822904233,0.74351644077754386,0.075457678220115815,0.44901593082797503,0.84885555054125184 +0.16846951495039789,0.92824479955006223,0.53133508400486618,0.34745004810099955,0.83220961525106019,0.14117890173918987,0.30414091608468674,0.043557728860334634 +0.77602830418258617,0.64592031355654966,0.66634082728983912,0.054457379508347337,0.87887742395196589,0.39973726951084393,0.88347296370321682,0.66562810858835841 +0.52063810736204896,0.94206967200098146,0.41001222512191554,0.032218875247831361,0.11458179055740544,0.35184956724957817,0.8151088709875639,0.38270592369415313 +0.0045380710433107953,0.61195057623249483,0.33795902568067604,0.67341481182705398,0.60774303767115501,0.16293837239649403,0.75371230183734661,0.95587371840404756 +0.95644272932528707,0.54753670500977958,0.79890596038378026,0.21701210456309217,0.08646765565076317,0.11334286863085356,0.68878563107146973,0.86569033271400841 +0.55619648651555409,0.7948782381266023,0.5598545269355012,0.43477889458000885,0.3490660480604536,0.074576264748874674,0.94790720897398639,0.7300812760009856 +0.38841485503391304,0.50038566469099599,0.18105037555768583,0.92632918767552552,0.28453585762600336,0.57477815934985743,0.86201687658158366,0.77622667043885241 +0.83282210649599886,0.88487912186084616,0.88101335836788564,0.80016249588407717,0.58618267900314536,0.23840929271605141,0.92339675445277669,0.50747438936161104 +0.45371483901965892,0.61753823730815327,0.001853261341052724,0.68963481626767431,0.31091510807902584,0.95626063453702359,0.45745691417381129,0.1888154531494744 +0.69732154186961748,0.30596093156156329,0.19610835659379805,0.74488840700022352,0.81552643124670321,0.050509931847235158,0.59415988982718315,0.39404465773931796 +0.86504477504702815,0.31967445649363591,0.35189134000884337,0.41927213994536716,0.25328134014292925,0.73568032987530141,0.089530365927370084,0.90364973608026433 +0.71098949187598781,0.99234226976054762,0.98876954364903613,0.40330074133916993,0.37435082969208627,0.48239562436614547,0.27708530572410583,0.18474849454672554 +0.19323228581497121,0.59560771310004279,0.16291633462177307,0.31399965044529055,0.13321123781651611,0.65110745719228036,0.34541018399546219,0.8536217595094433 +0.23882605698920706,0.67670212194371637,0.28476849235774371,0.53875663049398337,0.017929657190391458,0.29510052631366468,0.54740574674064357,0.27900357923811547 +0.43904225105924377,0.057180027599366148,0.27163468132161472,0.77591834820868266,0.99555295828300872,0.28141075127003595,0.41465560875959495,0.62081057369769588 +0.97531523639279794,0.95454495712988618,0.3305044981603748,0.6150066737903146,0.96695582788795331,0.79354595060394384,0.36040296094317814,0.44728782667161032 +0.25441966956833478,0.14140642687205213,0.045926112789273318,0.25095707268289386,0.8670780859699283,0.19106049970444336,0.89532462933040546,0.68072408402576301 +0.46631778660104961,0.97638570139169933,0.71496220507544828,0.57931438632094789,0.48469317396344891,0.16001027256734293,0.58514372673257808,0.5682813644371193 +0.91162067168716354,0.3509191879629337,0.93084590729663541,0.92078934042789684,0.090805921469379144,0.93382135644258724,0.73438556160002877,0.69846334480611616 +0.42590901074547605,0.23720159638589955,0.74757935772220407,0.98118550411310168,0.6579885169049583,0.91247555553144566,0.65837026365371165,0.066902447722907377 +0.044128637266163659,0.074949978125162145,0.45069302975652048,0.093716451839574347,0.33573598479414246,0.99818255747730211,0.20488744177506701,0.26240336550806653 +0.41580326689250374,0.29732095998452179,0.49864085819652881,0.24466284094005206,0.052308925056974306,0.26170851552317453,0.027700461594000393,0.060442039168098392 +0.29908806426452916,0.90848529905979658,0.2018848434199183,0.29908603837873604,0.56421015235957439,0.91736580604844709,0.67365636074389923,0.15731983221197451 +0.05899914048784085,0.56705250376650984,0.31491022516915179,0.98886618608531696,0.95992070110768091,0.34130318072116334,0.71934236267286267,0.3423245049070015 +0.94155210827523406,0.36562948913139254,0.23643824516861689,0.30557709947224782,0.53469893618055253,0.33396511749155006,0.40672414665314055,0.022532295025624299 +0.083257713977681516,0.84701065133553877,0.75374146813643583,0.73552705345302427,0.20664976191331741,0.43303407228684365,0.059358203298531212,0.63622424817954504 +0.99525685196011648,0.53284777741489753,0.46278624161528342,0.35317769946885236,0.68018265966240199,0.96658141307094014,0.93609288509798461,0.57708544648204529 +0.60660881928749399,0.42972260678841046,0.3655720380853682,0.77326996808719495,0.4722747144990973,0.7794380115964038,0.4749421294583952,0.10449513722627671 +0.76761664248844796,0.20842407761856813,0.31056080320134016,0.15909853943869595,0.51685726790285613,0.71946593041133955,0.9939247594587719,0.21643876892071018 +0.56710713221160136,0.015086924409817019,0.59044109520709531,0.2627472696579799,0.17857612797249653,0.51954551936013471,0.78184574210436675,0.451355080234904 +0.8527816708242193,0.75796144687116629,0.72812332327731411,0.56097282115132496,0.7546211103076681,0.5302981121664162,0.0059822558556910568,0.099258443230797827 +0.84752034599427239,0.26444024541141686,0.7724758774257281,0.12273468676077211,0.29197309464805304,0.18130284265739935,0.63782547891390107,0.27317006976029706 +0.97340113248751936,0.39800541377902882,0.027322578844861889,0.70513112554413471,0.1918516344111553,0.45474982123565261,0.80104365015277179,0.53669086181947623 +0.33789989878711146,0.97270467803845673,0.43909273023867018,0.84797926500094412,0.77914905027844805,0.73788429819241708,0.83359203663052173,0.81483312130984342 +0.67002923930507807,0.10395769566387203,0.6474684007931667,0.53548067137263344,0.73741832135461705,0.3637246718672334,0.31377215832110567,0.83523970232878919 +0.35574782078199441,0.66582574527594174,0.8187470253847634,0.28099663614429538,0.45956472112818325,0.22693565845789943,0.11859195903219569,0.48527474614432087 +0.53220232167295933,0.32641497323648222,0.91426032058937656,0.16759690648940931,0.5521247637673421,0.66477095220702442,0.68705939369236413,0.99512523525613883 diff --git a/data/borehole_Xt.csv b/data/borehole_Xt.csv new file mode 100644 index 0000000..63978c4 --- /dev/null +++ b/data/borehole_Xt.csv @@ -0,0 +1,200 @@ +0.44217227295272243,0.78901759462748711,0.92883517101843105,0.17042606197312338,0.85842780900467719,0.78008118189651876,0.61572682952990154,0.5515917799926181 +0.090008414040849916,0.066436011213980128,0.36395440971819548,0.56382552625339921,0.59603640790456369,0.46942500529135189,0.87341965920427,0.27891743261638047 +0.91470114458629015,0.12875580368063,0.72330765612661896,0.94299476206174915,0.0071488025233706898,0.0070146754608084727,0.65611115848812707,0.55445433290798118 +0.94651428377004299,0.48982475344992249,0.72627075322195833,0.63531945212934016,0.84154166038552913,0.87694418203435021,0.58085516499613843,0.81587202329207398 +0.69891279386383909,0.75791265396917706,0.18119157938347696,0.60311037652189481,0.14234958597001457,0.60032353632313884,0.7626038059122805,0.086820184077650908 +0.11186104036238775,0.74418916658213197,0.32062256074155193,0.19989839643733331,0.75007045378192894,0.19795292432136735,0.83072372731131638,0.82198204613595083 +0.053916963829623965,0.52273796131998462,0.06484583577276648,0.72666405218749575,0.0058732290974637946,0.94800955054377922,0.43537510251961531,0.71952125426558322 +0.20064650389128846,0.99076045724539419,0.084426903612385806,0.93958462242323071,0.61830845161439763,0.095757230018457151,0.14303257531275559,0.50325844688795984 +0.24117907360732416,0.82757768573576806,0.15501613891033339,0.45614272588285909,0.06757893150272487,0.19169136032447587,0.74394765454437939,0.42198575465404942 +0.79566433908098122,0.022432735582047836,0.45526027961659254,0.22670543264604381,0.041969134457666701,0.76797985085979048,0.75152717450358009,0.70027393045195352 +0.96431894999931078,0.71933691302467628,0.0083535630910052117,0.99577654434371621,0.49107197448063877,0.61224636436388424,0.3034401566374697,0.3576014547910763 +0.61338323716802057,0.7400432125811105,0.49722147209535672,0.5401526792567648,0.42882614983424183,0.79263106477152101,0.81662464835723969,0.99006174398007007 +0.051079393166953757,0.10305214156860165,0.088007351038719706,0.97598737516737288,0.20669571756588578,0.88706819131737635,0.78471862813803983,0.89799767861774571 +0.018750356594929407,0.0207711338308052,0.88592018667820327,0.929288750591311,0.91997170735839207,0.028439447273798768,0.19307172125323935,0.76951207043301029 +0.36299400492802036,0.026775567824404156,0.37115676600116632,0.14199663417590525,0.026832884623721798,0.9247216401204863,0.32301841517045682,0.33125143832922455 +0.77341326510301378,0.16933829864247008,0.055361531615479187,0.94052206655136317,0.79823699901359413,0.1706929874189751,0.058114782212122629,0.25603132525880468 +0.83719249918581173,0.87951861569513778,0.55581040738549514,0.16028806653964967,0.13601257793440014,0.97113546286683772,0.038006819017857851,0.72798416175824521 +0.78764745715918405,0.82311590735054008,0.15091833908836205,0.51382553647050577,0.78473567086699736,0.39662721649426991,0.11213203725839793,0.084201672054377896 +0.76902252303557894,0.67020564886877032,0.16420304940658359,0.34089838593329702,0.95308665451471919,0.42939797005730729,0.37560315982637904,0.46861740632813664 +0.90424178672680777,0.34788392337639784,0.77691560498790158,0.094489469117504776,0.15774909680215321,0.069962651281692656,0.80431459508952852,0.77203192874816617 +0.069217453967550524,0.70778546282554078,0.14042714760127439,0.73941084152373415,0.027547748316445131,0.64153621751536549,0.87848815126631119,0.18961070438471372 +0.63460597662510088,0.55744114580299786,0.31876176905004472,0.59772052057089065,0.1714939792670247,0.830576390685829,0.85561387892704721,0.60780675640160864 +0.83868485053539532,0.84832295939756008,0.58414593733301567,0.63177931856820291,0.71935086663360759,0.22990871470043284,0.89805754888889422,0.26428062486215154 +0.59442668440972568,0.71810648571140645,0.28522064665601476,0.88445798380784091,0.72809476916014881,0.13816718483617652,0.53822096846412115,0.27485705761574097 +0.438360416048446,0.78083629079733796,0.55920798299151964,0.055494012131647397,0.99823093169026944,0.77586784324529623,0.11198378257637132,0.85288615748942231 +0.40175513784851757,0.95757972182259288,0.90994983631532522,0.89179018757098727,0.88044050020888409,0.056091759467511104,0.36913982376475396,0.49712887776359471 +0.903073595780708,0.65300006886845552,0.13684116946487057,0.4955818059240078,0.35770027943443372,0.69926323197080609,0.71928705467098963,0.73161976946167728 +0.628561338397021,0.39341656595279551,0.1216973572747202,0.041119778717247923,0.37262318934523342,0.68044401276901534,0.54751122917157846,0.75576396515555433 +0.99789773101401624,0.14669331613891079,0.38516255091568152,0.46666822201279567,0.81840005805064731,0.9796538701898152,0.29071799279316068,0.1311310141822658 +0.42702936816549431,0.026150618155328154,0.84248706515183547,0.016683831236958824,0.53705704856426772,0.66264776493275201,0.41947305364252541,0.81743342743120073 +0.88046155051935504,0.63867680496949697,0.53664959001243018,0.25602224163988496,0.36451494535291451,0.68500917375131598,0.52413037160987364,0.11190905368843318 +0.76825221449813075,0.85876164123686083,0.35912254707983404,0.28465627959627482,0.26775242430060964,0.21837134941219793,0.89273880089008062,0.46320701864374059 +0.63641694396865356,0.49444250824172298,0.87211383416274912,0.031539296562799746,0.87076508373449524,0.48052446873597876,0.64192779885974338,0.095065021673584771 +0.8231433992053161,0.39175164168258059,0.33087687520954856,0.32143638706046096,0.018411316809590406,0.87319854586237666,0.92155373656305606,0.20581596094541221 +0.55188496454699454,0.48585080808920156,0.41974822079656038,0.57661563134774729,0.5446928112307059,0.24089044474948751,0.34466111301214686,0.3506568223103127 +0.2338606360193981,0.46557562048313106,0.88450809440822453,0.50715507176959351,0.14045160316228333,0.48509668009589624,0.360258546636122,0.96076386513093515 +0.56134634728644706,0.5340975518039186,0.50190811473723018,0.36381673371817891,0.37477072331755812,0.96508344731737683,0.39084217320433012,0.60933856666296793 +0.65122668995859001,0.23352871928950392,0.64063349602723219,0.99864098121415945,0.46209347036148074,0.12687596706969639,0.16318878439176177,0.15547728302456199 +0.23066575492329078,0.89434669536302658,0.49417315039274734,0.4662367306790135,0.65456402562107496,0.71058961869572645,0.1697284684164404,0.62702325115012358 +0.51030799374590718,0.39954206433522288,0.50285327550788073,0.73315356431785661,0.58604851504040334,0.13354311518037121,0.96578812327744989,0.11193849270848499 +0.083077899070327921,0.52442785427259542,0.93376825844193123,0.43383100103336025,0.94009652569179214,0.43501246449891506,0.47880682700956556,0.22037005458370629 +0.90536028488981424,0.31812781713096183,0.66675349078305979,0.3576766775612692,0.85944984250491008,0.82166195954009547,0.65694772385011224,0.55353558851297924 +0.020719990036436653,0.76842448392307683,0.78046209677104172,0.96397398901165343,0.36895580318514731,0.18426348727066455,0.47832392489354769,0.92493425117829353 +0.746115337111384,0.69967584280146111,0.3630711474317343,0.10918933553520471,0.29246668272083998,0.34602499230663553,0.48161928414880717,0.8415320136043527 +0.812147080924843,0.77126044170551766,0.078417122269472839,0.41628642897226109,0.39150900051524684,0.26150089118819875,0.74408763066647698,0.45085208448287306 +0.71285997807277601,0.2753306296098561,0.12898795235915839,0.34480539498689122,0.51982539467826849,0.91287365307545121,0.62585087258273753,0.88634058763789325 +0.45935240399237376,0.13251939589417994,0.086633350583403601,0.65651377323830051,0.62771050020820296,0.6797193086843597,0.083760739416245067,0.052430589482174716 +0.14747024041364887,0.43763308889551578,0.24296988898622252,0.13258663209207755,0.46523994895009613,0.24497744473370631,0.50496531693052349,0.67757190725172611 +0.71404821792359652,0.030626970897620343,0.94777396510898293,0.44567518604116774,0.17306471962612224,0.68298843561453237,0.36841050863965308,0.58994133912522584 +0.45936162343167131,0.53382226460184778,0.65002328895004424,0.10864465814886792,0.01942201492659601,0.76871317861074151,0.57692470809450225,0.48999684806854216 +0.35212090470108326,0.37408894416049887,0.22425360703023356,0.99199739291729983,0.79243589991150853,0.95534664543425973,0.057222604827159684,0.17337978654682307 +0.039427541404710831,0.14796363757148145,0.042743153993095917,0.85033417900154784,0.090283821204099612,0.60211540875303704,0.45826510070619542,0.97349262079913423 +0.048908932107002867,0.94179869200259558,0.22597852949657726,0.55279069088148436,0.85821145832999857,0.91872393589636125,0.67122847744385783,0.50109020180830954 +0.25223028027912497,0.17884147966941855,0.45599282138873942,0.22160909066511836,0.3186815135049762,0.02241249130413725,0.55773921291192075,0.38470098605538539 +0.022533572253770617,0.6050657075846082,0.043487613941568748,0.53934104885917711,0.2048044474554539,0.94910685929557614,0.097665495628938448,0.06057537661403456 +0.71700211525279567,0.65741920395293185,0.65794164196968086,0.35149971818475356,0.57547215780940353,0.087961234899303964,0.7781823974288542,0.74953302192572147 +0.63402143637809749,0.0034196367376099213,0.77070281331927726,0.1672559086248796,0.44526581792866182,0.83706659052125232,0.91078429152817331,0.1012808804706522 +0.73914808091126138,0.40854152210420125,0.63993167458299149,0.22520281850781543,0.47000318227460902,0.59929212221298844,0.98392019982401435,0.041155673701470885 +0.1190314594746964,0.99759038488170293,0.68235615733725707,0.18515863222281304,0.051831662179558036,0.38119375500461883,0.8487201410216858,0.36909893970598429 +0.90339893345591704,0.90182547532260471,0.61715655970918815,0.0097326073486683207,0.94038167174886045,0.86156765053749718,0.71930243613700473,0.87715670858166883 +0.22703695207567265,0.14161948706908112,0.74195050612809177,0.78085154440425664,0.78874025746431087,0.93057904211114939,0.29183740899010713,0.31095399060180351 +0.14695806170176184,0.88492340806357972,0.34925993492283247,0.85339396238931886,0.80163748078099806,0.7905610355581868,0.089924089264773799,0.73181692511633911 +0.22067059670934686,0.48735839477942899,0.098171496800493974,0.90675394017506905,0.069557294218870247,0.29097149092197205,0.72153013438523284,0.066503145201721736 +0.35312318593519987,0.9316807688165416,0.63936217566431408,0.735063372648463,0.94773068861385878,0.75631427041136379,0.52584791059636649,0.010543887407105879 +0.26303186664623135,0.36704837788007072,0.55744390330797899,0.18137362317991101,0.20461573382027953,0.035663941053448722,0.86734566249117118,0.68820239436232844 +0.18894522921359036,0.13793503477272884,0.2992724093786121,0.1033144512351708,0.61661691128291896,0.37718905345000275,0.18403654079553633,0.81884407314419161 +0.038227921629746975,0.77827057523899035,0.95559910361534939,0.9680367654491332,0.34413978459763217,0.65740085362300327,0.66738990604949111,0.2196608562894693 +0.82417232049083256,0.51559227582518863,0.86281306922552536,0.79908207705099499,0.62387497079999055,0.053929081730902717,0.48520601035204247,0.6815889692064212 +0.39069062396181164,0.48752675783748556,0.86363627145154276,0.94258203134459617,0.3438459442937547,0.88575784606477337,0.24673919626437923,0.11552216009078442 +0.81557778206623033,0.85147269494307232,0.29789800564710833,0.92137862449980523,0.43824762226175451,0.72211597104877523,0.61593717771593581,0.096236728196555599 +0.35524254697284363,0.090596855022192613,0.0052066592098156672,0.51197944610236534,0.71671354920978192,0.041369372745930111,0.18728004180511326,0.48797309456244464 +0.31041525280588955,0.38059334668669942,0.32347118376854567,0.4332118004061899,0.35469937052280043,0.59253875183505034,0.23446675682369533,0.36515867001065516 +0.23374602937207489,0.084866827899760189,0.86600386958042808,0.43821434737536791,0.17174863438835963,0.26280015327115558,0.48449978001673233,0.7255442833110054 +0.90821030764840693,0.71799816252840154,0.70948774121983949,0.58554834348194562,0.32636025957107351,0.82728362015938073,0.93560659543072677,0.45552524752096912 +0.050735310033296122,0.89609171757556638,0.70200978162362926,0.9593139683032641,0.08084401957295928,0.2055759450644209,0.032777485077280843,0.36923928401289874 +0.19562966593519204,0.67000987719163774,0.87797959340292653,0.91844558599993142,0.62749639708681149,0.48385771775668107,0.15657583516124118,0.31621307808415744 +0.023117826601279923,0.39018681787170595,0.88469695886663868,0.97410867088107178,0.63328279107388896,0.75383776376417999,0.17181079139539812,0.4859058368775957 +0.56490224579450377,0.67578022275089666,0.44838810619699032,0.77982377675830628,0.24582790037062197,0.34774287389834713,0.041440219818951185,0.87985957254836777 +0.42337189425547295,0.93685380430211251,0.57069314050569941,0.20144107825120194,0.23728723335854529,0.30594455490165928,0.8474504513121327,0.97053055503169405 +0.69613551352744996,0.85514532302406854,0.15947970995904992,0.29775244122404743,0.52164915568217574,0.41316694365458873,0.049324026155127987,0.40708616853581536 +0.19176145776066167,0.84139889183939254,0.96556365296488245,0.73181355634999901,0.69182938530202165,0.33190144621183271,0.31275211703589023,0.76889587169076756 +0.13446659114924453,0.59411596264439948,0.64976245379243125,0.9028897334515259,0.56068058180073299,0.47909671700562229,0.796119231512738,0.54759501298570901 +0.18059589527966124,0.7223896089572891,0.96471557609319381,0.040103130756295013,0.63861796653838787,0.069425191046543766,0.86189910308882034,0.53763450466049834 +0.10721909444783606,0.68671313839982162,0.22695555921241539,0.1511436390326274,0.58218362093428278,0.39867222230833077,0.97834076834871275,0.79419496945355939 +0.91688846467829155,0.10088753757059987,0.061061158050419895,0.67994784976851819,0.94473002488885538,0.48693371523909112,0.45230551948952658,0.86347275022214831 +0.73483144811492507,0.61597462372316369,0.50133213737902849,0.37047674441140865,0.93889451221873688,0.050517763859100784,0.40702045195936709,0.2520128613827759 +0.91048791909451443,0.6210800686899729,0.13293039403613427,0.58018436606904966,0.86709540445991629,0.2386789350796219,0.7611470916245201,0.83316604425063645 +0.72542682513110157,0.15182502861306413,0.11414082457865204,0.22301499358470256,0.97907239586245076,0.026828753687180962,0.68582980356557144,0.8257867815794887 +0.86321998962116198,0.068965574504487814,0.91365671035527862,0.67804381101798528,0.51271187788444672,0.32992202914947044,0.018593411057336695,0.98649494701104423 +0.99575981585553908,0.15449282819993471,0.0075029401545437802,0.0853378009980722,0.26938891150741306,0.89877445878559159,0.89482668519223396,0.50183461933017914 +0.89772161183359811,0.91576585919468545,0.94936120288973735,0.8006132457256766,0.82996224763659288,0.31378355366453536,0.46650679843440934,0.048753038691927708 +0.21783379289612492,0.18581071774583136,0.67946401684430324,0.87387870989271377,0.48242319638247833,0.10105266437421923,0.37374916281514636,0.30309233667316526 +0.56694523412393139,0.64956008799177822,0.58385576007038009,0.22667026416431113,0.035404301590516107,0.44375984715984662,0.96390861450255327,0.28781177105547895 +0.23927446828956567,0.39193415515326679,0.12659765990794758,0.35498077559143448,0.74298058546744905,0.30481885330080538,0.36533004094400434,0.92348815219573122 +0.92063031319191424,0.10319537359310382,0.72953854419588859,0.14119033530321234,0.51169459899501479,0.59631189908614024,0.36379823030362701,0.31954671312304739 +0.62839337995717526,0.85593516656389845,0.082475141169999766,0.72132538758238129,0.32261722513657021,0.84045739404870579,0.86030807052748182,0.65905577724684761 +0.38881656283350885,0.76828342213397205,0.087840165047588181,0.54490615915149576,0.70090603475589408,0.16071779263884556,0.64981221389800747,0.72896270716066769 +0.30010128734379748,0.42030552358465134,0.023834029280044078,0.72770535170143702,0.041065478093853525,0.58829055572290223,0.38977633533641498,0.12394911786575369 +0.75505511340810594,0.2516528178335985,0.81691505980908796,0.69422556543084346,0.33686592923689118,0.34085315989803178,0.10997690436711094,0.014615730536159832 +0.90140026987105737,0.70582528758884555,0.033948926123963385,0.96780092115233163,0.48098427107239827,0.80704925231076086,0.4016138563079743,0.47761334182649695 +0.54478012478765458,0.79279165463013923,0.8119440137195727,0.48390443797968641,0.043205209947738465,0.18923682408040665,0.63136182624828263,0.36063140012597406 +0.84046531669042512,0.50453699496169035,0.37876978662052663,0.89391899988227275,0.30766786855964323,0.69728586123579284,0.12794892735664287,0.86048543559990642 +0.3667486524913498,0.85129424712553947,0.61903184137359601,0.47941939819362334,0.028951289357288856,0.88357945915227987,0.46152872502924891,0.40630351400190245 +0.80438192536214281,0.83929464175771673,0.59201420849470854,0.84892570599595485,0.31762397122187636,0.65119808025248571,0.074918947297574201,0.42430200739887991 +0.68319137780224126,0.026385705695325612,0.78443919907425774,0.84226661745847287,0.7388340968521504,0.19768508223489634,0.70718691928919974,0.03282856076125773 +0.050421917217190582,0.69109925367859393,0.91270131219360884,0.052336807224433346,0.028399105424875115,0.95786572596658759,0.53103684530405582,0.12441641597413466 +0.45054279204975844,0.24387187346127104,0.71992509700707119,0.35674916058041628,0.82961737375837608,0.61393608699210434,0.94579291150500933,0.56279509237920433 +0.36120436250107413,0.94652080281476558,0.03843713656802783,0.75959031707576974,0.87437311903398052,0.7826298948077971,0.93266265213537558,0.6395496467121129 +0.079647500740178123,0.23851633551525819,0.40083526511666412,0.30824960165983017,0.46591343539191576,0.029110584260529704,0.55903086708913119,0.62870504795616367 +0.19987232402341448,0.99050508950152183,0.80934789042512845,0.18889673732461143,0.86923523130183011,0.98282929428628019,0.55337935818177231,0.23830369928024575 +0.055001152598691427,0.6188277861334307,0.4507960458537531,0.88137978253280591,0.15912585948848723,0.77092669570045647,0.7847948366931381,0.48959215032588133 +0.45663956371196401,0.97184250386187487,0.19433505102092086,0.92874576130787923,0.80743892749148494,0.085658229249642059,0.19677816653433489,0.7405131385723368 +0.0041907453513444493,0.093642119157934522,0.17599325273462463,0.44031544778058695,0.83067534197192172,0.033216904361171307,0.11536314079123389,0.5102245080163077 +0.19975112738002865,0.47334358882048322,0.3892065888026135,0.47136128607269001,0.70634787103884056,0.54831799980223583,0.32118601218049914,0.7127945499431414 +0.63041430807405863,0.16569460626807175,0.16467161191532376,0.51788647221253425,0.12321578772179198,0.62635913076423166,0.3473813448187425,0.013672929948132961 +0.96722920280071789,0.58083477150455731,0.23410249450737397,0.14943842268776031,0.87270580540630671,0.13351920794120509,0.13839938104898442,0.94945686047652811 +0.35781798311027213,0.29042048844606638,0.25659301491676823,0.51113766718523024,0.069944319042078318,0.72956998055276978,0.24046945366911276,0.47926709176862203 +0.86150930024220562,0.93776137063029896,0.79961375311870964,0.57352066347839303,0.54687475122806206,0.79701385928830137,0.24049999507281461,0.8923586512130115 +0.88595316541027214,0.7011924428707581,0.18086625815434398,0.22427904669059784,0.2172542482382156,0.25587863317403803,0.49572774951863241,0.70608566234036385 +0.32747976366673992,0.74569487886630648,0.2525031568953543,0.051859747826207281,0.74998658677599617,0.12725236482497748,0.74432618566669639,0.94554634537489302 +0.53521700907806102,0.4292642983712478,0.8362033661423105,0.90786359259667515,0.36569056760258634,0.69810575079842341,0.46857246829945842,0.77541842292209096 +0.41839559157955608,0.91399241412279653,0.44672437217750061,0.929993753115961,0.1827743058783794,0.75168291245225727,0.99092399795516783,0.6547922256280948 +0.62593000924033271,0.97172193190022982,0.49426505564929013,0.074673729150325663,0.026643871818220144,0.28555075490696369,0.93431449589252202,0.36267420243642123 +0.043263642931501534,0.49482710905529148,0.68145334890461029,0.86734524868509466,0.48991150036119724,0.15036155373951821,0.070699988013595894,0.89092188654327276 +0.41068427588057155,0.12904950055791686,0.69480302829489937,0.63811020194559054,0.91645550073587567,0.80519115086716908,0.20448242260775351,0.65580934314829409 +0.31610984258610797,0.71882020623546794,0.49933191141932265,0.17370058475240502,0.43664258477117079,0.13213499611154855,0.57241200696193018,0.64659183916080187 +0.40434200997818226,0.13706664014649761,0.94688717379212495,0.3456492750002581,0.4097899759128949,0.6800439294463565,0.39162593481651997,0.53413350489466993 +0.16586847401344718,0.75004337084359929,0.16880412045512316,0.55111807718709793,0.04124219864380263,0.097387769016449743,0.53597843997870231,0.96906732612953117 +0.033759297611836825,0.080543627269731632,0.30751999374680328,0.90671689149995993,0.71676463320806938,0.45668113516834119,0.86045634448611841,0.67817457949070359 +0.76830519894521954,0.94151978121710789,0.90158141824004079,0.4350335622514202,0.15162898604538966,0.57115468157057248,0.49682434529215125,0.47504690685260709 +0.40643948497113924,0.58488253805387713,0.52598456595070342,0.083234337347750431,0.25972916074751795,0.32479992053604689,0.87976216641538307,0.18281629421166912 +0.60392680893505768,0.91502705153329733,0.43761627976593176,0.80402706064220264,0.6273339145980138,0.91964097832028024,0.0069584446486300466,0.35858935042458795 +0.99776579023296974,0.7323106124664065,0.99444462067086303,0.55064076720201616,0.71939543855946153,0.62808702365087288,0.90713340981631985,0.42434953499263839 +0.93352118483345892,0.017882640552630558,0.11284835462198939,0.90593461379764106,0.97130593278351851,0.88437487478432852,0.29144714740062461,0.51169737233458612 +0.90264150197047111,0.084603800550850261,0.047578061348511169,0.39185415987882333,0.56579641705722716,0.9703654809895027,0.19162801281176312,0.90942963948342381 +0.78936801934838474,0.96366512280364558,0.79546593890566186,0.51733728330674489,0.53131372842752622,0.058047284954782619,0.5510772172123295,0.65550318812135622 +0.38847954488973191,0.20952386730577777,0.80298268666000816,0.56386534327119386,0.17702671758543154,0.52157483705587826,0.63490662224693062,0.67355732771199317 +0.86694370389754416,0.82664105523217957,0.61290885664942873,0.70453214097933803,0.057231918716460384,0.6356421796118229,0.25177116384795695,0.026212725189714181 +0.016332821808610842,0.67417874241694964,0.29587373750678081,0.60391582960635115,0.31060116437129026,0.5148496315152985,0.39519336740118216,0.9437901148496084 +0.068512495585680178,0.10202691734057756,0.87438986974275668,0.4044480483208176,0.3685948147671122,0.49743580637592844,0.56648106716672342,0.0663101445444787 +0.70621096223442148,0.78807750016456113,0.24614380757596077,0.19484757623423465,0.7407319611530323,0.23318265165834229,0.96815638318953501,0.89814593040818691 +0.17026309949355967,0.72357573313558776,0.91385990336862355,0.88894099766731061,0.92411026260362639,0.78121779355801568,0.11438992060412345,0.39207367439807572 +0.33613695489064233,0.14399554259385206,0.40273485677983234,0.52376121799828113,0.34245397428486735,0.1150028789727513,0.45570771502642549,0.19598429104075388 +0.61569976967822571,0.89769875441783908,0.51837735923761041,0.48726460537185312,0.31807454995711926,0.029483566217309853,0.2781932543050919,0.38524745626844503 +0.75111062013684271,0.098987349494230603,0.64538644785786936,0.59707018330581374,0.90519726845275605,0.55281969873704251,0.1355291667418157,0.54622204312372336 +0.34644435831582443,0.011798206828782409,0.033624094666158455,0.13111743614643834,0.93603263346549592,0.25383384436455003,0.089321005177919965,0.36246965586368962 +0.17282340990041845,0.29667443322592668,0.30109520068203777,0.53805030661366138,0.43837640338587136,0.37811696104088988,0.2228190908919494,0.2395078365420924 +0.4877752618543284,0.10133006020089352,0.80213383266877658,0.89330135769306229,0.516591662836415,0.2665953691470716,0.34799564525466264,0.083593657450046988 +0.21986175388036688,0.91735603319940828,0.32311664810286811,0.78571978455261315,0.69882957820690517,0.38134381502748993,0.8731953076335266,0.70847543251200062 +0.36706615561146438,0.15047533613460107,0.58838048858907388,0.72142234610318901,0.83310000614149171,0.99348567886576322,0.88626076722980962,0.58600384594427168 +0.071064618405297653,0.45022896624802711,0.61609271018660083,0.57020580461244486,0.40867053409891707,0.084786105750508822,0.1261390549865341,0.77315390085067481 +0.048642258402415095,0.51357533654142173,0.7134156110195744,0.85618922371762696,0.28669346411736907,0.69066218911038757,0.78540007097325581,0.54095364696099824 +0.97693470407831917,0.11047936343123899,0.0039574991424570216,0.20906389440804529,0.419520335001628,0.98306568147979256,0.29447929497269831,0.18004496981367957 +0.92308388556053156,0.37318280351831146,0.37133070921852529,0.32771215056177772,0.048543406758702945,0.082178926209890313,0.022553256124886234,0.97804368579032264 +0.15707730865507974,0.35264200827806458,0.5494017387618283,0.27715751614132822,0.99968970496257303,0.51613319470867447,0.46384160559205645,0.90071638906040719 +0.64019945916917109,0.083880918490503609,0.014673582286673703,0.93527608055813904,0.50334605718037295,0.58065118099725477,0.90010773799194821,0.40677096486187847 +0.14980690865529567,0.21879624052434443,0.3383333607546537,0.63703710406742797,0.51944877591564798,0.30541143964406736,0.068279547082978453,0.13504003659347297 +0.46543616001525123,0.24360397691905999,0.4569859247211312,0.48745270216499992,0.11719329537449574,0.18659198836429958,0.088591415467569878,0.80785550422589203 +0.5203501400211421,0.078227991059971691,0.3967793082294383,0.20112073222820692,0.33071669250705105,0.89460186941146091,0.44242173111097094,0.64280918848416702 +0.078158529302830404,0.057433465432000785,0.61977782222415623,0.60159456416898138,0.81945147287972009,0.88949149256402904,0.44256229774907319,0.095515370151494139 +0.66198591493673697,0.26553021835494561,0.93838284941652195,0.44829690078027151,0.90161272992592623,0.12719461064858373,0.52885140773545647,0.028437434718284749 +0.23218535415322095,0.35057799013979196,0.82654896701313374,0.29602537301849774,0.5609117377016799,0.82575640590929922,0.41907906131370776,0.099734135471285845 +0.263516248128428,0.17039079021163439,0.68131728348481235,0.13015771941664511,0.52129640759000062,0.31092457857323896,0.95568319550503222,0.60032331939653172 +0.11959036821517344,0.45023225013144363,0.61877374756693559,0.36408222776856525,0.97594868413947289,0.61418443054373573,0.35318423217077133,0.16176635301398734 +0.27131467126368103,0.45826215243268842,0.099335544554979727,0.51421571104676711,0.35165527698100363,0.63080141332765272,0.92949954393035605,0.58346198539704364 +0.75418680278459116,0.57507964757200847,0.026746021581144808,0.18986895896498379,0.68655319957786254,0.30666133449057698,0.78901391069445026,0.98260441997100079 +0.70506762706878123,0.19638641225687037,0.28279738346229899,0.13610867799891713,0.39988482787193924,0.73655505226419238,0.61022858042433392,0.53982948473622838 +0.5364029977431507,0.49993741005092607,0.50438661073282298,0.47108570008705131,0.41399738016055054,0.48636951308340293,0.92242047650162529,0.6620380845292686 +0.89537317370329994,0.44606231161437282,0.26672879751112566,0.079200550145315907,0.29024629216737075,0.53529711948052727,0.50184417603338571,0.45302197204632366 +0.18360987096329928,0.32945974256008392,0.76330987594779465,0.0019189673453937628,0.58893257660505716,0.71207978554267881,0.5199031429024652,0.25722429221210275 +0.43242169164747624,0.42860117700126021,0.18381516568864587,0.26345822194136403,0.56283920951444366,0.62001785090349115,0.34483490378826465,0.91855869465229811 +0.23861845158652262,0.31726648876831853,0.37051571833054253,0.42892097601040535,0.33729565610681878,0.77680169087175155,0.377390729781792,0.072142987020896499 +0.84899358059453212,0.73848916663753161,0.81892793242302875,0.82298476451471259,0.4737007484160658,0.94240703348959387,0.64543660570230066,0.16712184443718126 +0.51302891799220351,0.55025789541546477,0.040998120973373742,0.029511115918076314,0.89801241977212842,0.42259591423166176,0.13845550710277954,0.88842184331064811 +0.73252483912896715,0.82333936893582937,0.99990415461188176,0.61465768135713394,0.2037860726237124,0.7278518499352209,0.56732964243598549,0.4049969252153317 +0.20511197426770744,0.32405797067182129,0.2823869297088254,0.11935552529507121,0.47326605380078668,0.0068486466150113889,0.81314588783246244,0.83268335893763523 +0.63505965103507067,0.066755431275853927,0.34009399288588371,0.46122143558914108,0.84464997350101756,0.11313043198878581,0.051678455756629216,0.021960453240060573 +0.1201545118817845,0.169332271379941,0.35810625984311784,0.84782666332974466,0.12780243475728348,0.75379862307493828,0.1827755780116328,0.67584947919513438 +0.20650234596239636,0.22848670989831199,0.78618876651725889,0.86042684791796598,0.9142992336903758,0.93619308043801663,0.69092288695200565,0.28414837407897386 +0.89303030414726214,0.36079475583669152,0.26319701618441405,0.39482166732266144,0.1951946060269224,0.89886725099347176,0.03144113423571282,0.39641464383877745 +0.1540399117674035,0.48697197595130615,0.3760069600199164,0.031259192090522858,0.049146560310475929,0.93393399461294369,0.25915064751333017,0.83210305899945258 +0.81814471361521679,0.80685904008690446,0.29479639437137484,0.27390796372475834,0.76383199592698758,0.025340844430415954,0.6210257850787535,0.48390300507788708 +0.15822101814143219,0.60241929930088656,0.35993747692045519,0.58598217001847452,0.62575160665788587,0.85645654158136209,0.70257090805642253,0.89400610988684792 +0.88411862212519543,0.44680777722871823,0.85302783572318541,0.51246289306602699,0.74636524960673201,0.13075653858888614,0.48810142638732812,0.47066692368482754 +0.43500062573468767,0.33276606108584539,0.0043503671805481536,0.68706094458541622,0.79985100385606978,0.14144280502176587,0.20916734954896721,0.14018008647456093 +0.73536056263394611,0.68287300743876123,0.94126718150280231,0.14717243128499669,0.00086513592592085686,0.10303821767059962,0.57552133553919127,0.25257163603811594 +0.27082663649055116,0.84297478083338451,0.43696055554872415,0.32497711725867218,0.26082803054396297,0.7398051345818657,0.2594321201544737,0.39116953555338652 +0.31654454628864481,0.75614665911664081,0.36407362393620601,0.84744872256712289,0.27935136815580286,0.7326104648406675,0.2906406400472249,0.89936777212349073 +0.96749833169767074,0.4937502314891552,0.71243974909587771,0.89595856160170706,0.43965468854205259,0.97347310944553023,0.93587641523256582,0.097493100709422964 +0.34168593110030232,0.53528798774960518,0.63430273825273431,0.28120678612555305,0.94791628851677923,0.52200089067470934,0.49730514703506901,0.24511882499303184 +0.84915609663078273,0.20718612690320115,0.054277789564034951,0.72962755095852094,0.72079282063465921,0.35776490664193816,0.72378342151822272,0.17054552979609794 +0.088214735325851668,0.59975752470398724,0.77102307481947874,0.48057160507284968,0.68322377027038217,0.54536303205896364,0.093961440084399883,0.033122574212533729 +0.26299044114663561,0.66628566556398838,0.0024049354089710206,0.12776250549730461,0.41189772087839649,0.15346375265719381,0.77424916119398257,0.10202204791392644 +0.29861798994087607,0.26345159252448236,0.73145975465341395,0.33839036147055912,0.13833264589690686,0.60198792066589313,0.37633881457370344,0.061030717643603238 +0.23507915888740383,0.73017085904660628,0.3886385676930314,0.98653414338232637,0.22440119712388651,0.88665954879561659,0.89648886318961984,0.81013471525594605 +0.60388135657950204,0.24039289302414213,0.63874227154469654,0.24354715127340387,0.40695808730500538,0.50512838914829428,0.23086976500973411,0.45599089792523506 +0.49669140056929428,0.80494506420395251,0.60737558130261049,0.068334458192574221,0.4461301414413632,0.74236635699702891,0.77054399664301421,0.72241847848767426 +0.60293630680270371,0.69276296230436385,0.078757972302510471,0.46149749982366406,0.16814850412565285,0.88263983132508872,0.8860061988943676,0.11085978684752795 +0.59628173662958117,0.3223188874491727,0.83235124702589858,0.31041338190083312,0.037710722864766955,0.99455467878947812,0.92510438852644239,0.11084741465637471 +0.76022534932309049,0.92091786594799074,0.093295547264289125,0.041981999095378697,0.8467714659365394,0.70125208085742485,0.4592278979043406,0.93519512149670758 diff --git a/data/borehole_y.csv b/data/borehole_y.csv new file mode 100644 index 0000000..3a91ba7 --- /dev/null +++ b/data/borehole_y.csv @@ -0,0 +1,80 @@ +16.841943481641277 +147.98987681904245 +180.84286662842061 +25.533591328603766 +31.807360190425864 +60.588513865214203 +67.643778067397037 +115.24528099323008 +68.813745944502827 +52.393103767584336 +22.363285265582384 +162.34444801908407 +32.246090353265252 +23.543673289943943 +26.585244581792463 +91.535923945196245 +153.71021477653295 +93.212360179499427 +131.24913149131055 +14.21503362103139 +97.238304300770253 +27.546701911349423 +35.950732716963223 +83.406065633835965 +40.903365364035771 +72.797265500263677 +133.46911939606625 +93.15716113904918 +41.990163143801958 +109.0223826883678 +34.394544680240834 +98.253135028452135 +35.455696599719253 +80.450122024444681 +149.83276361700757 +113.91569578614394 +66.578724619233498 +36.416462055643692 +26.27475589695057 +78.233884562508521 +41.382498668321062 +162.87478082500809 +45.136377913901519 +34.000537425963699 +88.505420682200622 +55.490142242971658 +21.7353034649325 +154.9864233861731 +80.535419382369952 +59.113737191301553 +132.54667184275937 +54.714290035604819 +123.32390572499261 +147.09088804306151 +103.02944869128422 +33.498391584372762 +40.001604539550719 +79.999770436523562 +149.0513531979056 +37.10613681506841 +76.139193316093326 +132.54995371821491 +53.779941155846302 +14.232777429002541 +66.286668244519234 +29.399766668553198 +25.105866826387849 +136.15663452360963 +33.91115128633497 +101.84262347516136 +80.469183685294226 +68.849061240365984 +64.223387389709146 +149.89136914088795 +113.16486517566811 +151.62276838851952 +48.895170776256492 +119.68545996976678 +62.458862552317214 +61.320805917309187 diff --git a/data/borehole_yt.csv b/data/borehole_yt.csv new file mode 100644 index 0000000..a623b2e --- /dev/null +++ b/data/borehole_yt.csv @@ -0,0 +1,200 @@ +45.489337121460018 +21.405352741476424 +185.9052669499778 +136.90866244412595 +84.421002257094202 +25.053020381180396 +21.227026521085389 +54.919261488775362 +38.802060005088272 +86.967344262234278 +184.40243699124721 +76.557260368927629 +21.707111123230874 +31.589545072174484 +37.142212918071067 +174.68838034511717 +106.85926716801386 +133.29529845765336 +114.7789554071702 +131.85074550178388 +19.527583815129812 +73.565478049404177 +121.54388004033021 +104.49525377500417 +55.540739593739659 +81.911469871378202 +122.20329785400116 +68.341404175883198 +126.17105666475069 +49.11518552520667 +101.00343447661814 +100.13162754635255 +64.84005909911123 +76.744502848442636 +92.330918252615973 +44.603942307727586 +63.977821989655126 +137.27303561057627 +40.266821053130009 +70.30138745150488 +23.022029350664447 +107.29891277446352 +27.916818818940392 +106.30269815772684 +116.6977902931413 +81.618663669000142 +70.62999189663239 +29.364160298681764 +100.95236897730423 +45.842878745002025 +59.436478328182517 +25.398502916938099 +17.031268983019771 +41.547432809602029 +17.527502326787911 +109.78136849968195 +52.16840759407021 +69.933713085506625 +21.386508396989456 +89.085583773827409 +36.995935204447278 +38.202557608387416 +38.834237360596269 +45.764220486586808 +39.66010301590493 +36.417667014535056 +20.575720373145447 +168.88579548296016 +59.141343679753156 +116.26164071494566 +70.753203268254154 +47.390306274577895 +43.127847877582731 +106.2611569504068 +33.862089168425975 +45.297467042960648 +24.383690916666833 +124.30457332019562 +55.529209277873107 +115.143001947375 +45.141530197525839 +30.370609135176618 +28.635935626627727 +20.922103259580069 +167.59600421007042 +120.67129657234585 +154.94171331098437 +114.06543631154662 +203.61854816597065 +90.411148632991143 +152.94679064162821 +48.895215469017614 +59.218376367808176 +45.43876748886926 +117.89163832042404 +77.234273247392636 +64.119257706175659 +46.462105046121032 +136.35865267767602 +152.96058082411662 +79.979529090827342 +172.0608530624439 +44.165991734177311 +153.70864580927903 +105.34637680141833 +12.304677455979688 +49.274043156386007 +45.728276313286202 +26.71395027347906 +21.698348493046495 +20.521336348694366 +104.36555948401065 +25.033986740991793 +37.828030122505993 +82.971792039662688 +194.41898655054814 +52.696156964708301 +142.5323393836203 +139.07672175140297 +46.741092693320361 +88.093696457204757 +55.330511547931046 +67.758917141707542 +35.420109092091351 +64.086456649904918 +48.10410651003852 +52.612005006642043 +40.311440598994913 +21.772776978232049 +106.56118185420202 +42.660319671695319 +99.313845113496598 +129.86333417405572 +160.47283750151246 +130.22176244999326 +141.14710127023122 +55.916287966493613 +135.72944495424713 +22.339626199633447 +19.669595096264285 +92.189140217167818 +38.592413524596964 +55.486462193214969 +112.17200234821834 +133.27309197601198 +56.245416339935318 +36.567616503201769 +85.079925018229943 +38.974007992103481 +41.771995051822614 +34.522509229970282 +20.70179043358473 +106.60361023300661 +211.2158038357428 +30.324129957295867 +89.417429973534624 +38.246785518268041 +94.161518213975285 +55.380742017626147 +19.71305588135246 +97.107943223667661 +28.282221651081514 +33.076569269142553 +24.251667106781191 +34.898576965122793 +104.06359803301251 +74.529139525557895 +66.809621036120163 +110.14974321989844 +22.146050138795538 +60.702693857949143 +31.901505665449573 +107.77248075930829 +76.706382754450914 +97.609794468559784 +35.052123482321683 +115.13147659996028 +33.654450146730959 +30.569570338697929 +129.98158171976885 +22.449167940150428 +128.80475628062854 +27.298491767936138 +156.53044959339363 +79.864875448779372 +101.06301226500194 +37.797996255295452 +58.355199862493535 +115.2412330772008 +43.054404379670729 +127.45006273206197 +25.921473857701308 +33.942776861764806 +38.699564763419119 +36.628790313821561 +85.390428201308765 +47.847390526708189 +56.798195030840667 +48.063688068517102 +90.206930186932311 diff --git a/data/branin_X.csv b/data/branin_X.csv new file mode 100644 index 0000000..cb4c01c --- /dev/null +++ b/data/branin_X.csv @@ -0,0 +1,25 @@ +0.056449627378410791,0.26943178240238125 +0.435784572810065,0.99276704745212585 +0.58513364076693897,0.50139523202364711 +0.17296674648074872,0.88802135612900668 +0.51086597720977056,0.45532769944704887 +0.24407090662753825,0.42926538696100086 +0.36771272054128568,0.32500168639495103 +0.29612710533183523,0.19260851685293484 +0.72378100383836252,0.67431072738879794 +0.4676219187647323,0.12140279166878087 +0.23174624354457879,0.82229097731628653 +0.79147370617064128,0.22698601379669112 +0.14887834401031938,0.071427071950469698 +0.86927748521729487,0.39321523001171804 +0.60016791604538133,0.029030199829788779 +0.92097511209485849,0.092733837920627393 +0.80444231847874037,0.93737075795896674 +0.98604909686289544,0.7971647286307938 +0.55019182917366938,0.74051020555790859 +0.036850978649520041,0.71683862731609793 +0.66127923812261569,0.84802286246194014 +0.34131164744698639,0.61784671831119309 +0.71981481426686456,0.30854726749829542 +0.88904932273205839,0.56664411876284182 +0.10561412433242597,0.55904510843487853 diff --git a/data/branin_Xt.csv b/data/branin_Xt.csv new file mode 100644 index 0000000..7a87967 --- /dev/null +++ b/data/branin_Xt.csv @@ -0,0 +1,200 @@ +0.44217227295272243,0.78901759462748711 +0.92883517101843105,0.17042606197312338 +0.85842780900467719,0.78008118189651876 +0.61572682952990154,0.5515917799926181 +0.090008414040849916,0.066436011213980128 +0.36395440971819548,0.56382552625339921 +0.59603640790456369,0.46942500529135189 +0.87341965920427,0.27891743261638047 +0.91470114458629015,0.12875580368063 +0.72330765612661896,0.94299476206174915 +0.0071488025233706898,0.0070146754608084727 +0.65611115848812707,0.55445433290798118 +0.94651428377004299,0.48982475344992249 +0.72627075322195833,0.63531945212934016 +0.84154166038552913,0.87694418203435021 +0.58085516499613843,0.81587202329207398 +0.69891279386383909,0.75791265396917706 +0.18119157938347696,0.60311037652189481 +0.14234958597001457,0.60032353632313884 +0.7626038059122805,0.086820184077650908 +0.11186104036238775,0.74418916658213197 +0.32062256074155193,0.19989839643733331 +0.75007045378192894,0.19795292432136735 +0.83072372731131638,0.82198204613595083 +0.053916963829623965,0.52273796131998462 +0.06484583577276648,0.72666405218749575 +0.0058732290974637946,0.94800955054377922 +0.43537510251961531,0.71952125426558322 +0.20064650389128846,0.99076045724539419 +0.084426903612385806,0.93958462242323071 +0.61830845161439763,0.095757230018457151 +0.14303257531275559,0.50325844688795984 +0.24117907360732416,0.82757768573576806 +0.15501613891033339,0.45614272588285909 +0.06757893150272487,0.19169136032447587 +0.74394765454437939,0.42198575465404942 +0.79566433908098122,0.022432735582047836 +0.45526027961659254,0.22670543264604381 +0.041969134457666701,0.76797985085979048 +0.75152717450358009,0.70027393045195352 +0.96431894999931078,0.71933691302467628 +0.0083535630910052117,0.99577654434371621 +0.49107197448063877,0.61224636436388424 +0.3034401566374697,0.3576014547910763 +0.61338323716802057,0.7400432125811105 +0.49722147209535672,0.5401526792567648 +0.42882614983424183,0.79263106477152101 +0.81662464835723969,0.99006174398007007 +0.051079393166953757,0.10305214156860165 +0.088007351038719706,0.97598737516737288 +0.20669571756588578,0.88706819131737635 +0.78471862813803983,0.89799767861774571 +0.018750356594929407,0.0207711338308052 +0.88592018667820327,0.929288750591311 +0.91997170735839207,0.028439447273798768 +0.19307172125323935,0.76951207043301029 +0.36299400492802036,0.026775567824404156 +0.37115676600116632,0.14199663417590525 +0.026832884623721798,0.9247216401204863 +0.32301841517045682,0.33125143832922455 +0.77341326510301378,0.16933829864247008 +0.055361531615479187,0.94052206655136317 +0.79823699901359413,0.1706929874189751 +0.058114782212122629,0.25603132525880468 +0.83719249918581173,0.87951861569513778 +0.55581040738549514,0.16028806653964967 +0.13601257793440014,0.97113546286683772 +0.038006819017857851,0.72798416175824521 +0.78764745715918405,0.82311590735054008 +0.15091833908836205,0.51382553647050577 +0.78473567086699736,0.39662721649426991 +0.11213203725839793,0.084201672054377896 +0.76902252303557894,0.67020564886877032 +0.16420304940658359,0.34089838593329702 +0.95308665451471919,0.42939797005730729 +0.37560315982637904,0.46861740632813664 +0.90424178672680777,0.34788392337639784 +0.77691560498790158,0.094489469117504776 +0.15774909680215321,0.069962651281692656 +0.80431459508952852,0.77203192874816617 +0.069217453967550524,0.70778546282554078 +0.14042714760127439,0.73941084152373415 +0.027547748316445131,0.64153621751536549 +0.87848815126631119,0.18961070438471372 +0.63460597662510088,0.55744114580299786 +0.31876176905004472,0.59772052057089065 +0.1714939792670247,0.830576390685829 +0.85561387892704721,0.60780675640160864 +0.83868485053539532,0.84832295939756008 +0.58414593733301567,0.63177931856820291 +0.71935086663360759,0.22990871470043284 +0.89805754888889422,0.26428062486215154 +0.59442668440972568,0.71810648571140645 +0.28522064665601476,0.88445798380784091 +0.72809476916014881,0.13816718483617652 +0.53822096846412115,0.27485705761574097 +0.438360416048446,0.78083629079733796 +0.55920798299151964,0.055494012131647397 +0.99823093169026944,0.77586784324529623 +0.11198378257637132,0.85288615748942231 +0.40175513784851757,0.95757972182259288 +0.90994983631532522,0.89179018757098727 +0.88044050020888409,0.056091759467511104 +0.36913982376475396,0.49712887776359471 +0.903073595780708,0.65300006886845552 +0.13684116946487057,0.4955818059240078 +0.35770027943443372,0.69926323197080609 +0.71928705467098963,0.73161976946167728 +0.628561338397021,0.39341656595279551 +0.1216973572747202,0.041119778717247923 +0.37262318934523342,0.68044401276901534 +0.54751122917157846,0.75576396515555433 +0.99789773101401624,0.14669331613891079 +0.38516255091568152,0.46666822201279567 +0.81840005805064731,0.9796538701898152 +0.29071799279316068,0.1311310141822658 +0.42702936816549431,0.026150618155328154 +0.84248706515183547,0.016683831236958824 +0.53705704856426772,0.66264776493275201 +0.41947305364252541,0.81743342743120073 +0.88046155051935504,0.63867680496949697 +0.53664959001243018,0.25602224163988496 +0.36451494535291451,0.68500917375131598 +0.52413037160987364,0.11190905368843318 +0.76825221449813075,0.85876164123686083 +0.35912254707983404,0.28465627959627482 +0.26775242430060964,0.21837134941219793 +0.89273880089008062,0.46320701864374059 +0.63641694396865356,0.49444250824172298 +0.87211383416274912,0.031539296562799746 +0.87076508373449524,0.48052446873597876 +0.64192779885974338,0.095065021673584771 +0.8231433992053161,0.39175164168258059 +0.33087687520954856,0.32143638706046096 +0.018411316809590406,0.87319854586237666 +0.92155373656305606,0.20581596094541221 +0.55188496454699454,0.48585080808920156 +0.41974822079656038,0.57661563134774729 +0.5446928112307059,0.24089044474948751 +0.34466111301214686,0.3506568223103127 +0.2338606360193981,0.46557562048313106 +0.88450809440822453,0.50715507176959351 +0.14045160316228333,0.48509668009589624 +0.360258546636122,0.96076386513093515 +0.56134634728644706,0.5340975518039186 +0.50190811473723018,0.36381673371817891 +0.37477072331755812,0.96508344731737683 +0.39084217320433012,0.60933856666296793 +0.65122668995859001,0.23352871928950392 +0.64063349602723219,0.99864098121415945 +0.46209347036148074,0.12687596706969639 +0.16318878439176177,0.15547728302456199 +0.23066575492329078,0.89434669536302658 +0.49417315039274734,0.4662367306790135 +0.65456402562107496,0.71058961869572645 +0.1697284684164404,0.62702325115012358 +0.51030799374590718,0.39954206433522288 +0.50285327550788073,0.73315356431785661 +0.58604851504040334,0.13354311518037121 +0.96578812327744989,0.11193849270848499 +0.083077899070327921,0.52442785427259542 +0.93376825844193123,0.43383100103336025 +0.94009652569179214,0.43501246449891506 +0.47880682700956556,0.22037005458370629 +0.90536028488981424,0.31812781713096183 +0.66675349078305979,0.3576766775612692 +0.85944984250491008,0.82166195954009547 +0.65694772385011224,0.55353558851297924 +0.020719990036436653,0.76842448392307683 +0.78046209677104172,0.96397398901165343 +0.36895580318514731,0.18426348727066455 +0.47832392489354769,0.92493425117829353 +0.746115337111384,0.69967584280146111 +0.3630711474317343,0.10918933553520471 +0.29246668272083998,0.34602499230663553 +0.48161928414880717,0.8415320136043527 +0.812147080924843,0.77126044170551766 +0.078417122269472839,0.41628642897226109 +0.39150900051524684,0.26150089118819875 +0.74408763066647698,0.45085208448287306 +0.71285997807277601,0.2753306296098561 +0.12898795235915839,0.34480539498689122 +0.51982539467826849,0.91287365307545121 +0.62585087258273753,0.88634058763789325 +0.45935240399237376,0.13251939589417994 +0.086633350583403601,0.65651377323830051 +0.62771050020820296,0.6797193086843597 +0.083760739416245067,0.052430589482174716 +0.14747024041364887,0.43763308889551578 +0.24296988898622252,0.13258663209207755 +0.46523994895009613,0.24497744473370631 +0.50496531693052349,0.67757190725172611 +0.71404821792359652,0.030626970897620343 +0.94777396510898293,0.44567518604116774 +0.17306471962612224,0.68298843561453237 +0.36841050863965308,0.58994133912522584 +0.45936162343167131,0.53382226460184778 +0.65002328895004424,0.10864465814886792 +0.01942201492659601,0.76871317861074151 +0.57692470809450225,0.48999684806854216 diff --git a/data/branin_y.csv b/data/branin_y.csv new file mode 100644 index 0000000..560b783 --- /dev/null +++ b/data/branin_y.csv @@ -0,0 +1,25 @@ +121.48190599474232 +132.03742032150166 +34.640118861733328 +10.415272536214367 +18.716568889555884 +15.906681327590452 +18.468017595964984 +34.461435671415757 +99.827412317368413 +8.1210955299534646 +23.477267434509667 +23.007704402098216 +107.58906048281199 +27.102349910019207 +5.3393982799416833 +2.511675114913217 +182.0881798494803 +84.874781734255166 +79.985148463123508 +31.302134533810641 +142.46598142595028 +31.479573472413108 +30.832792916092949 +51.656097368202694 +21.531893250415575 diff --git a/data/branin_yt.csv b/data/branin_yt.csv new file mode 100644 index 0000000..d5bc6df --- /dev/null +++ b/data/branin_yt.csv @@ -0,0 +1,200 @@ +74.843579760045472 +1.7544465779275278 +114.28579756826056 +50.453527769751005 +158.6967763729435 +28.599798564014584 +31.467022034411301 +14.403513807506229 +2.6825094424612068 +188.61021452089682 +293.10424256092097 +60.157534311067643 +26.220073670456962 +89.824755787706778 +150.95530498884841 +109.38481199029145 +120.96940611068656 +5.3145755385084739 +7.6003704831372971 +19.523416724730112 +2.9567897813343507 +30.382012367004243 +23.094742414914815 +134.96622414937272 +55.612760190598337 +16.920324174699644 +19.269921963420138 +58.358285147812765 +32.942124488014912 +2.1549724202027205 +5.9441436861106007 +17.16781400176702 +27.534632546345016 +20.253103179187576 +136.37857600542623 +46.899598143114957 +18.337880210874808 +7.5636227885014709 +22.018924684017641 +108.04476731154379 +68.98220060255008 +15.12921001492143 +41.909190885143175 +20.545148639076547 +95.676452528928152 +29.949504417789399 +73.800083421600107 +200.40672548773304 +188.04909774873107 +2.8243682191064057 +21.46636389419038 +170.35745823117904 +268.46799724225298 +156.16087820694523 +4.6838913414182315 +7.7519315635634758 +42.832838905502292 +27.149790972970873 +13.638277537630941 +21.127270648188279 +21.128215949084911 +5.6394203181091633 +19.299044250963043 +124.20339602325819 +153.04175931500077 +0.65758895751351965 +7.9796925681660928 +29.092687888623527 +143.5678155608615 +14.241755407452708 +41.552052432277094 +131.4203526021173 +99.165666085236595 +35.227907253783002 +17.059931631385947 +21.681773614407177 +15.126515965553352 +19.033239534844853 +102.3799429935243 +124.49174212939948 +17.259796048923036 +1.0466142387926283 +49.691956805548578 +8.4384065003504798 +55.982964334384597 +26.194976656049221 +6.1057119046810744 +68.818817497723671 +141.78626940099844 +60.495551197664369 +23.901592192833501 +9.1815347437450399 +84.558432428150567 +53.822568781101786 +19.915920605068717 +3.6387192963507644 +72.354006056967876 +2.2712283018044737 +76.825451033806331 +0.5579069250375106 +112.21425064188696 +134.89570676722005 +7.3202244752462526 +23.423387151446242 +67.256753354792636 +19.757650938481376 +44.504767589764732 +115.64482312430772 +26.809730581531404 +138.17254592886709 +43.98988810442895 +83.527889279501338 +2.3775041112671422 +21.506014341859604 +195.76858317019244 +43.748752558020378 +24.72903941989486 +13.340578360446653 +58.146109810652085 +78.288232837112972 +69.737809739016612 +2.666260103950993 +43.483715650836018 +1.4507470992022284 +157.77106357519907 +20.178116272459512 +34.811780781685741 +32.200089427219545 +44.19862346443081 +9.0466714935021049 +39.862390363345689 +9.2000532009673339 +35.785707278227378 +21.126440007447567 +21.019192719648171 +3.2340494896762362 +26.670528817653093 +32.810146490984067 +2.2533983676230998 +19.687982948920411 +13.573027407363762 +41.369841356117433 +20.137397926604343 +100.42786510877386 +36.095789029143901 +9.1986662306091507 +106.45952576761508 +35.754341664144079 +15.180384164131263 +193.18629458481399 +8.8765159853802924 +75.483507957521809 +31.990754834382695 +19.496018400217302 +98.034024346360582 +4.213680665779405 +12.455148129458184 +69.456331600923605 +2.3832707464308625 +1.1367482733206735 +37.287457842101858 +20.258502631327715 +19.466522715025839 +4.5156718692658382 +12.131519978006285 +29.493918197026986 +127.13147299354395 +60.143737612509305 +32.600075922630147 +196.25318716954339 +24.132185579956026 +119.55809342221517 +107.8649964976714 +32.189697029558971 +21.216594854418858 +94.867147336503521 +122.80650435695073 +62.302468066013468 +16.791733962988104 +51.620193077591502 +26.9996905339049 +48.312820279188884 +124.93514984608022 +145.97283878810975 +9.131325236586088 +16.381457551458439 +82.963889873493784 +170.62076328631503 +24.7749250785327 +53.086259176394805 +6.292965348733885 +56.731552760542399 +18.513084006404185 +19.790674888303577 +3.0009108323109617 +31.631426320643641 +27.725178135794117 +10.439999349696391 +33.270072843857477 +31.230154962590639 diff --git a/data/hartmann3_X.csv b/data/hartmann3_X.csv new file mode 100644 index 0000000..aff192e --- /dev/null +++ b/data/hartmann3_X.csv @@ -0,0 +1,40 @@ +0.62302414207256118,0.18209410987328403,0.52144420461799867 +0.3442912357561066,0.16028101711150675,0.43857616247630904 +0.81981149348128712,0.62440319277179912,0.40795864870039217 +0.7207595187573238,0.043143874893617984,0.20312605399684439 +0.77236535800629058,0.38585672372435426,0.70352064117875313 +0.38893186082267911,0.51789686669690016,0.23415617193985661 +0.7757997975742229,0.21645018505697,0.16281887847369286 +0.31436005519873228,0.90964801746472457,0.14980673101613307 +0.94194755533797569,0.091866258622931934,0.65482045033830361 +0.9937164065970624,0.53982503223719802,0.92995102493578585 +0.72888624121330359,0.83949249597755293,0.89988425891679036 +0.91829086685062555,0.35565582670753659,0.069841402215361753 +0.36337202001477947,0.42364359141248692,0.60254431664221131 +0.64453698471548748,0.79477451846926217,0.020380323033084274 +0.0023631273989765779,0.33615419894449566,0.34100882770776619 +0.13001334758062916,0.47134013952059411,0.19780059828775781 +0.29803186165595003,0.81409103872026944,0.95759702541293978 +0.22466308865432807,0.28975460803315523,0.86810397829982389 +0.55237313555042555,0.64495175485145617,0.57394262234440063 +0.24396858015088693,0.73574736177938083,0.3180031822416145 +0.48702671673577147,0.88044179512065701,0.63737100822620341 +0.062363379108193208,0.93587218201782463,0.49184815523713626 +0.15415257422677614,0.2433948640014883,0.59165359447409049 +0.69157868085579344,0.48503864604789915,0.45957981215440552 +0.043482098686570346,0.56804896500644964,0.74226369922795776 +0.58536531932480385,0.13122076106558939,0.7527764490492127 +0.11027312468110409,0.066280685539309672,0.92322795539424596 +0.86967106635665081,0.26964191996904352,0.81127272146492968 +0.076401429582656222,0.67560944505928655,0.096793508905553508 +0.26882937042622473,0.11285537394320555,0.040708525479336871 +0.66329952382663482,0.6704794046575786,0.83991382834762263 +0.53310421655046791,0.30087674479298804,0.29055631031782514 +0.83507944083239705,0.97170933398282722,0.35501428903871257 +0.18449421008955752,0.76886989323354338,0.6858544722328741 +0.89051742724519389,0.72248597422535166,0.54402130267107762 +0.96784204218643466,0.87379469053385184,0.26801205628199348 +0.43831977965436647,0.00010494752836333054,0.37708165623090401 +0.41541184738830622,0.99950952470963694,0.784668759631858 +0.52007607108947396,0.59329842826080925,0.12162952853081421 +0.45742772104191909,0.43725799595264636,0.9965310968853307 diff --git a/data/hartmann3_Xt.csv b/data/hartmann3_Xt.csv new file mode 100644 index 0000000..5d5c526 --- /dev/null +++ b/data/hartmann3_Xt.csv @@ -0,0 +1,200 @@ +0.44217227295272243,0.78901759462748711,0.92883517101843105 +0.17042606197312338,0.85842780900467719,0.78008118189651876 +0.61572682952990154,0.5515917799926181,0.090008414040849916 +0.066436011213980128,0.36395440971819548,0.56382552625339921 +0.59603640790456369,0.46942500529135189,0.87341965920427 +0.27891743261638047,0.91470114458629015,0.12875580368063 +0.72330765612661896,0.94299476206174915,0.0071488025233706898 +0.0070146754608084727,0.65611115848812707,0.55445433290798118 +0.94651428377004299,0.48982475344992249,0.72627075322195833 +0.63531945212934016,0.84154166038552913,0.87694418203435021 +0.58085516499613843,0.81587202329207398,0.69891279386383909 +0.75791265396917706,0.18119157938347696,0.60311037652189481 +0.14234958597001457,0.60032353632313884,0.7626038059122805 +0.086820184077650908,0.11186104036238775,0.74418916658213197 +0.32062256074155193,0.19989839643733331,0.75007045378192894 +0.19795292432136735,0.83072372731131638,0.82198204613595083 +0.053916963829623965,0.52273796131998462,0.06484583577276648 +0.72666405218749575,0.0058732290974637946,0.94800955054377922 +0.43537510251961531,0.71952125426558322,0.20064650389128846 +0.99076045724539419,0.084426903612385806,0.93958462242323071 +0.61830845161439763,0.095757230018457151,0.14303257531275559 +0.50325844688795984,0.24117907360732416,0.82757768573576806 +0.15501613891033339,0.45614272588285909,0.06757893150272487 +0.19169136032447587,0.74394765454437939,0.42198575465404942 +0.79566433908098122,0.022432735582047836,0.45526027961659254 +0.22670543264604381,0.041969134457666701,0.76797985085979048 +0.75152717450358009,0.70027393045195352,0.96431894999931078 +0.71933691302467628,0.0083535630910052117,0.99577654434371621 +0.49107197448063877,0.61224636436388424,0.3034401566374697 +0.3576014547910763,0.61338323716802057,0.7400432125811105 +0.49722147209535672,0.5401526792567648,0.42882614983424183 +0.79263106477152101,0.81662464835723969,0.99006174398007007 +0.051079393166953757,0.10305214156860165,0.088007351038719706 +0.97598737516737288,0.20669571756588578,0.88706819131737635 +0.78471862813803983,0.89799767861774571,0.018750356594929407 +0.0207711338308052,0.88592018667820327,0.929288750591311 +0.91997170735839207,0.028439447273798768,0.19307172125323935 +0.76951207043301029,0.36299400492802036,0.026775567824404156 +0.37115676600116632,0.14199663417590525,0.026832884623721798 +0.9247216401204863,0.32301841517045682,0.33125143832922455 +0.77341326510301378,0.16933829864247008,0.055361531615479187 +0.94052206655136317,0.79823699901359413,0.1706929874189751 +0.058114782212122629,0.25603132525880468,0.83719249918581173 +0.87951861569513778,0.55581040738549514,0.16028806653964967 +0.13601257793440014,0.97113546286683772,0.038006819017857851 +0.72798416175824521,0.78764745715918405,0.82311590735054008 +0.15091833908836205,0.51382553647050577,0.78473567086699736 +0.39662721649426991,0.11213203725839793,0.084201672054377896 +0.76902252303557894,0.67020564886877032,0.16420304940658359 +0.34089838593329702,0.95308665451471919,0.42939797005730729 +0.37560315982637904,0.46861740632813664,0.90424178672680777 +0.34788392337639784,0.77691560498790158,0.094489469117504776 +0.15774909680215321,0.069962651281692656,0.80431459508952852 +0.77203192874816617,0.069217453967550524,0.70778546282554078 +0.14042714760127439,0.73941084152373415,0.027547748316445131 +0.64153621751536549,0.87848815126631119,0.18961070438471372 +0.63460597662510088,0.55744114580299786,0.31876176905004472 +0.59772052057089065,0.1714939792670247,0.830576390685829 +0.85561387892704721,0.60780675640160864,0.83868485053539532 +0.84832295939756008,0.58414593733301567,0.63177931856820291 +0.71935086663360759,0.22990871470043284,0.89805754888889422 +0.26428062486215154,0.59442668440972568,0.71810648571140645 +0.28522064665601476,0.88445798380784091,0.72809476916014881 +0.13816718483617652,0.53822096846412115,0.27485705761574097 +0.438360416048446,0.78083629079733796,0.55920798299151964 +0.055494012131647397,0.99823093169026944,0.77586784324529623 +0.11198378257637132,0.85288615748942231,0.40175513784851757 +0.95757972182259288,0.90994983631532522,0.89179018757098727 +0.88044050020888409,0.056091759467511104,0.36913982376475396 +0.49712887776359471,0.903073595780708,0.65300006886845552 +0.13684116946487057,0.4955818059240078,0.35770027943443372 +0.69926323197080609,0.71928705467098963,0.73161976946167728 +0.628561338397021,0.39341656595279551,0.1216973572747202 +0.041119778717247923,0.37262318934523342,0.68044401276901534 +0.54751122917157846,0.75576396515555433,0.99789773101401624 +0.14669331613891079,0.38516255091568152,0.46666822201279567 +0.81840005805064731,0.9796538701898152,0.29071799279316068 +0.1311310141822658,0.42702936816549431,0.026150618155328154 +0.84248706515183547,0.016683831236958824,0.53705704856426772 +0.66264776493275201,0.41947305364252541,0.81743342743120073 +0.88046155051935504,0.63867680496949697,0.53664959001243018 +0.25602224163988496,0.36451494535291451,0.68500917375131598 +0.52413037160987364,0.11190905368843318,0.76825221449813075 +0.85876164123686083,0.35912254707983404,0.28465627959627482 +0.26775242430060964,0.21837134941219793,0.89273880089008062 +0.46320701864374059,0.63641694396865356,0.49444250824172298 +0.87211383416274912,0.031539296562799746,0.87076508373449524 +0.48052446873597876,0.64192779885974338,0.095065021673584771 +0.8231433992053161,0.39175164168258059,0.33087687520954856 +0.32143638706046096,0.018411316809590406,0.87319854586237666 +0.92155373656305606,0.20581596094541221,0.55188496454699454 +0.48585080808920156,0.41974822079656038,0.57661563134774729 +0.5446928112307059,0.24089044474948751,0.34466111301214686 +0.3506568223103127,0.2338606360193981,0.46557562048313106 +0.88450809440822453,0.50715507176959351,0.14045160316228333 +0.48509668009589624,0.360258546636122,0.96076386513093515 +0.56134634728644706,0.5340975518039186,0.50190811473723018 +0.36381673371817891,0.37477072331755812,0.96508344731737683 +0.39084217320433012,0.60933856666296793,0.65122668995859001 +0.23352871928950392,0.64063349602723219,0.99864098121415945 +0.46209347036148074,0.12687596706969639,0.16318878439176177 +0.15547728302456199,0.23066575492329078,0.89434669536302658 +0.49417315039274734,0.4662367306790135,0.65456402562107496 +0.71058961869572645,0.1697284684164404,0.62702325115012358 +0.51030799374590718,0.39954206433522288,0.50285327550788073 +0.73315356431785661,0.58604851504040334,0.13354311518037121 +0.96578812327744989,0.11193849270848499,0.083077899070327921 +0.52442785427259542,0.93376825844193123,0.43383100103336025 +0.94009652569179214,0.43501246449891506,0.47880682700956556 +0.22037005458370629,0.90536028488981424,0.31812781713096183 +0.66675349078305979,0.3576766775612692,0.85944984250491008 +0.82166195954009547,0.65694772385011224,0.55353558851297924 +0.020719990036436653,0.76842448392307683,0.78046209677104172 +0.96397398901165343,0.36895580318514731,0.18426348727066455 +0.47832392489354769,0.92493425117829353,0.746115337111384 +0.69967584280146111,0.3630711474317343,0.10918933553520471 +0.29246668272083998,0.34602499230663553,0.48161928414880717 +0.8415320136043527,0.812147080924843,0.77126044170551766 +0.078417122269472839,0.41628642897226109,0.39150900051524684 +0.26150089118819875,0.74408763066647698,0.45085208448287306 +0.71285997807277601,0.2753306296098561,0.12898795235915839 +0.34480539498689122,0.51982539467826849,0.91287365307545121 +0.62585087258273753,0.88634058763789325,0.45935240399237376 +0.13251939589417994,0.086633350583403601,0.65651377323830051 +0.62771050020820296,0.6797193086843597,0.083760739416245067 +0.052430589482174716,0.14747024041364887,0.43763308889551578 +0.24296988898622252,0.13258663209207755,0.46523994895009613 +0.24497744473370631,0.50496531693052349,0.67757190725172611 +0.71404821792359652,0.030626970897620343,0.94777396510898293 +0.44567518604116774,0.17306471962612224,0.68298843561453237 +0.36841050863965308,0.58994133912522584,0.45936162343167131 +0.53382226460184778,0.65002328895004424,0.10864465814886792 +0.01942201492659601,0.76871317861074151,0.57692470809450225 +0.48999684806854216,0.35212090470108326,0.37408894416049887 +0.22425360703023356,0.99199739291729983,0.79243589991150853 +0.95534664543425973,0.057222604827159684,0.17337978654682307 +0.039427541404710831,0.14796363757148145,0.042743153993095917 +0.85033417900154784,0.090283821204099612,0.60211540875303704 +0.45826510070619542,0.97349262079913423,0.048908932107002867 +0.94179869200259558,0.22597852949657726,0.55279069088148436 +0.85821145832999857,0.91872393589636125,0.67122847744385783 +0.50109020180830954,0.25223028027912497,0.17884147966941855 +0.45599282138873942,0.22160909066511836,0.3186815135049762 +0.02241249130413725,0.55773921291192075,0.38470098605538539 +0.022533572253770617,0.6050657075846082,0.043487613941568748 +0.53934104885917711,0.2048044474554539,0.94910685929557614 +0.097665495628938448,0.06057537661403456,0.71700211525279567 +0.65741920395293185,0.65794164196968086,0.35149971818475356 +0.57547215780940353,0.087961234899303964,0.7781823974288542 +0.74953302192572147,0.63402143637809749,0.0034196367376099213 +0.77070281331927726,0.1672559086248796,0.44526581792866182 +0.83706659052125232,0.91078429152817331,0.1012808804706522 +0.73914808091126138,0.40854152210420125,0.63993167458299149 +0.22520281850781543,0.47000318227460902,0.59929212221298844 +0.98392019982401435,0.041155673701470885,0.1190314594746964 +0.99759038488170293,0.68235615733725707,0.18515863222281304 +0.051831662179558036,0.38119375500461883,0.8487201410216858 +0.36909893970598429,0.90339893345591704,0.90182547532260471 +0.61715655970918815,0.0097326073486683207,0.94038167174886045 +0.86156765053749718,0.71930243613700473,0.87715670858166883 +0.22703695207567265,0.14161948706908112,0.74195050612809177 +0.78085154440425664,0.78874025746431087,0.93057904211114939 +0.29183740899010713,0.31095399060180351,0.14695806170176184 +0.88492340806357972,0.34925993492283247,0.85339396238931886 +0.80163748078099806,0.7905610355581868,0.089924089264773799 +0.73181692511633911,0.22067059670934686,0.48735839477942899 +0.098171496800493974,0.90675394017506905,0.069557294218870247 +0.29097149092197205,0.72153013438523284,0.066503145201721736 +0.35312318593519987,0.9316807688165416,0.63936217566431408 +0.735063372648463,0.94773068861385878,0.75631427041136379 +0.52584791059636649,0.010543887407105879,0.26303186664623135 +0.36704837788007072,0.55744390330797899,0.18137362317991101 +0.20461573382027953,0.035663941053448722,0.86734566249117118 +0.68820239436232844,0.18894522921359036,0.13793503477272884 +0.2992724093786121,0.1033144512351708,0.61661691128291896 +0.37718905345000275,0.18403654079553633,0.81884407314419161 +0.038227921629746975,0.77827057523899035,0.95559910361534939 +0.9680367654491332,0.34413978459763217,0.65740085362300327 +0.66738990604949111,0.2196608562894693,0.82417232049083256 +0.51559227582518863,0.86281306922552536,0.79908207705099499 +0.62387497079999055,0.053929081730902717,0.48520601035204247 +0.6815889692064212,0.39069062396181164,0.48752675783748556 +0.86363627145154276,0.94258203134459617,0.3438459442937547 +0.88575784606477337,0.24673919626437923,0.11552216009078442 +0.81557778206623033,0.85147269494307232,0.29789800564710833 +0.92137862449980523,0.43824762226175451,0.72211597104877523 +0.61593717771593581,0.096236728196555599,0.35524254697284363 +0.090596855022192613,0.0052066592098156672,0.51197944610236534 +0.71671354920978192,0.041369372745930111,0.18728004180511326 +0.48797309456244464,0.31041525280588955,0.38059334668669942 +0.32347118376854567,0.4332118004061899,0.35469937052280043 +0.59253875183505034,0.23446675682369533,0.36515867001065516 +0.23374602937207489,0.084866827899760189,0.86600386958042808 +0.43821434737536791,0.17174863438835963,0.26280015327115558 +0.48449978001673233,0.7255442833110054,0.90821030764840693 +0.71799816252840154,0.70948774121983949,0.58554834348194562 +0.32636025957107351,0.82728362015938073,0.93560659543072677 +0.45552524752096912,0.050735310033296122,0.89609171757556638 +0.70200978162362926,0.9593139683032641,0.08084401957295928 +0.2055759450644209,0.032777485077280843,0.36923928401289874 diff --git a/data/hartmann3_y.csv b/data/hartmann3_y.csv new file mode 100644 index 0000000..d82a2b0 --- /dev/null +++ b/data/hartmann3_y.csv @@ -0,0 +1,40 @@ +-0.23613813851434406 +-0.43717764125912023 +-0.22504523156942588 +-0.57726407325352436 +-1.8118217063860205 +-0.22460601752279963 +-0.39739797665366339 +-0.020301993517470543 +-0.31313778772470774 +-2.998175496845195 +-1.6265154734237697 +-0.071047937940617839 +-1.0567344607321085 +-0.0015206087924520368 +-0.39577549438705789 +-0.22077827927097554 +-1.5523605796870705 +-1.9853018986047837 +-1.3605569543754574 +-0.4579805385249231 +-1.852520147040978 +-2.5598792804084955 +-0.48947467702299186 +-0.31568052471948593 +-3.0047528425641081 +-0.70899020309299809 +-0.32856690047212928 +-1.7567644922017924 +-0.018027061827510019 +-0.20793908318458354 +-3.215940701275847 +-0.65595645227084165 +-0.16998136442497755 +-2.4925597650486582 +-0.55018914703329125 +-0.029061637802246118 +-0.60078336460847137 +-0.81341091106545438 +-0.05408521487595766 +-1.7936922862355713 diff --git a/data/hartmann3_yt.csv b/data/hartmann3_yt.csv new file mode 100644 index 0000000..c05ce94 --- /dev/null +++ b/data/hartmann3_yt.csv @@ -0,0 +1,200 @@ +-1.984026384124711 +-1.826363591237814 +-0.049832337836311073 +-0.65548545518453982 +-3.462357802696443 +-0.012648268415514776 +-0.00021242133331988186 +-2.0868493877192842 +-2.3245282111002501 +-1.6981911123271107 +-1.5951559607198642 +-0.3662801938298702 +-3.2096965411070224 +-0.6026014492894034 +-1.1083514130258139 +-2.000838739617735 +-0.042507354959886126 +-0.14844000505713159 +-0.062963986913777206 +-0.32778155724103142 +-0.51977612847708587 +-1.5787491474383792 +-0.083827071758918653 +-1.4879897953157262 +-0.19477597955488055 +-0.36281386252147996 +-2.1756373720810465 +-0.1008769683261455 +-0.22755069875656944 +-2.8633885551660283 +-0.4976705280310782 +-1.1617957771678655 +-0.28102553352367038 +-1.1019795748206898 +-0.00034651675386555451 +-1.2166540750917425 +-0.31514292277640954 +-0.059491341242546986 +-0.17534829429958801 +-0.23590179348473969 +-0.15477193578395754 +-0.0070133728757716708 +-1.7221901251212395 +-0.049040035592249058 +-0.0010231606387752332 +-2.098439058135003 +-3.4364257307975312 +-0.36485053670296014 +-0.026629282081892788 +-1.4991581095075972 +-3.2964585028472362 +-0.0092465755969006417 +-0.47525696482095636 +-0.37106500955987365 +-0.0037660982142699246 +-0.025525334057298329 +-0.20014653155148809 +-1.0170390188966705 +-3.4514548423026574 +-1.1438601557712904 +-1.2729049849287062 +-2.7138263915992273 +-1.7955828332568837 +-0.23774854597117323 +-2.1505373316502494 +-0.99191447645247166 +-1.4868250289475962 +-1.0227886437872311 +-0.32407189521206597 +-1.6857733788831648 +-0.3892183347898997 +-1.9798258277919596 +-0.20190855565344101 +-1.6266057502121531 +-1.4634485584507144 +-0.42517231843457581 +-0.073555897958854838 +-0.056488786417656672 +-0.097053561527257887 +-3.1058053525953775 +-0.50227941216265681 +-1.6396693607375405 +-0.64051796873392941 +-0.27349896786792077 +-1.2446303153739648 +-1.1706983732761538 +-0.28774035681088184 +-0.027206076568089543 +-0.24120750952683684 +-0.26127567866181595 +-0.23420165192389869 +-0.79457120347400201 +-0.66442879581888226 +-0.35275099867155918 +-0.061414060705475185 +-1.832633681704162 +-0.65604309302432851 +-1.8968412246097801 +-2.0201935890392515 +-1.9982803375123201 +-0.70321831553073577 +-1.3385654182229703 +-1.6086688755165075 +-0.43023407574277223 +-0.422806281031539 +-0.04550707949071419 +-0.12403332296691955 +-1.1165898623682362 +-0.18502304529465236 +-0.53569262343167046 +-2.6117694853559441 +-0.67405699080355952 +-2.4719430018602555 +-0.14941690875331878 +-1.2365502649120748 +-0.18588571864544975 +-0.39015468512519191 +-1.5313726462571575 +-0.38100694269691843 +-1.745987092544838 +-0.30755095437980501 +-3.4252366951589748 +-1.0351662440265978 +-0.32070986860375295 +-0.01373683109764008 +-0.33562118920194822 +-0.33298334732591411 +-2.1269839024484094 +-0.19227618466863255 +-0.68394979037207415 +-0.93077316221974504 +-0.027990238363607645 +-2.8142818469753941 +-0.44812892675262689 +-0.92995679213196714 +-0.26392494357391072 +-0.15755518978433217 +-0.20509450787129582 +-0.0010261335228726805 +-0.25320493023286617 +-0.65036333596228835 +-0.62556403678178418 +-0.81632419352156282 +-0.53261878996692591 +-0.014901928048360847 +-0.84942649547296167 +-0.36543929849734436 +-0.2589260034907544 +-0.53873746985096738 +-0.0055888600197336906 +-0.26038162476761167 +-0.001680459107081895 +-1.1731060046498494 +-1.2660765896565738 +-0.15695878447375508 +-0.013464654826596936 +-2.9301026486764896 +-1.1832042831948488 +-0.16511789901104021 +-2.7368318978305002 +-0.74385377521349483 +-1.882857459358718 +-0.43750805225205036 +-2.4763342910631549 +-0.0033946446046906288 +-0.22674618565497384 +-0.0030299246827193353 +-0.0092729704571478735 +-2.1370217488248917 +-0.77953863660509115 +-0.82893679343960003 +-0.12902976958099174 +-0.31548063612001986 +-0.42335355857221574 +-0.27494498370220893 +-1.1236784105996722 +-1.856337835201586 +-1.1168444858602753 +-1.3778872620608635 +-1.5495063355959935 +-0.21725244060856083 +-0.29771311215775975 +-0.1373064832095417 +-0.19000904218691975 +-0.0949998318092663 +-2.174182760204916 +-0.66018073795530452 +-0.14476343749704723 +-0.54216809856665171 +-0.49115184642615417 +-0.40998262230792631 +-0.57597275376735024 +-0.49584386362219357 +-0.95739122352820716 +-2.6891356737483862 +-1.0798451638334057 +-1.6267512448558112 +-0.32399360756735285 +-0.0013613999011384428 +-0.63203754118163336 diff --git a/src/aggregations-large-n_NestedKriging_python.py b/src/aggregations-large-n_NestedKriging_python.py new file mode 100644 index 0000000..abdaa93 --- /dev/null +++ b/src/aggregations-large-n_NestedKriging_python.py @@ -0,0 +1,181 @@ +# %% [markdown] +# # Python · NestedKriging — aggregations, large n +# +# **Idea.** NestedKriging splits a large design into groups and aggregates their +# predictions; how far is it from the exact predictor as the number of groups grows, +# and which aggregation is closest? +# +# **Reference.** The exact predictor at the same θ (limit: 1 group), and the exact +# fit. +# +# **Tags.** class: NestedKriging · aggregation: NK, PoE, gPoE, BCM, rBCM · +# topic: large n +# +# `NestedKriging` partitions the design into groups, fits one `Kriging` +# sub-model per group (common hyperparameters), and aggregates the sub-model +# predictions: `NK` (nested kriging: kriging of the sub-predictors, a proper +# kriging predictor), or the products of experts `PoE`, `gPoE`, `BCM`, `rBCM` +# (cheaper, variances not fully consistent). +# +# Problems: as in `large-n_Kriging-vecchia-nystrom_python.ipynb`, Hartmann 3 +# and the OTL circuit with n = 1500 (exact fit as reference) and n = 5000; then +# every aggregation on Branin. +# +# Checks: +# * **limit and asymptotic behaviour** at fixed θ (Hartmann 3, n = 300): with one +# group, `NK` *is* the exact kriging predictor; the gap to the exact predictor +# as the number of groups grows, against the fit and prediction cost; +# * accuracy close to the exact fit (warn-level gap, error-level floor), and +# faster than the exact fit at n = 1500 (warn-level); +# * `NK` interpolates the data (it is a proper kriging predictor); the PoE family +# does not need to; all stdev ≥ 0; +# * `NK` with a non-constant trend is rejected (simple-kriging theory). + +# %% +import time + +import matplotlib.pyplot as plt +import numpy as np +import pandas as pd +import pylibkriging as lk + +import gallery +from gallery import TESTS, design, metrics, compat + +suite = gallery.Suite("aggregations-large-n_NestedKriging_python") +print("pylibkriging", compat.version()) +if not compat.has("NestedKriging"): + suite.check_skip("NestedKriging", "not available in this pylibkriging") + + +def pred(k, X, **kw): + m, s, *_ = k.predict(X, return_stdev=True, **kw) + return np.ravel(m), np.ravel(s) + + +def timed(f): + t0 = time.time() + out = f() + return out, time.time() - t0 + +# %% [markdown] +# ## From one group (the exact predictor) to many, at fixed θ +# +# Hartmann 3, n = 300, θ of the exact fit kept fixed in every sub-model: the gap +# is that of the aggregation alone. RMS gap of the predicted mean to the exact +# predictor on 500 test points, relative to its standard deviation, and fit time. + +# %% +if compat.has("NestedKriging"): + tf = TESTS["hartmann3"] + X, y, Xt, yt = design.dataset(tf, 300, n_test=500, seed=54) + theta = np.asarray(lk.Kriging(y, X, "matern5_2").theta()).reshape(1, -1) + fixed = {"theta": theta, "is_theta_estim": False} + m_e = pred(lk.Kriging(y, X, "matern5_2", optim="none", parameters=fixed), Xt)[0] + rows = [] + for agg in ["NK", "rBCM"]: + for g in [1, 2, 4, 8, 16, 32]: + with suite.check_runs(f"nested {agg}, {g} groups, fixed theta: fit + predict"): + k, t = timed(lambda: lk.NestedKriging(y, X, "matern5_2", nb_groups=g, aggregation=agg, seed=1, + optim="none", parameters=fixed)) + m = pred(k, Xt)[0] + rows.append(dict(aggregation=agg, groups=g, time_s=t, + mean_gap=np.sqrt(np.mean((m - m_e) ** 2)) / np.std(m_e), + q2=metrics.q2(yt, m))) + conv = pd.DataFrame(rows) + nk = conv[conv["aggregation"] == "NK"] + suite.check_less("nested NK, 1 group == exact predictor (RMS gap / sd)", nk["mean_gap"].iloc[0], 1e-6) + suite.check_less("nested NK, 32 groups: RMS gap to the exact predictor / sd", nk["mean_gap"].iloc[-1], 0.1, + severity="warn") + rb = conv[conv["aggregation"] == "rBCM"] + suite.check_true("nested NK closer to the exact predictor than rBCM (2 to 32 groups)", + bool(np.all(nk["mean_gap"].to_numpy()[1:] <= rb["mean_gap"].to_numpy()[1:])), severity="warn") + fig, axes = plt.subplots(1, 2, figsize=(10, 3.8)) + for agg, g in conv.groupby("aggregation"): + axes[0].semilogy(g["groups"], np.maximum(g["mean_gap"], 1e-12), "o-", label=agg) + axes[1].plot(g["groups"], g["time_s"], "o-", label=agg) + axes[0].set_ylabel("RMS(mean − mean(exact)) / sd"); axes[1].set_ylabel("fit time (s)") + for ax in axes: + ax.set_xscale("log", base=2); ax.set_xlabel("number of groups"); ax.grid(True, which="both", alpha=.3) + axes[0].legend(); fig.suptitle("Hartmann 3, n = 300, θ fixed: NestedKriging vs the exact predictor"); plt.show() + display(conv.round(5)) + +# %% [markdown] +# ## Accuracy and cost against the exact fit + +# %% +CASES = {"hartmann3": (1500, 5000), "otl_circuit": (1500, 5000)} +FLOORS = {"hartmann3": 0.98, "otl_circuit": 0.99} +if compat.has("NestedKriging"): + for fname, (n_mid, n_big) in CASES.items(): + tf = TESTS[fname] + X, y, Xt, yt = design.dataset(tf, n_big, n_test=2000, seed=52) + ke, t_exact = timed(lambda: lk.Kriging(y[:n_mid], X[:n_mid], "matern5_2")) + res_e = metrics.all_metrics(yt, *pred(ke, Xt)) + suite.metric(f"{fname}/n={n_mid}", "exact", fit_s=t_exact, **res_e) + for agg in ["NK", "rBCM"]: + label = f"nested {agg} (20 groups)" + for n in (n_mid, n_big): + case = f"{fname}/n={n}" + with suite.check_runs(f"{case}/{label}: fit + predict"): + k, t = timed(lambda: lk.NestedKriging(y[:n], X[:n], "matern5_2", nb_groups=20, aggregation=agg, + seed=1)) + m, s = pred(k, Xt) + suite.check_finite(f"{case}/{label}: finite predictions", m) + res = metrics.all_metrics(yt, m, s) + suite.metric(case, label, fit_s=t, **res) + suite.check_greater(f"{case}/{label}: Q2", res["q2"], FLOORS[fname] - 0.05) + if n == n_mid: + suite.check_greater(f"{case}/{label}: Q2 - Q2(exact)", res["q2"] - res_e["q2"], -0.02, + severity="warn") + suite.check_less(f"{case}/{label}: fit time / exact fit time", t / t_exact, 1.0, severity="warn") + display(suite.table().round(4)) + +# %% [markdown] +# ## `NestedKriging` aggregations: interpolation and variance + +# %% +if compat.has("NestedKriging"): + tf = TESTS["branin"] + X, y, Xt, yt = design.dataset(tf, 200, n_test=500, seed=53) + for agg in ["NK", "PoE", "gPoE", "BCM", "rBCM"]: + with suite.check_runs(f"nested {agg}: fit"): + k = lk.NestedKriging(y, X, "matern5_2", nb_groups=8, aggregation=agg, seed=2) + m, s = (np.ravel(a) for a in k.predict(Xt, return_stdev=True)[:2]) + suite.check_true(f"nested {agg}: stdev >= 0", np.all(s >= 0)) + suite.metric("branin/nested", agg, **metrics.all_metrics(yt, m, s)) + if agg == "NK": + mX = np.ravel(k.predict(X, return_stdev=True)[0]) + suite.check_less("nested NK: interpolates (max |m(X)-y| / sd(y))", np.max(np.abs(mX - y)) / np.std(y), + 1e-4) + display(suite.table().loc["branin/nested"].round(4)) + suite.check_raises("nested NK with a linear trend is rejected", + lambda: lk.NestedKriging(y, X, "matern5_2", nb_groups=4, aggregation="NK", regmodel="linear"), + severity="warn") + +# %% [markdown] +# ## save / load, pickle, copy +# +# On Hartmann 3 (n = 40): a model saved and loaded back (class-specific and +# generic `pylibkriging.load`), pickled, or deep-copied predicts exactly as the +# original. + +# %% +if compat.has("NestedKriging"): + X, y, Xt, yt = design.dataset(TESTS["hartmann3"], 40, n_test=100, seed=61) + with suite.check_runs("NestedKriging: fit (persistence)"): + gallery.persistence.check(suite, "NestedKriging", lk.NestedKriging(y, X, "matern5_2", nb_groups=4, seed=1), Xt, pred) + +# %% [markdown] +# ## Summary +# +# The metrics of this notebook, the number of checks per status, and every check that +# did not pass (FAIL: error-level, fails the run; WARN: warn-level; KNOWN: listed in +# `tools/known_issues.csv`, with its issue; SKIP: feature absent from the installed version). +# All of them are also written to `results/aggregations-large-n_NestedKriging_python.checks.csv` and `.metrics.csv`. + +# %% +suite.summary() + +# %% +suite.finalize() diff --git a/src/cross-binding-reference_all-classes_python.py b/src/cross-binding-reference_all-classes_python.py new file mode 100644 index 0000000..c9363f0 --- /dev/null +++ b/src/cross-binding-reference_all-classes_python.py @@ -0,0 +1,171 @@ +# %% [markdown] +# # Python · all classes — cross-binding consistency: the Python reference +# +# **Idea.** All bindings wrap the same C++ library: on the same data with the same +# options, they must give the same numbers, for every model class and fit method. This +# notebook writes the Python values, the reference of the comparison. +# +# **Reference.** The R, Octave and Julia notebooks `cross-binding_all-classes_*`, +# compared by `tools/report.py`. +# +# **Tags.** class: Kriging, NestedKriging, WarpKriging, MLPKriging · kernel: matern5_2 · +# objective: LL · method: exact, vecchia, nystrom · noise: none, nugget, variances · +# warping: kumaraswamy · aggregation: NK · topic: cross-binding +# +# All bindings (Python, R, Octave, Julia) wrap the same C++ library: on the +# same data with the same options they must return the same numbers. Each +# language runs the protocol below on the shared datasets of `data/` and writes +# `results/xbind_.csv` (`lang,dataset,quantity,index,value`); +# `tools/report.py` then compares every language to Python. +# +# Protocol, for each dataset (Branin n=25, Hartmann 3 n=40, Borehole n=80, on +# the unit cube), kernel `matern5_2`, trend `constant`, θ₀ = 0.3 in every +# dimension: +# +# | quantity | how | tolerance in the report | +# |---|---|---| +# | `fixed_sigma2`, `fixed_beta`, `fixed_ll`, `fixed_loo` | `optim="none"`, θ = θ₀ fixed | rel. 1e-8 (error) | +# | `fixed_mean`, `fixed_stdev` | predict at the first 50 test points | rel. 1e-8 (error) | +# | `llfun`, `llfun_grad` | `logLikelihoodFun(θ₀)` with gradient | rel. 1e-8 (error) | +# | `fit_theta`, `fit_ll`, `fit_mean` | default fit (`BFGS`) | rel. 1e-4 (warn: optimizer paths may differ) | +# | `sim` | `simulate(nsim=3, seed=123)` at the first 5 test points (5 × 3, column-major) | rel. 1e-8 (warn: RNG may be binding-specific) | +# +# Then, on Branin only, every other model class and fit method, so that each +# binding is checked on all of them (the class-specific notebooks are in Python +# only): with fixed hyperparameters the numbers must be identical (error); a +# fit is compared at warn level, as above. +# +# | quantity | how | tolerance in the report | +# |---|---|---| +# | `nugget_fixed_mean`, `nugget_fixed_stdev` | `noise="nugget"`, θ = θ₀, σ² = var(y), nugget = 0.01 var(y), all fixed | rel. 1e-8 (error) | +# | `noise_fixed_mean`, `noise_fixed_stdev` | known noise variances 0.01 var(y), θ = θ₀ and σ² fixed | rel. 1e-8 (error) | +# | `nugget_fit_ll`, `nugget_fit_mean` | `noise="nugget"`, default fit | rel. 1e-4 (warn) | +# | `vecchia_fixed_ll`, `vecchia_fixed_mean`, `nystrom_fixed_ll`, `nystrom_fixed_mean` | `method="vecchia(10)"` / `"nystrom(10)"`, θ = θ₀ fixed; likelihood and prediction with that method | rel. 1e-8 (error) | +# | `warp_fit_mean` | `WarpKriging`, `kumaraswamy` on both inputs, default fit | rel. 1e-4 (warn) | +# | `mlp_fit_mean` | `MLPKriging`, hidden layer of 8 (`tanh`), 2 features, `BFGS10` | rel. 1e-4 (warn) | +# | `nested_fit_mean` | `NestedKriging`, 2 groups (`kmeans`, seed 1), `NK` | rel. 1e-4 (warn) | + +# %% +import csv + +import matplotlib.pyplot as plt +import numpy as np +import pandas as pd +import pylibkriging as lk + +import gallery +from gallery import compat +from gallery import DATA_DIR as DATA, RESULTS_DIR + +suite = gallery.Suite("cross-binding-reference_all-classes_python") +print("pylibkriging", compat.version()) +DATASETS = ["branin", "hartmann3", "borehole"] +rows = [] + + +def put(dataset, quantity, values): + for i, v in enumerate(np.ravel(values), start=1): + rows.append(("python", dataset, quantity, i, f"{float(v):.17g}")) + + +def load(name, stem): + return np.loadtxt(DATA / f"{name}_{stem}.csv", delimiter=",", ndmin=2) + +# %% +for name in DATASETS: + X, y, Xt = load(name, "X"), load(name, "y").ravel(), load(name, "Xt")[:50] + d = X.shape[1] + theta0 = np.full((1, d), 0.3) + kf = lk.Kriging(y, X, "matern5_2", regmodel="constant", optim="none", + parameters={"theta": theta0, "is_theta_estim": False}) + put(name, "fixed_sigma2", kf.sigma2()) + put(name, "fixed_beta", kf.beta()) + put(name, "fixed_ll", kf.logLikelihood()) + put(name, "fixed_loo", kf.leaveOneOut()) + m, s, *_ = kf.predict(Xt, return_stdev=True) + put(name, "fixed_mean", m) + put(name, "fixed_stdev", s) + ll, grad, _ = kf.logLikelihoodFun(theta0.ravel(), True) + put(name, "llfun", ll) + put(name, "llfun_grad", grad) + suite.check_close(f"{name}: logLikelihood() == logLikelihoodFun(theta0)", kf.logLikelihood(), ll, rtol=1e-10) + k = lk.Kriging(y, X, "matern5_2") + put(name, "fit_theta", k.theta()) + put(name, "fit_ll", k.logLikelihood()) + put(name, "fit_mean", k.predict(Xt, return_stdev=False)[0]) + put(name, "sim", np.ravel(np.asarray(kf.simulate(3, 123, Xt[:5])), order="F")) # column-major, as R/Octave/Julia + +# %% [markdown] +# ## Other classes and fit methods (Branin) + +# %% +name = "branin" +X, y, Xt = load(name, "X"), load(name, "y").ravel(), load(name, "Xt")[:50] +theta0 = np.full((1, 2), 0.3) +s2 = float(np.var(y)); nug = 0.01 * s2 +fixed = {"theta": theta0, "is_theta_estim": False, "sigma2": s2, "is_sigma2_estim": False} +kn = lk.Kriging(y, X, "matern5_2", noise="nugget", optim="none", + parameters={**fixed, "nugget": nug, "is_nugget_estim": False}) +m, s, *_ = kn.predict(Xt, return_stdev=True) +put(name, "nugget_fixed_mean", m); put(name, "nugget_fixed_stdev", s) +kh = lk.Kriging(y, X, "matern5_2", noise=np.full(len(y), nug), optim="none", parameters=fixed) +m, s, *_ = kh.predict(Xt, return_stdev=True) +put(name, "noise_fixed_mean", m); put(name, "noise_fixed_stdev", s) +kf = lk.Kriging(y, X, "matern5_2", noise="nugget") +put(name, "nugget_fit_ll", kf.logLikelihood()); put(name, "nugget_fit_mean", kf.predict(Xt, return_stdev=False)[0]) +for meth in ["vecchia", "nystrom"] if compat.has("method") else []: + ka = lk.Kriging(y, X, "matern5_2", optim="none", parameters={"theta": theta0, "is_theta_estim": False}, + method=f"{meth}(10)") + put(name, f"{meth}_fixed_ll", ka.logLikelihood(method=meth)) + put(name, f"{meth}_fixed_mean", ka.predict(Xt, return_stdev=False, method=meth)[0]) +if not compat.has("method"): + suite.check_skip("vecchia / nystrom quantities", "pylibkriging without the method= argument") +kw = lk.WarpKriging(y, X, ["kumaraswamy", "kumaraswamy"], "matern5_2") +put(name, "warp_fit_mean", kw.predict(Xt, return_stdev=False)[0]) +km = lk.MLPKriging(y, X, hidden_dims=[8], d_out=2, activation="tanh", kernel="matern5_2", optim="BFGS10") +put(name, "mlp_fit_mean", km.predict(Xt, return_stdev=False)[0]) +kk = lk.NestedKriging(y, X, "matern5_2", nb_groups=2, aggregation="NK", seed=1) +put(name, "nested_fit_mean", kk.predict(Xt, return_stdev=False)[0]) + +# %% +RESULTS_DIR.mkdir(exist_ok=True) +with open(RESULTS_DIR / "xbind_python.csv", "w", newline="") as fh: + w = csv.writer(fh) + w.writerow(["lang", "dataset", "quantity", "index", "value"]) + w.writerows(rows) +print(len(rows), "values written") + +# %% [markdown] +# ## The reference values +# +# Number of values per quantity and dataset (the report compares each of them +# with the other bindings), and the default fits they come from: predicted vs +# true values on the 50 test points. + +# %% +ref = pd.DataFrame(rows, columns=["lang", "dataset", "quantity", "index", "value"]) +ref["value"] = ref["value"].astype(float) +display(ref.groupby(["quantity", "dataset"])["value"].agg(["size", "first"]).unstack("dataset")) +fig, axes = plt.subplots(1, len(DATASETS), figsize=(12, 3.6)) +for ax, name in zip(axes, DATASETS): + yt = load(name, "yt").ravel()[:50] + m = ref[(ref.dataset == name) & (ref.quantity == "fit_mean")]["value"].to_numpy() + ax.plot(yt, m, "o", ms=4); ax.plot([yt.min(), yt.max()], [yt.min(), yt.max()], "k--", lw=.8) + q2 = 1 - np.sum((yt - m) ** 2) / np.sum((yt - yt.mean()) ** 2) + suite.metric(name, "pylibkriging default fit", q2=q2) + ax.set_title(f"{name}: Q² = {q2:.3f}"); ax.set_xlabel("true"); ax.set_ylabel("predicted") +plt.tight_layout(); plt.show() + +# %% [markdown] +# ## Summary +# +# The metrics of this notebook, the number of checks per status, and every check that +# did not pass (FAIL: error-level, fails the run; WARN: warn-level; KNOWN: listed in +# `tools/known_issues.csv`, with its issue; SKIP: feature absent from the installed version). +# All of them are also written to `results/cross-binding-reference_all-classes_python.checks.csv` and `.metrics.csv`. + +# %% +suite.summary() + +# %% +suite.finalize() diff --git a/src/cross-binding_all-classes_R.R b/src/cross-binding_all-classes_R.R new file mode 100644 index 0000000..a02300d --- /dev/null +++ b/src/cross-binding_all-classes_R.R @@ -0,0 +1,121 @@ +# %% [markdown] +# # R · all classes — cross-binding consistency: the R side +# +# **Idea.** The R side of the cross-binding protocol: rlibkriging must give the +# numbers of pylibkriging on the same data, for every model class and fit method. +# +# **Reference.** The Python values (`cross-binding-reference_all- +# classes_python.ipynb`), compared by `tools/report.py`. +# +# **Tags.** class: Kriging, NestedKriging, WarpKriging, MLPKriging · kernel: matern5_2 · +# objective: LL · method: exact, vecchia, nystrom · noise: none, nugget, variances · +# warping: kumaraswamy · aggregation: NK · topic: cross-binding +# +# Same protocol as `cross-binding-reference_all-classes_python.ipynb` (see there), on the +# shared datasets of `data/`; writes `results/xbind_R.csv`, compared with the +# Python values by `tools/report.py`. + +# %% +suppressPackageStartupMessages(library(rlibkriging)) +source("tools/R/gallery.R") +s <- suite("cross-binding_all-classes_R") +cat("rlibkriging", as.character(packageVersion("rlibkriging")), "\n") +rows <- list() +put <- function(dataset, quantity, values) { + v <- as.numeric(values) + rows[[length(rows) + 1]] <<- data.frame(lang = "R", dataset = dataset, quantity = quantity, + index = seq_along(v), value = sprintf("%.17g", v)) +} + +# %% +for (name in c("branin", "hartmann3", "borehole")) { + X <- read_data(name, "X"); y <- as.numeric(read_data(name, "y")); Xt <- read_data(name, "Xt")[1:50, , drop = FALSE] + d <- ncol(X) + theta0 <- matrix(0.3, nrow = 1, ncol = d) + kf <- Kriging(y, X, "matern5_2", regmodel = "constant", optim = "none", + parameters = list(theta = theta0, is_theta_estim = FALSE)) + put(name, "fixed_sigma2", kf$sigma2()) + put(name, "fixed_beta", kf$beta()) + put(name, "fixed_ll", logLikelihood(kf)) + put(name, "fixed_loo", leaveOneOut(kf)) + p <- predict(kf, Xt, return_stdev = TRUE) + put(name, "fixed_mean", p$mean) + put(name, "fixed_stdev", p$stdev) + llf <- logLikelihoodFun(kf, theta0, return_grad = TRUE) + put(name, "llfun", llf$logLikelihood) + put(name, "llfun_grad", llf$logLikelihoodGrad) + check_close(s, paste0(name, ": logLikelihood() == logLikelihoodFun(theta0)"), logLikelihood(kf), llf$logLikelihood, rtol = 1e-10) + k <- Kriging(y, X, "matern5_2") + put(name, "fit_theta", k$theta()) + put(name, "fit_ll", logLikelihood(k)) + put(name, "fit_mean", predict(k, Xt, return_stdev = FALSE)$mean) + put(name, "sim", simulate(kf, nsim = 3, seed = 123, x = Xt[1:5, , drop = FALSE])) +} + +# %% [markdown] +# ## Other classes and fit methods (Branin) + +# %% +name <- "branin" +X <- read_data(name, "X"); y <- as.numeric(read_data(name, "y")); Xt <- read_data(name, "Xt")[1:50, , drop = FALSE] +theta0 <- matrix(0.3, nrow = 1, ncol = 2) +s2 <- mean((y - mean(y))^2); nug <- 0.01 * s2 # population variance, as numpy.var +fixed <- list(theta = theta0, is_theta_estim = FALSE, sigma2 = s2, is_sigma2_estim = FALSE) +kn <- Kriging(y, X, "matern5_2", noise = "nugget", optim = "none", + parameters = c(fixed, list(nugget = nug, is_nugget_estim = FALSE))) +p <- predict(kn, Xt, return_stdev = TRUE) +put(name, "nugget_fixed_mean", p$mean); put(name, "nugget_fixed_stdev", p$stdev) +kh <- Kriging(y, X, "matern5_2", noise = rep(nug, length(y)), optim = "none", parameters = fixed) +p <- predict(kh, Xt, return_stdev = TRUE) +put(name, "noise_fixed_mean", p$mean); put(name, "noise_fixed_stdev", p$stdev) +kf <- Kriging(y, X, "matern5_2", noise = "nugget") +put(name, "nugget_fit_ll", logLikelihood(kf)); put(name, "nugget_fit_mean", predict(kf, Xt, return_stdev = FALSE)$mean) +for (meth in c("vecchia", "nystrom")) tryCatch({ # method= is absent from older releases + ka <- Kriging(y, X, "matern5_2", optim = "none", parameters = list(theta = theta0, is_theta_estim = FALSE), + method = paste0(meth, "(10)")) + put(name, paste0(meth, "_fixed_ll"), ka$logLikelihood(method = meth)) + put(name, paste0(meth, "_fixed_mean"), predict(ka, Xt, return_stdev = FALSE, method = meth)$mean) +}, error = function(e) message(meth, " skipped (reported as missing): ", conditionMessage(e))) +kw <- WarpKriging(y, X, warping = c("kumaraswamy", "kumaraswamy"), kernel = "matern5_2") +put(name, "warp_fit_mean", predict(kw, Xt, return_stdev = FALSE)$mean) +km <- MLPKriging(y, X, hidden_dims = c(8), d_out = 2, activation = "tanh", kernel = "matern5_2", optim = "BFGS10") +put(name, "mlp_fit_mean", predict(km, Xt, return_stdev = FALSE)$mean) +kk <- NestedKriging(y, X, kernel = "matern5_2", nb_groups = 2, aggregation = "NK", seed = 1) +put(name, "nested_fit_mean", predict(kk, Xt, return_stdev = FALSE)$mean) + +# %% +out <- do.call(rbind, rows) +write.csv(out, file.path(results_dir, "xbind_R.csv"), row.names = FALSE, quote = FALSE) +cat(nrow(out), "values written\n") + +# %% [markdown] +# ## The values written +# +# Number of values per quantity and dataset (each compared with Python by the +# report), and the default fits they come from: predicted vs true on the 50 test points. + +# %% +print(with(out, table(quantity, dataset))) +op <- par(mfrow = c(1, 3), mar = c(4, 4, 2, 1)) +for (name in c("branin", "hartmann3", "borehole")) { + yt <- as.numeric(read_data(name, "yt"))[1:50] + m <- as.numeric(out$value[out$dataset == name & out$quantity == "fit_mean"]) + q <- q2(yt, m); metric(s, name, "rlibkriging default fit", q2 = q) + plot(yt, m, xlab = "true", ylab = "predicted", main = sprintf("%s: Q2 = %.3f", name, q), pch = 16, cex = .7) + abline(0, 1, lty = 2) +} +par(op) + +# %% [markdown] +# ## Summary +# +# The metrics of this notebook, the number of checks per status, and every check that +# did not pass (FAIL: error-level, fails the run; WARN: warn-level; KNOWN: listed in +# `tools/known_issues.csv`, with its issue; SKIP: feature absent from the installed version). +# All of them are also written to `results/cross-binding_all-classes_R.checks.csv` and `.metrics.csv`. + +# %% +suite_summary(s) + +# %% +suite_finalize(s) diff --git a/src/cross-binding_all-classes_julia.jl b/src/cross-binding_all-classes_julia.jl new file mode 100644 index 0000000..ac3112d --- /dev/null +++ b/src/cross-binding_all-classes_julia.jl @@ -0,0 +1,118 @@ +# %% [markdown] +# # Julia · all classes — cross-binding consistency: the Julia side +# +# **Idea.** The Julia side of the cross-binding protocol: JLibKriging must give the +# numbers of pylibkriging on the same data, for every model class and fit method. +# +# **Reference.** The Python values (`cross-binding-reference_all- +# classes_python.ipynb`), compared by `tools/report.py`. +# +# **Tags.** class: Kriging, NestedKriging, WarpKriging, MLPKriging · kernel: matern5_2 · +# objective: LL · method: exact, vecchia, nystrom · noise: none, nugget, variances · +# warping: kumaraswamy · aggregation: NK · topic: cross-binding +# +# Same protocol as `cross-binding-reference_all-classes_python.ipynb` (see there), on +# the shared datasets of `data/`; writes `results/xbind_julia.csv`, compared with +# the Python values by `tools/report.py`. +# +# The module is `JLibKriging` (the packaged binding, [JLibKriging.jl](https://github.com/libKriging/JLibKriging.jl)), +# imported under a short name so that its exported accessors (`X`, `y`, `theta`, ...) +# do not clash with the variables of the notebook. + +# %% +import JLibKriging as LK +include("tools/julia/gallery.jl") +s = Suite("cross-binding_all-classes_julia") +rows = Vector{Vector{String}}() +record!(dataset, quantity, values) = + for (i, v) in enumerate(Float64.(vec(collect(values)))) # vec first: Float64.() of a 0-dim array is a scalar + push!(rows, ["julia", dataset, quantity, string(i), @sprintf("%.17g", v)]) + end + +# %% +for name in ("branin", "hartmann3", "borehole") + Xd = read_data(name, "X"); yd = vec(read_data(name, "y")); Xt = read_data(name, "Xt")[1:50, :] + d = size(Xd, 2) + theta0 = fill(0.3, d) + kf = LK.Kriging(yd, Xd, "matern5_2"; regmodel="constant", optim="none", theta=theta0, is_theta_estim=false) + record!(name, "fixed_sigma2", LK.sigma2(kf)) + record!(name, "fixed_beta", LK.beta(kf)) + record!(name, "fixed_ll", LK.log_likelihood(kf)) + record!(name, "fixed_loo", LK.leave_one_out(kf)) + p = LK.predict(kf, Xt; return_stdev=true) + record!(name, "fixed_mean", p.mean) + record!(name, "fixed_stdev", p.stdev) + llf = LK.log_likelihood_fun(kf, theta0; return_grad=true) + record!(name, "llfun", llf.ll) + record!(name, "llfun_grad", llf.grad) + check_close!(s, "$name: log_likelihood() == log_likelihood_fun(theta0)", LK.log_likelihood(kf), llf.ll; rtol=1e-10) + k = LK.Kriging(yd, Xd, "matern5_2") + record!(name, "fit_theta", LK.theta(k)) + record!(name, "fit_ll", LK.log_likelihood(k)) + record!(name, "fit_mean", LK.predict(k, Xt; return_stdev=false).mean) + record!(name, "sim", LK.simulate(kf, 3, 123, Xt[1:5, :])) +end + +# %% [markdown] +# ## Other classes and fit methods (Branin) + +# %% +name = "branin" +Xd = read_data(name, "X"); yd = vec(read_data(name, "y")); Xt = read_data(name, "Xt")[1:50, :] +theta0 = [0.3, 0.3] +s2 = sum((yd .- sum(yd) / length(yd)) .^ 2) / length(yd); nug = 0.01 * s2 # population variance, as numpy.var +kn = LK.Kriging(yd, Xd, "matern5_2"; noise="nugget", optim="none", theta=theta0, is_theta_estim=false, + sigma2=s2, is_sigma2_estim=false, nugget=nug, is_nugget_estim=false) +p = LK.predict(kn, Xt; return_stdev=true) +record!(name, "nugget_fixed_mean", p.mean); record!(name, "nugget_fixed_stdev", p.stdev) +kh = LK.Kriging(yd, Xd, "matern5_2"; noise=fill(nug, length(yd)), optim="none", theta=theta0, is_theta_estim=false, + sigma2=s2, is_sigma2_estim=false) +p = LK.predict(kh, Xt; return_stdev=true) +record!(name, "noise_fixed_mean", p.mean); record!(name, "noise_fixed_stdev", p.stdev) +kf = LK.Kriging(yd, Xd, "matern5_2"; noise="nugget") +record!(name, "nugget_fit_ll", LK.log_likelihood(kf)) +record!(name, "nugget_fit_mean", LK.predict(kf, Xt; return_stdev=false).mean) +for meth in ("vecchia", "nystrom") + ka = LK.Kriging(yd, Xd, "matern5_2"; optim="none", theta=theta0, is_theta_estim=false, method="$meth(10)") + record!(name, "$(meth)_fixed_ll", LK.log_likelihood(ka; method=meth)) + record!(name, "$(meth)_fixed_mean", LK.predict(ka, Xt; return_stdev=false, method=meth).mean) +end +kw = LK.WarpKriging(yd, Xd, ["kumaraswamy", "kumaraswamy"], "matern5_2") +record!(name, "warp_fit_mean", LK.predict(kw, Xt; return_stdev=false).mean) +km = LK.MLPKriging(yd, Xd, [8], 2; activation="tanh", kernel="matern5_2", optim="BFGS10") +record!(name, "mlp_fit_mean", LK.predict(km, Xt; return_stdev=false).mean) +kk = LK.NestedKriging(yd, Xd, "matern5_2", 2; aggregation="NK", seed=1) +record!(name, "nested_fit_mean", LK.predict(kk, Xt; return_stdev=false).mean) + +# %% +open(joinpath(RESULTS_DIR, "xbind_julia.csv"), "w") do io + println(io, "lang,dataset,quantity,index,value") + for r in rows; println(io, join(r, ",")); end +end +println(length(rows), " values written") + +# %% [markdown] +# ## The values written +# +# First value of each quantity and dataset (each compared with Python by the +# report; plots are left to the Python reference, to avoid a plotting package here). + +# %% +for key in unique([(r[2], r[3]) for r in rows]) + vals = [r[5] for r in rows if (r[2], r[3]) == key] + println(rpad("$(key[1]) $(key[2])", 36), vals[1], " (", length(vals), " values)") +end + +# %% [markdown] +# ## Summary +# +# The metrics of this notebook, the number of checks per status, and every check that +# did not pass (FAIL: error-level, fails the run; WARN: warn-level; KNOWN: listed in +# `tools/known_issues.csv`, with its issue; SKIP: feature absent from the installed version). +# All of them are also written to `results/cross-binding_all-classes_julia.checks.csv` and `.metrics.csv`. + +# %% +suite_summary(s) + +# %% +suite_finalize!(s) diff --git a/src/cross-binding_all-classes_octave.m b/src/cross-binding_all-classes_octave.m new file mode 100644 index 0000000..8cd587b --- /dev/null +++ b/src/cross-binding_all-classes_octave.m @@ -0,0 +1,136 @@ +% %% [markdown] +% # Octave · all classes — cross-binding consistency: the Octave side +% +% **Idea.** The Octave side of the cross-binding protocol: mLibKriging must give the +% numbers of pylibkriging on the same data, for every model class and fit method. +% +% **Reference.** The Python values (`cross-binding-reference_all- +% classes_python.ipynb`), compared by `tools/report.py`. +% +% **Tags.** class: Kriging, NestedKriging, WarpKriging, MLPKriging · kernel: matern5_2 · +% objective: LL · method: exact, vecchia, nystrom · noise: none, nugget, variances · +% warping: kumaraswamy · aggregation: NK · topic: cross-binding +% +% Same protocol as `cross-binding-reference_all-classes_python.ipynb` (see there), on the +% shared datasets of `data/`; writes `results/xbind_octave.csv`, compared with +% the Python values by `tools/report.py`. mLibKriging must be on the path +% (`OCTAVE_PATH` or `addpath`). + +% %% +source("tools/octave/gallery.m"); +s = suite("cross-binding_all-classes_octave"); +out = {}; +function out = put(out, dataset, quantity, values) + v = double(values(:)); + for i = 1:numel(v) + out(end + 1, :) = {"octave", dataset, quantity, i, sprintf("%.17g", v(i))}; + end +end + +% %% +for name = {"branin", "hartmann3", "borehole"} + name = name{1}; + X = read_data(name, "X"); y = read_data(name, "y"); Xt = read_data(name, "Xt")(1:50, :); + d = columns(X); + theta0 = 0.3 * ones(1, d); + kf = Kriging(y, X, "matern5_2", "constant", false, "none", "LL", Params("theta", theta0, "is_theta_estim", false)); + v = kf.sigma2(); out = put(out, name, "fixed_sigma2", v); + v = kf.beta(); out = put(out, name, "fixed_beta", v); + ll = kf.logLikelihood(); + out = put(out, name, "fixed_ll", ll); + loo = kf.leaveOneOut(); + out = put(out, name, "fixed_loo", loo); + [m, sd] = kf.predict(Xt, true, false, false); + out = put(out, name, "fixed_mean", m); + out = put(out, name, "fixed_stdev", sd); + [llf, grad] = kf.logLikelihoodFun(theta0', true, false); + out = put(out, name, "llfun", llf); + out = put(out, name, "llfun_grad", grad); + s = check_close(s, [name ": logLikelihood() == logLikelihoodFun(theta0)"], ll, llf, 1e-10); + k = Kriging(y, X, "matern5_2"); + v = k.theta(); out = put(out, name, "fit_theta", v); + llk = k.logLikelihood(); + out = put(out, name, "fit_ll", llk); + [mk, ~] = k.predict(Xt, false, false, false); + out = put(out, name, "fit_mean", mk); + S = kf.simulate(int32(3), int32(123), Xt(1:5, :), false); + out = put(out, name, "sim", S); +end + +% %% [markdown] +% ## Other classes and fit methods (Branin) + +% %% +name = "branin"; +X = read_data(name, "X"); y = read_data(name, "y"); Xt = read_data(name, "Xt")(1:50, :); +theta0 = [0.3, 0.3]; +s2 = mean((y - mean(y)).^2); nug = 0.01 * s2; % population variance, as numpy.var +kn = Kriging(y, X, "matern5_2", "constant", false, "none", "LL", ... + Params("theta", theta0, "is_theta_estim", false, "sigma2", s2, "is_sigma2_estim", false, ... + "nugget", nug, "is_nugget_estim", false), "nugget"); +[m, sd] = kn.predict(Xt, true, false, false); +out = put(out, name, "nugget_fixed_mean", m); out = put(out, name, "nugget_fixed_stdev", sd); +kh = Kriging(y, X, "matern5_2", "constant", false, "none", "LL", ... + Params("theta", theta0, "is_theta_estim", false, "sigma2", s2, "is_sigma2_estim", false), ... + "heterogeneous", nug * ones(size(y))); +[m, sd] = kh.predict(Xt, true, false, false); +out = put(out, name, "noise_fixed_mean", m); out = put(out, name, "noise_fixed_stdev", sd); +kf = Kriging(y, X, "matern5_2", "constant", false, "BFGS", "LL", [], "nugget"); +ll = kf.logLikelihood(); out = put(out, name, "nugget_fit_ll", ll); +m = kf.predict(Xt, false, false, false); out = put(out, name, "nugget_fit_mean", m); +for meth = {"vecchia", "nystrom"} + meth = meth{1}; + try % method is absent from older releases + ka = Kriging(y, X, "matern5_2", "constant", false, "none", "LL", Params("theta", theta0, "is_theta_estim", false), ... + "none", [], [meth "(10)"]); + ll = ka.logLikelihood(meth); out = put(out, name, [meth "_fixed_ll"], ll); + m = ka.predict(Xt, false, false, false, meth); out = put(out, name, [meth "_fixed_mean"], m); + catch err + printf("%s skipped (reported as missing): %s\n", meth, err.message); + end +end +kw = WarpKriging(y, X, {"kumaraswamy", "kumaraswamy"}, "matern5_2"); +m = kw.predict(Xt, false, false, false); out = put(out, name, "warp_fit_mean", m); +km = MLPKriging(y, X, [8], 2, "tanh", "matern5_2", "constant", false, "BFGS10"); +m = km.predict(Xt, false, false, false); out = put(out, name, "mlp_fit_mean", m); +kk = NestedKriging(y, X, "matern5_2", 2, "NK", "kmeans", 1); +m = kk.predict(Xt); out = put(out, name, "nested_fit_mean", m); + +% %% +d = getenv("LKGALLERY_RESULTS"); if isempty(d), d = "results"; end +if ~exist(d, "dir"), mkdir(d); end +fid = fopen(fullfile(d, "xbind_octave.csv"), "w"); +fprintf(fid, "lang,dataset,quantity,index,value\n"); +for i = 1:size(out, 1) + fprintf(fid, "%s,%s,%s,%d,%s\n", out{i, :}); +end +fclose(fid); +printf("%d values written\n", size(out, 1)); + +% %% [markdown] +% ## The values written +% +% First value of each quantity and dataset (each compared with Python by the +% report; plots are left to the Python reference, Octave's graphics toolkit not +% being available on every runner). + +% %% +keys = strcat(out(:, 2), " ", out(:, 3)); +[u, first] = unique(keys, "first"); +for i = 1:numel(u) + printf("%-36s %s (%d values)\n", u{i}, out{first(i), 5}, sum(strcmp(keys, u{i}))); +end + +% %% [markdown] +% ## Summary +% +% The metrics of this notebook, the number of checks per status, and every check that +% did not pass (FAIL: error-level, fails the run; WARN: warn-level; KNOWN: listed in +% `tools/known_issues.csv`, with its issue; SKIP: feature absent from the installed version). +% All of them are also written to `results/cross-binding_all-classes_octave.checks.csv` and `.metrics.csv`. + +% %% +suite_summary(s); + +% %% +suite_finalize(s); diff --git a/src/deep-kernel-projected-branin_MLPKriging_python.py b/src/deep-kernel-projected-branin_MLPKriging_python.py new file mode 100644 index 0000000..a0bfa89 --- /dev/null +++ b/src/deep-kernel-projected-branin_MLPKriging_python.py @@ -0,0 +1,108 @@ +# %% [markdown] +# # Python · MLPKriging — deep kernel on a 10D function with a 2D structure +# +# **Idea.** A deep kernel (an MLP feature map before the kernel) can find a low- +# dimensional structure that a per-axis kernel cannot: Branin of a 2D projection of 10 +# inputs. +# +# **Reference.** Plain `Kriging` on the same data. +# +# **Tags.** class: MLPKriging, Kriging · kernel: matern5_2 · +# topic: deep kernel, dimension reduction +# +# f(x) = Branin(Wx) for x ∈ [0,1]¹⁰ and a fixed 2 × 10 projection W: the +# function only varies in a 2D subspace, which a joint feature map (an MLP from +# ℝ¹⁰ to ℝ², then a stationary kernel) can find, whereas a per-axis kernel +# cannot. +# +# Checks: finite predictions; with a multistart optimizer, a Q² floor +# (error-level); the single-start fits compared with plain `Kriging` +# (warn-level), for two activations (`tanh`, smooth; `selu`, the default, with +# a kink at 0). + +# %% +import time + +import matplotlib.pyplot as plt +import numpy as np +import pylibkriging as lk + +import gallery +from gallery import TESTS, design, metrics, compat + +suite = gallery.Suite("deep-kernel-projected-branin_MLPKriging_python") +print("pylibkriging", compat.version()) + + +def pred(k, X): + m, s, *_ = k.predict(X, return_stdev=True) + return np.ravel(m), np.ravel(s) + +# %% +if compat.has("MLPKriging"): + rng = np.random.default_rng(43) + W = rng.normal(size=(2, 10)) + W /= np.abs(W).sum(axis=1, keepdims=True) + tfb = TESTS["branin"] + + def projected(X): + Z = X @ W.T + Z = (Z - (W.clip(max=0).sum(axis=1))) / np.abs(W).sum(axis=1) # back to [0,1]^2 + return tfb.unit(Z) + + X = design.lhs(120, 10, seed=44); Xt = design.uniform(2000, 10, seed=45) + y, yt = projected(X), projected(Xt) + kp = lk.Kriging(y, X, "matern5_2") + res_k = metrics.all_metrics(yt, *pred(kp, Xt)) + suite.metric("projected_branin_10d", "Kriging", **res_k) + for act in ["tanh", "selu"]: + for optim in ["BFGS", "BFGS10"]: + case = f"MLPKriging({act}, {optim})" + with suite.check_runs(f"{case}: fit"): + t0 = time.time() + km = lk.MLPKriging(y, X, hidden_dims=[16, 8], d_out=2, activation=act, kernel="matern5_2", + optim=optim) + fit_s = time.time() - t0 + m, s = pred(km, Xt) + suite.check_finite(f"{case}: predictions finite", m) + res = metrics.all_metrics(yt, m, s) + suite.metric("projected_branin_10d", f"MLPKriging {act} {optim}", fit_s=fit_s, + ll=float(km.logLikelihood()), **res) + if optim == "BFGS10": + suite.check_greater(f"{case}: Q2", res["q2"], 0.8) + else: + suite.check_greater(f"{case}: Q2 - Q2(Kriging)", res["q2"] - res_k["q2"], -0.1, severity="warn") + tab = suite.table().loc["projected_branin_10d"] + tab["q2"].plot.barh(figsize=(6, 3)); plt.xlabel("Q² (test set)"); plt.xlim(min(0, tab["q2"].min()), 1) + plt.title("Branin(Wx), x in [0,1]^10"); plt.show() + display(tab.round(4)) +else: + suite.check_skip("MLPKriging", "not available in this pylibkriging") + +# %% [markdown] +# ## save / load, pickle, copy +# +# On Hartmann 3 (n = 40): a model saved and loaded back (class-specific and +# generic `pylibkriging.load`), pickled, or deep-copied predicts exactly as the +# original. + +# %% +if compat.has("MLPKriging"): + X, y, Xt, yt = design.dataset(TESTS["hartmann3"], 40, n_test=100, seed=61) + with suite.check_runs("MLPKriging: fit (persistence)"): + gallery.persistence.check(suite, "MLPKriging", lk.MLPKriging(y, X, hidden_dims=[8], d_out=2, activation="tanh", + kernel="matern5_2"), Xt, pred) + +# %% [markdown] +# ## Summary +# +# The metrics of this notebook, the number of checks per status, and every check that +# did not pass (FAIL: error-level, fails the run; WARN: warn-level; KNOWN: listed in +# `tools/known_issues.csv`, with its issue; SKIP: feature absent from the installed version). +# All of them are also written to `results/deep-kernel-projected-branin_MLPKriging_python.checks.csv` and `.metrics.csv`. + +# %% +suite.summary() + +# %% +suite.finalize() diff --git a/src/high-dimension-normalize-screening_Kriging_python.py b/src/high-dimension-normalize-screening_Kriging_python.py new file mode 100644 index 0000000..94b25fb --- /dev/null +++ b/src/high-dimension-normalize-screening_Kriging_python.py @@ -0,0 +1,121 @@ +# %% [markdown] +# # Python · Kriging — moderate and high dimension: engineering functions, `normalize`, screening +# +# **Idea.** From 6 to 20 inputs on engineering functions: accuracy, the effect of +# `normalize` on inputs in physical units, and the ability of the fitted ranges θ to +# tell influential inputs from inactive ones. +# +# **Reference.** The same model on inputs rescaled by hand, and the known active +# inputs of the Welch function. +# +# **Tags.** class: Kriging · kernel: matern5_2 · +# topic: high dimension, normalize, screening +# +# | function | d | character | +# |---|---|---| +# | OTL circuit | 6 | smooth, inputs on very different scales | +# | Piston | 7 | smooth, strongly non-linear in a few inputs | +# | Borehole | 8 | smooth, 2–3 dominant inputs, raw scales from 0.05 to 10⁵ | +# | Robot arm | 8 | oscillating (angles): hard for a stationary GP | +# | Dette & Pepelyshev | 8 | curved, logarithmic terms | +# | Wing weight | 10 | smooth, nearly additive | +# | Welch et al. (1992) | 20 | screening problem: few active inputs, interactions | +# +# Checks: +# * **`normalize=True` on raw inputs** gives the same accuracy as an explicit +# rescaling to [0, 1] (they are the same model up to the optimizer); +# inputs in native units (Borehole spans 0.05 to 50 000) must not break a fit; +# * Q² floors per function (regression detection); +# * the fitted θ of Welch's **inactive** inputs are larger than those of the +# active ones (screening ability, warn-level). +# +# `MLPKriging` on a high-dimensional function with a low-dimensional structure: +# `deep-kernel-projected-branin_MLPKriging_python.ipynb`. + +# %% +import time + +import matplotlib.pyplot as plt +import numpy as np +import pandas as pd +import pylibkriging as lk + +import gallery +from gallery import TESTS, design, metrics, compat + +suite = gallery.Suite("high-dimension-normalize-screening_Kriging_python") +print("pylibkriging", compat.version()) + + +def pred(k, X): + m, s, *_ = k.predict(X, return_stdev=True) + return np.ravel(m), np.ravel(s) + +# %% [markdown] +# ## Accuracy, unit-cube inputs vs native inputs with `normalize=True` + +# %% +CASES = { # function: (n, Q2 floor on the unit cube) + "otl_circuit": (60, 0.999), + "piston": (70, 0.98), + "borehole": (80, 0.995), + "robot_arm": (160, 0.45), + "dette_pepelyshev": (80, 0.99), + "wing_weight": (100, 0.999), + "welch": (200, 0.98), +} +for fname, (n, floor) in CASES.items(): + tf = TESTS[fname] + U, y, Ut, yt = design.dataset(tf, n, n_test=2000, seed=41) + t0 = time.time() + k = lk.Kriging(y, U, "matern5_2") + fit_s = time.time() - t0 + res = metrics.all_metrics(yt, *pred(k, Ut)) + suite.metric(fname, "unit cube", fit_s=fit_s, **res) + suite.check_greater(f"{fname}: Q2 (unit cube, matern5_2)", res["q2"], floor) + # same problem in native units, rescaled internally + Xn, Xtn = tf.scale(U), tf.scale(Ut) + with suite.check_runs(f"{fname}: fit on native inputs with normalize=True"): + kn = lk.Kriging(y, Xn, "matern5_2", normalize=True) + res_n = metrics.all_metrics(yt, *pred(kn, Xtn)) + suite.metric(fname, "native + normalize", **res_n) + suite.check_greater(f"{fname}: Q2(normalize) - Q2(unit cube)", res_n["q2"] - res["q2"], -0.02) +tab = suite.table() +tab.round(4) + +# %% [markdown] +# ## Screening: Welch et al. (1992) +# +# Inputs 8 and 16 do not appear in the function; inputs 1, 4, 5, 12, 19, 20 are +# the most influential ones. A small θ_j (short correlation length) means an +# influential input, so the inactive inputs should get the largest θ. + +# %% +tf = TESTS["welch"] +U, y, _, _ = design.dataset(tf, 200, seed=42) +k = lk.Kriging(y, U, "matern5_2") +theta = np.ravel(k.theta()) +inactive = [7, 15] # 0-based indices of x8, x16 +active = [0, 3, 4, 11, 18, 19] # x1, x4, x5, x12, x19, x20 +suite.check_greater("welch: min theta(inactive) / max theta(active)", theta[inactive].min() / theta[active].max(), 1.0, + severity="warn") +colors = ["tab:red" if i in active else "tab:gray" if i in inactive else "tab:blue" for i in range(20)] +plt.figure(figsize=(9, 3)) +plt.bar([f"x{i + 1}" for i in range(20)], theta, color=colors) +plt.yscale("log"); plt.ylabel("fitted θ (log)") +plt.title("Welch (20D): red = most influential inputs, gray = inactive (x8, x16)"); plt.show() +pd.Series(theta, index=[f"x{i + 1}" for i in range(20)]).round(3).to_frame("theta").T + +# %% [markdown] +# ## Summary +# +# The metrics of this notebook, the number of checks per status, and every check that +# did not pass (FAIL: error-level, fails the run; WARN: warn-level; KNOWN: listed in +# `tools/known_issues.csv`, with its issue; SKIP: feature absent from the installed version). +# All of them are also written to `results/high-dimension-normalize-screening_Kriging_python.checks.csv` and `.metrics.csv`. + +# %% +suite.summary() + +# %% +suite.finalize() diff --git a/src/kernels-trends-interpolation_Kriging_python.py b/src/kernels-trends-interpolation_Kriging_python.py new file mode 100644 index 0000000..ccb9784 --- /dev/null +++ b/src/kernels-trends-interpolation_Kriging_python.py @@ -0,0 +1,216 @@ +# %% [markdown] +# # Python · Kriging — kernels × trends, interpolation, predict (covariance, derivatives) +# +# **Idea.** The basic promise of noise-free Kriging, over the whole grid of kernels +# and trends: it interpolates the data, its predictive variance vanishes at the design +# points, its predicted derivatives are the derivatives of its prediction, and it is +# accurate on smooth 1D to 3D functions. +# +# **Reference.** Finite differences of the prediction, the data themselves, and per- +# function Q² floors. +# +# **Tags.** class: Kriging · kernel: gauss, exp, matern3_2, matern5_2 · +# trend: constant, linear, interactive, quadratic · topic: interpolation, derivatives +# +# Plain noise-free `Kriging`, every kernel (`gauss`, `exp`, `matern3_2`, +# `matern5_2`) crossed with every trend (`constant`, `linear`, `interactive`, +# `quadratic`) on functions of different regularity: +# +# | function | d | character | +# |---|---|---| +# | Forrester | 1 | smooth, one oscillation | +# | Gramacy & Lee (2012) | 1 | fast oscillation then polynomial: non-stationary | +# | Branin | 2 | smooth, three global minima | +# | Six-hump camel | 2 | polynomial, steep borders | +# | log-Goldstein-Price | 2 | strongly varying | +# | Lim polynomial | 2 | low-degree polynomial: the trend should help | +# | Ishigami | 3 | strongly non-linear, interactions | +# | Hartmann 3 | 3 | multimodal, localized | +# +# Checks: +# * **interpolation**: a noise-free kriging mean reproduces the observations, with +# a (near) zero predictive standard deviation at the design points. Error-level +# for every kernel but `gauss`: its correlation matrices are notoriously +# ill-conditioned, and the numerical jitter libKriging adds to factorize them +# breaks exact interpolation (warn-level, with the size of the gap); +# * **accuracy**: Q² on a test set above a per-function floor for +# `matern5_2` / `constant` with a multistart optimizer (`BFGS10`): a +# regression shows up here first. The same floor for the default single-start +# `BFGS` is warn-level: a single start can end in a poor local optimum of the +# likelihood (see `objectives-LL-LOO-LMP-gradients_Kriging_python.ipynb`), +# which is worth seeing in the report but is not a defect of the model; +# * **sanity of the fitted hyperparameters**: finite, positive θ and σ²; +# * **consistency**: `predict` on a batch equals `predict` point by point, the +# returned covariance has the predicted variances on its diagonal, and is +# symmetric positive semi-definite. + +# %% +import time + +import numpy as np +import pandas as pd +import matplotlib.pyplot as plt +import pylibkriging as lk + +import gallery +from gallery import TESTS, design, metrics, compat + +suite = gallery.Suite("kernels-trends-interpolation_Kriging_python") +print("pylibkriging", compat.version()) + +KERNELS = ["gauss", "exp", "matern3_2", "matern5_2"] +TRENDS = ["constant", "linear", "interactive", "quadratic"] +CASES = { # function: (n, Q2 floor for matern5_2/constant) + "forrester": (12, 0.95), + "gramacy_lee_2012": (20, 0.60), + "branin": (30, 0.97), + "six_hump_camel": (30, 0.95), + "log_goldstein_price": (40, 0.85), + "lim_polynomial": (15, 0.99), + "ishigami": (80, 0.75), + "hartmann3": (40, 0.85), +} + + +def predict(model, X, **kw): + m, s, cov, *_ = model.predict(X, return_stdev=True, **kw) + return np.asarray(m).ravel(), np.asarray(s).ravel(), cov + +# %% [markdown] +# ## Kernel × trend grid + +# %% +for fname, (n, q2_floor) in CASES.items(): + tf = TESTS[fname] + X, y, Xt, yt = design.dataset(tf, n, n_test=1000, seed=1) + for kernel in KERNELS: + for trend in TRENDS: + n_beta = {"constant": 1, "linear": 1 + tf.d, "interactive": 1 + tf.d + tf.d * (tf.d - 1) // 2, + "quadratic": 1 + 2 * tf.d + tf.d * (tf.d - 1) // 2}[trend] + case = f"{fname}/{kernel}/{trend}" + if n_beta >= n - 1: + suite.check_skip(case, f"{n_beta} trend coefficients for n={n}") + continue + t0 = time.time() + try: + k = lk.Kriging(y, X, kernel, regmodel=trend, optim="BFGS") + except Exception as exc: # noqa: BLE001 + suite.check_true(f"{case}: fit", False, detail=str(exc)[:200]) + continue + fit_s = time.time() - t0 + m, s, _ = predict(k, Xt) + res = metrics.all_metrics(yt, m, s) + suite.metric(case, "pylibkriging", fit_s=fit_s, **res) + theta, sigma2 = np.asarray(k.theta()).ravel(), float(k.sigma2()) + suite.check_true(f"{case}: theta, sigma2 finite and > 0", + np.all(np.isfinite(theta)) and np.all(theta > 0) and np.isfinite(sigma2) and sigma2 > 0, + detail=f"theta={theta}, sigma2={sigma2:.3g}") + # interpolation at the design points (relative to the spread of y) + md, sd, _ = predict(k, X) + scale = np.std(y) + sev = "warn" if kernel == "gauss" else "error" + suite.check_less(f"{case}: interpolates y (max |m(X)-y| / sd(y))", np.max(np.abs(md - y)) / scale, 1e-4, + severity=sev) + suite.check_less(f"{case}: stdev(X) ~ 0 (max sd / sd(y))", np.max(sd) / scale, 1e-2, severity=sev) + if kernel == "matern5_2" and trend == "constant": + suite.check_greater(f"{case}: Q2 (single-start BFGS)", res["q2"], q2_floor, severity="warn") + k10 = lk.Kriging(y, X, kernel, regmodel=trend, optim="BFGS10") + res10 = metrics.all_metrics(yt, *predict(k10, Xt)[:2]) + suite.metric(case, "pylibkriging BFGS10", **res10) + suite.check_greater(f"{case}: Q2 (BFGS10)", res10["q2"], q2_floor) + +# %% +tab = suite.table().reset_index() +tab = tab[tab["package"] == "pylibkriging"] +tab[["function", "kernel", "trend"]] = tab["case"].str.split("/", expand=True) +pivot = tab.pivot_table(index=["function", "trend"], columns="kernel", values="q2", sort=False) +pivot.style.format("{:.3f}").background_gradient(axis=None, vmin=0.5, vmax=1, cmap="RdYlGn") + +# %% [markdown] +# Expectations that should hold whatever the version (warn-level: they are +# empirical, not mathematical, statements): +# * on the smooth Branin the `exp` kernel (non-differentiable paths) is the least accurate; +# * on the Lim polynomial a `quadratic` trend is at least as good as a `constant` one. + +# %% +q = tab.set_index(["function", "kernel", "trend"])["q2"] +suite.check_true("branin: exp is the worst kernel (constant trend)", + q["branin", "exp", "constant"] <= min(q["branin", k, "constant"] for k in KERNELS), + severity="warn") +suite.check_greater("lim_polynomial/matern5_2: Q2(quadratic) - Q2(constant)", + q["lim_polynomial", "matern5_2", "quadratic"] - q["lim_polynomial", "matern5_2", "constant"], -1e-3, + severity="warn") + +# %% [markdown] +# ## Consistency of `predict` + +# %% +tf = TESTS["branin"] +X, y, Xt, yt = design.dataset(tf, 30, n_test=50, seed=3) +k = lk.Kriging(y, X, "matern5_2") +m, s, cov = predict(k, Xt, return_cov=True) +cov = np.asarray(cov) +suite.check_close("predict: diag(cov) == stdev^2", np.diag(cov), s ** 2, rtol=1e-6, atol=1e-10) +suite.check_close("predict: cov symmetric", cov, cov.T, rtol=1e-10, atol=1e-12) +suite.check_greater("predict: cov PSD (min eigenvalue / max)", np.linalg.eigvalsh(cov).min() / np.abs(cov).max(), -1e-8) +m1 = np.array([predict(k, Xt[i:i + 1])[0][0] for i in range(len(Xt))]) +s1 = np.array([predict(k, Xt[i:i + 1])[1][0] for i in range(len(Xt))]) +suite.check_close("predict: batch == point by point (mean)", m, m1, rtol=1e-8, atol=1e-10 * np.std(y)) +suite.check_close("predict: batch == point by point (stdev)", s, s1, rtol=1e-6, atol=1e-10 * np.std(y)) +# the mean is a function of the point only: duplicating/permuting the batch changes nothing +perm = np.random.default_rng(0).permutation(len(Xt)) +suite.check_close("predict: invariant to the order of the batch", predict(k, Xt[perm])[0], m[perm], rtol=1e-10, atol=1e-10) + +# %% [markdown] +# ### Derivatives of the predictor +# +# `return_deriv=True` gives the gradients of the mean and of the stdev w.r.t. x; +# compared with central finite differences. + +# %% +with suite.check_runs("predict(return_deriv=True)"): + out = k.predict(Xt[:5], True, False, True) + dm, ds = np.asarray(out[3]), np.asarray(out[4]) + h = 1e-6 + fd_m, fd_s = np.zeros_like(dm), np.zeros_like(ds) + for j in range(2): + e = np.zeros(2); e[j] = h + mp, sp, _ = predict(k, Xt[:5] + e) + mm, sm, _ = predict(k, Xt[:5] - e) + fd_m[:, j], fd_s[:, j] = (mp - mm) / (2 * h), (sp - sm) / (2 * h) + suite.check_close("d mean / dx == finite differences", dm, fd_m, rtol=1e-4, atol=1e-4 * np.abs(fd_m).max()) + suite.check_close("d stdev / dx == finite differences", ds, fd_s, rtol=1e-3, atol=1e-3 * np.abs(fd_s).max()) + +# %% [markdown] +# ## A look at the 1D fits + +# %% +fig, axes = plt.subplots(1, 2, figsize=(13, 4)) +for ax, fname in zip(axes, ["forrester", "gramacy_lee_2012"]): + tf = TESTS[fname] + n = CASES[fname][0] + X, y, _, _ = design.dataset(tf, n, seed=1) + u = np.linspace(0, 1, 400)[:, None] + ax.plot(u, tf.unit(u), "k--", label="truth") + for kernel in ["gauss", "matern5_2", "exp"]: + mm, ss, _ = predict(lk.Kriging(y, X, kernel), u) + ax.plot(u, mm, label=kernel) + if kernel == "matern5_2": + ax.fill_between(u[:, 0], mm - 2 * ss, mm + 2 * ss, alpha=0.2) + ax.plot(X, y, "ko") + ax.set_title(fname) + ax.legend() + +# %% [markdown] +# ## Summary +# +# The metrics of this notebook, the number of checks per status, and every check that +# did not pass (FAIL: error-level, fails the run; WARN: warn-level; KNOWN: listed in +# `tools/known_issues.csv`, with its issue; SKIP: feature absent from the installed version). +# All of them are also written to `results/kernels-trends-interpolation_Kriging_python.checks.csv` and `.metrics.csv`. + +# %% +suite.summary() + +# %% +suite.finalize() diff --git a/src/large-n_Kriging-vecchia-nystrom_python.py b/src/large-n_Kriging-vecchia-nystrom_python.py new file mode 100644 index 0000000..0098dd2 --- /dev/null +++ b/src/large-n_Kriging-vecchia-nystrom_python.py @@ -0,0 +1,185 @@ +# %% [markdown] +# # Python · Kriging with `method="vecchia"` / `"nystrom"` — large n +# +# **Idea.** Vecchia and Nyström approximations make Kriging affordable for large n; +# how much accuracy do they cost, and do they converge to the exact model as their +# size parameter grows? +# +# **Reference.** The exact model at the same θ (limits m = n − 1 and k = n), and the +# exact fit. +# +# **Tags.** class: Kriging · method: exact, vecchia, nystrom · topic: large n +# +# The exact fit costs O(n³). `Kriging` offers: +# +# * `method="vecchia(m)"`: Vecchia likelihood with m conditioning neighbours +# (local, recommended in low dimension); +# * `method="nystrom(k)"`: low-rank Nyström approximation with k landmarks +# (global); +# * `Kriging.subsetOfData`: a representative subset before an exact fit. +# +# (`NestedKriging`, the partition-based alternative, on the same problems: +# `aggregations-large-n_NestedKriging_python.ipynb`.) +# +# Problems: Hartmann 3 (3D, local structure: Vecchia's ground) and the OTL +# circuit (6D, smooth: Nyström's ground), n = 1500 (exact fit still affordable, +# for reference) and n = 5000 (approximations only). +# +# Checks: +# * **convergence to the exact model** at fixed θ (Hartmann 3, n = 300): the gap +# between the approximate and the exact log-likelihood and predicted mean, +# as m (Vecchia) and k (Nyström) grow, down to the limit cases m = n − 1 and +# k = n where the approximation *is* the exact model; +# * **limit cases**: Vecchia with m = n − 1 neighbours *is* the exact likelihood; +# * the method is explicit at prediction time: `predict(method="exact")` on a +# Nyström/Vecchia fit raises, `method=` gives finite predictions; +# * accuracy of each approximation close to the exact fit (warn-level gaps, +# error-level floors) and faster than the exact fit at n = 1500 (warn-level). + +# %% +import time + +import matplotlib.pyplot as plt +import numpy as np +import pandas as pd +import pylibkriging as lk + +import gallery +from gallery import TESTS, design, metrics, compat + +suite = gallery.Suite("large-n_Kriging-vecchia-nystrom_python") +print("pylibkriging", compat.version()) +HAS_METHOD = compat.has("method") +if not HAS_METHOD: + suite.check_skip("method=vecchia/nystrom", "pylibkriging without the method= argument") + + +def pred(k, X, **kw): + m, s, *_ = k.predict(X, return_stdev=True, **kw) + return np.ravel(m), np.ravel(s) + + +def timed(f): + t0 = time.time() + out = f() + return out, time.time() - t0 + +# %% [markdown] +# ## Limit case: Vecchia with all the previous points as neighbours is exact + +# %% +if HAS_METHOD: + tf = TESTS["branin"] + X, y, Xt, _ = design.dataset(tf, 40, n_test=100, seed=51) + ke = lk.Kriging(y, X, "matern5_2") + fixed = {"theta": np.asarray(ke.theta()).reshape(1, -1), "is_theta_estim": False} + kv = lk.Kriging(y, X, "matern5_2", optim="none", parameters=fixed, method=f"vecchia({len(y) - 1})") + suite.check_close("vecchia(n-1) LL == exact LL (same theta)", kv.logLikelihood(method="vecchia"), + ke.logLikelihood(), rtol=1e-6) + suite.check_close("vecchia(n-1) predict == exact predict", pred(kv, Xt, method="vecchia")[0], pred(ke, Xt)[0], + rtol=1e-5, atol=1e-6 * np.std(y), severity="warn") + kn = lk.Kriging(y, X, "matern5_2", optim="none", parameters=fixed, method=f"nystrom({len(y)})") + suite.check_close("nystrom(n) LL == exact LL (same theta)", kn.logLikelihood(method="nystrom"), ke.logLikelihood(), + rtol=1e-3, severity="warn") + # the method must be explicit after an approximate fit + suite.check_raises("nystrom fit: predict(method='exact') raises", lambda: kn.predict(Xt, return_stdev=True, + method="exact")) + suite.check_true("method() reports the fit method", kv.method().startswith("vecchia") and + kn.method().startswith("nystrom"), detail=f"{kv.method()}, {kn.method()}") + +# %% [markdown] +# ## Convergence to the exact model, at fixed θ +# +# Hartmann 3, n = 300, θ of the exact fit kept fixed: the reference is the exact +# model with the same hyperparameters, so the gap measures the approximation +# alone (no optimizer effect). Relative gap of the log-likelihood, and RMS gap +# of the predicted mean on 500 test points relative to its standard deviation. + +# %% +if HAS_METHOD: + tf = TESTS["hartmann3"] + X, y, Xt, yt = design.dataset(tf, 300, n_test=500, seed=54) + n = len(y) + theta = np.asarray(lk.Kriging(y, X, "matern5_2").theta()).reshape(1, -1) + fixed = {"theta": theta, "is_theta_estim": False} + ke = lk.Kriging(y, X, "matern5_2", optim="none", parameters=fixed) + ll_e, m_e = ke.logLikelihood(), pred(ke, Xt)[0] + rows = [] + for meth, sizes in [("vecchia", [2, 5, 10, 20, 40, 80, 160, n - 1]), ("nystrom", [10, 25, 50, 100, 200, n])]: + for size in sizes: + with suite.check_runs(f"{meth}({size}) at fixed theta: fit + predict"): + ka = lk.Kriging(y, X, "matern5_2", optim="none", parameters=fixed, method=f"{meth}({size})") + rows.append(dict(method=meth, size=size, + ll_gap=abs(ka.logLikelihood(method=meth) - ll_e) / abs(ll_e), + mean_gap=np.sqrt(np.mean((pred(ka, Xt, method=meth)[0] - m_e) ** 2)) / np.std(m_e))) + conv = pd.DataFrame(rows) + for meth, g in conv.groupby("method"): + suite.metric(f"hartmann3/n={n}/fixed theta", meth, **{f"ll_gap_{s}": v for s, v in zip(g["size"], g["ll_gap"])}) + for col, lim in [("ll_gap", 1e-6), ("mean_gap", 1e-2)]: + suite.check_less(f"{meth}: {col} at the limit size ({g['size'].iloc[-1]})", g[col].iloc[-1], lim) + suite.check_true(f"{meth}: {col} decreases with the size", bool(np.all(np.diff(g[col].to_numpy()) < 0)), + detail=", ".join(f"{v:.2g}" for v in g[col]), severity="warn") + fig, axes = plt.subplots(1, 2, figsize=(10, 3.8)) + for ax, col, label in [(axes[0], "ll_gap", "|LL − LL(exact)| / |LL(exact)|"), + (axes[1], "mean_gap", "RMS(mean − mean(exact)) / sd")]: + for meth, g in conv.groupby("method"): + ax.loglog(g["size"], g[col], "o-", label=f"{meth}(m)" if meth == "vecchia" else f"{meth}(k)") + ax.set_xlabel("m (neighbours) or k (landmarks)"); ax.set_ylabel(label); ax.grid(True, which="both", alpha=.3) + axes[0].legend(); fig.suptitle(f"Hartmann 3, n = {n}, θ fixed: approximation → exact model"); plt.show() + display(conv.pivot(index="size", columns="method", values=["ll_gap", "mean_gap"]).round(6)) + +# %% [markdown] +# ## Accuracy and cost + +# %% +CASES = {"hartmann3": (1500, 5000), "otl_circuit": (1500, 5000)} +FLOORS = {"hartmann3": 0.98, "otl_circuit": 0.99} +for fname, (n_mid, n_big) in CASES.items(): + tf = TESTS[fname] + X, y, Xt, yt = design.dataset(tf, n_big, n_test=2000, seed=52) + Xm, ym = X[:n_mid], y[:n_mid] + ke, t_exact = timed(lambda: lk.Kriging(ym, Xm, "matern5_2")) + res_e = metrics.all_metrics(yt, *pred(ke, Xt)) + suite.metric(f"{fname}/n={n_mid}", "exact", fit_s=t_exact, **res_e) + suite.check_greater(f"{fname}/n={n_mid}: Q2 exact", res_e["q2"], FLOORS[fname]) + variants = {} + if HAS_METHOD: + variants.update({ + "vecchia(30)": lambda Xa, ya: (lk.Kriging(ya, Xa, "matern5_2", method="vecchia(30)"), "vecchia"), + "nystrom(100)": lambda Xa, ya: (lk.Kriging(ya, Xa, "matern5_2", method="nystrom(100)"), "nystrom"), + }) + if compat.has("subsetOfData"): + def sod(Xa, ya): + idx = np.ravel(lk.Kriging.subsetOfData(Xa, n_max=500, method="kmeans", seed=1)).astype(int) + return lk.Kriging(ya[idx], Xa[idx], "matern5_2"), None + variants["subset of data (500)"] = sod + for label, make in variants.items(): + for n in (n_mid, n_big): + case = f"{fname}/n={n}" + with suite.check_runs(f"{case}/{label}: fit + predict"): + (k, meth), t = timed(lambda: make(X[:n], y[:n])) + m, s = pred(k, Xt, method=meth) if meth else ( + lambda o: (np.ravel(o[0]), np.ravel(o[1])))(k.predict(Xt, return_stdev=True)) + suite.check_finite(f"{case}/{label}: finite predictions", m) + res = metrics.all_metrics(yt, m, s) + suite.metric(case, label, fit_s=t, **res) + suite.check_greater(f"{case}/{label}: Q2", res["q2"], FLOORS[fname] - 0.05) + if n == n_mid: + suite.check_greater(f"{case}/{label}: Q2 - Q2(exact)", res["q2"] - res_e["q2"], -0.02, + severity="warn") + suite.check_less(f"{case}/{label}: fit time / exact fit time", t / t_exact, 1.0, severity="warn") +suite.table().round(4) + +# %% [markdown] +# ## Summary +# +# The metrics of this notebook, the number of checks per status, and every check that +# did not pass (FAIL: error-level, fails the run; WARN: warn-level; KNOWN: listed in +# `tools/known_issues.csv`, with its issue; SKIP: feature absent from the installed version). +# All of them are also written to `results/large-n_Kriging-vecchia-nystrom_python.checks.csv` and `.metrics.csv`. + +# %% +suite.summary() + +# %% +suite.finalize() diff --git a/src/noisy-data_Kriging-nugget-noise_python.py b/src/noisy-data_Kriging-nugget-noise_python.py new file mode 100644 index 0000000..8c30625 --- /dev/null +++ b/src/noisy-data_Kriging-nugget-noise_python.py @@ -0,0 +1,177 @@ +# %% [markdown] +# # Python · Kriging with noise (`noise="nugget"`, known variances) — noisy observations +# +# **Idea.** Kriging with noisy observations: an estimated homogeneous nugget, or known +# (heteroscedastic) noise variances, must recover the noise level and predict the +# noise-free function with calibrated intervals. +# +# **Reference.** The true noise variance, replicate averaging, and the interpolating +# (noise-free) fit on the same data. +# +# **Tags.** class: Kriging · noise: none, nugget, variances · +# topic: noisy data, calibration +# +# `Kriging` handles noisy responses through its `noise` argument: `"nugget"` +# estimates one homogeneous noise variance, a vector gives a known variance per +# observation (heteroscedastic case). +# +# | case | function | noise | +# |---|---|---| +# | 1D homoscedastic | Forrester | N(0, 0.5²) | +# | 5D homoscedastic | Friedman | N(0, 1) (the classical "Friedman 1" setup) | +# | 1D heteroscedastic | Gramacy & Lee (2012) | sd growing linearly with x | +# | 2D with replicates | Branin | 4 replicates of 15 points, N(0, 2²) | +# +# Checks: +# * the estimated nugget is close to the true noise variance (factor 3); +# * the noisy fit predicts the noise-free function well (Q² floor), and is +# **smooth**: it does not chase the noisy data between the observations. (At the +# design points themselves the `"nugget"` predictor may return y exactly, as +# DiceKriging's nugget does: the nugget is then a micro-scale variation, and +# the predictor is discontinuous there. This is reported, not checked.); +# * prediction intervals are calibrated: 95% coverage of the *noise-free* truth; +# * a zero noise vector gives the noise-free model; +# * with heteroscedastic noise, giving the true variances beats a single nugget +# (NLPD, warn-level); +# * replicates: averaging the replicates with variance σ²/r is equivalent to +# fitting all the replicates (same predictor, up to the optimizer). + +# %% +import numpy as np +import pandas as pd +import matplotlib.pyplot as plt +import pylibkriging as lk + +import gallery +from gallery import TESTS, design, metrics, compat + +suite = gallery.Suite("noisy-data_Kriging-nugget-noise_python") +print("pylibkriging", compat.version()) + + +def pred(k, X): + m, s, *_ = k.predict(X, return_stdev=True) + return np.ravel(m), np.ravel(s) + +# %% [markdown] +# ## Homoscedastic noise, estimated nugget + +# %% +for fname, n, sd, q2_floor in [("forrester", 40, 0.5, 0.95), ("friedman", 200, 1.0, 0.90)]: + tf = TESTS[fname] + X, y, Xt, yt = design.dataset(tf, n, n_test=2000, seed=11, noise_sd=sd) + k = lk.Kriging(y, X, "matern5_2", noise="nugget", optim="BFGS10") + nugget = float(np.ravel(k.nugget())[0]) + m, s = pred(k, Xt) + res = metrics.all_metrics(yt, m, s) + suite.metric(fname, "nugget", nugget=nugget, true_var=sd ** 2, **res) + suite.check_between(f"{fname}: nugget / true noise variance", nugget / sd ** 2, 1 / 3, 3) + k1 = lk.Kriging(y, X, "matern5_2", noise="nugget") # default single start + suite.metric(fname, "nugget, single start", nugget=float(np.ravel(k1.nugget())[0]), ll=float(k1.logLikelihood()), + **metrics.all_metrics(yt, *pred(k1, Xt))) + suite.check_greater(f"{fname}: LL(single start) - LL(10 restarts)", k1.logLikelihood() - k.logLikelihood(), -1, + severity="warn") + suite.check_greater(f"{fname}: Q2 vs noise-free truth", res["q2"], q2_floor) + mX, _ = pred(k, X) + mX_near, _ = pred(k, np.clip(X + 1e-7, 0, 1)) + suite.metric(fname, "nugget", rms_resid_at_X=metrics.rmse(y, mX) / sd, + rms_resid_near_X=metrics.rmse(y, mX_near) / sd) + suite.check_greater(f"{fname}: smooths the noisy data next to the design points (rms residual / sd)", + metrics.rmse(y, mX_near) / sd, 0.3) + suite.check_between(f"{fname}: coverage of the 95% intervals (noise-free truth)", res["coverage95"], 0.80, 1.0, + severity="warn") + # the noise-free model on the same noisy data interpolates, and generalizes worse + k0 = lk.Kriging(y, X, "matern5_2") + m0, s0 = pred(k0, Xt) + suite.metric(fname, "noise-free fit", **metrics.all_metrics(yt, m0, s0)) + suite.check_less(f"{fname}: RMSE(nugget) < RMSE(interpolating fit)", res["rmse"], metrics.rmse(yt, m0), + severity="warn") +suite.table() + +# %% [markdown] +# ## A zero noise vector is the noise-free model + +# %% +tf = TESTS["branin"] +X, y, Xt, yt = design.dataset(tf, 25, n_test=200, seed=12) +k0 = lk.Kriging(y, X, "matern5_2") +theta = np.asarray(k0.theta()).reshape(1, -1) +fixed = {"theta": theta, "is_theta_estim": False, "sigma2": float(k0.sigma2()), "is_sigma2_estim": False} +kf = lk.Kriging(y, X, "matern5_2", optim="none", parameters=fixed) +with suite.check_runs("noise=zeros, theta and sigma2 fixed: fit"): + kz = lk.Kriging(y, X, "matern5_2", noise=np.zeros(len(y)), optim="none", parameters=fixed) + suite.check_close("noise=zeros: same mean as the noise-free model", pred(kz, Xt)[0], pred(kf, Xt)[0], rtol=1e-6, + atol=1e-8 * np.std(y)) + suite.check_close("noise=zeros: same stdev as the noise-free model", pred(kz, Xt)[1], pred(kf, Xt)[1], rtol=1e-4, + atol=1e-6 * np.std(y)) +with suite.check_runs("noise=zeros, sigma2 estimated: fit", severity="warn"): + lk.Kriging(y, X, "matern5_2", noise=np.zeros(len(y)), optim="none", + parameters={"theta": theta, "is_theta_estim": False}) + +# %% [markdown] +# ## Heteroscedastic noise: known variances vs a single nugget + +# %% +tf = TESTS["gramacy_lee_2012"] +n = 60 +X, _, Xt, yt = design.dataset(tf, n, n_test=2000, seed=13) +sd_x = 0.02 + 0.4 * X[:, 0] # noise sd grows with x (in unit coordinates) +y = tf.unit(X) + np.random.default_rng(13).normal(0, sd_x) +k_known = lk.Kriging(y, X, "matern5_2", noise=sd_x ** 2, optim="BFGS10") +k_nug = lk.Kriging(y, X, "matern5_2", noise="nugget", optim="BFGS10") +res_known = metrics.all_metrics(yt, *pred(k_known, Xt)) +res_nug = metrics.all_metrics(yt, *pred(k_nug, Xt)) +suite.metric("gramacy_lee_2012/hetero", "known variances", **res_known) +suite.metric("gramacy_lee_2012/hetero", "nugget", **res_nug) +suite.check_finite("hetero: predictions finite", pred(k_known, Xt)[0]) +suite.check_less("hetero: NLPD(known variances) <= NLPD(nugget)", res_known["nlpd"] - res_nug["nlpd"], 0.05, + severity="warn") +suite.check_close("noise(): returns the given variances", np.ravel(k_known.noise()), sd_x ** 2, rtol=1e-12) + +u = np.linspace(0, 1, 400)[:, None] +fig, ax = plt.subplots(figsize=(10, 4)) +ax.plot(u, tf.unit(u), "k--", label="truth") +ax.errorbar(X[:, 0], y, yerr=2 * sd_x, fmt="k.", alpha=0.4) +for k, lab in [(k_known, "known variances"), (k_nug, "nugget")]: + m, s = pred(k, u) + ax.plot(u, m, label=lab) + ax.fill_between(u[:, 0], m - 2 * s, m + 2 * s, alpha=0.15) +ax.legend(); + +# %% [markdown] +# ## Replicates +# +# r noisy replicates at the same point carry the same information as their mean +# with variance σ²/r: fitting the replicates with `noise=σ²` or their means with +# `noise=σ²/r` must give the same predictor at a fixed θ. + +# %% +tf = TESTS["branin"] +r, sd = 4, 2.0 +X1, _, Xt, yt = design.dataset(tf, 15, n_test=300, seed=14) +Xr = np.repeat(X1, r, axis=0) +yr = tf.unit(Xr) + np.random.default_rng(14).normal(0, sd, len(Xr)) +ybar = yr.reshape(-1, r).mean(axis=1) +k_rep = lk.Kriging(yr, Xr, "matern5_2", noise=np.full(len(yr), sd ** 2)) +fixed = {"theta": np.asarray(k_rep.theta()).reshape(1, -1), "is_theta_estim": False, + "sigma2": float(k_rep.sigma2()), "is_sigma2_estim": False} +k_rep = lk.Kriging(yr, Xr, "matern5_2", noise=np.full(len(yr), sd ** 2), optim="none", parameters=fixed) +k_avg = lk.Kriging(ybar, X1, "matern5_2", noise=np.full(len(ybar), sd ** 2 / r), optim="none", parameters=fixed) +suite.check_close("replicates == averages with variance/r (mean)", pred(k_rep, Xt)[0], pred(k_avg, Xt)[0], + rtol=1e-6, atol=1e-6 * np.std(yr)) +suite.check_close("replicates == averages with variance/r (stdev)", pred(k_rep, Xt)[1], pred(k_avg, Xt)[1], + rtol=1e-4, atol=1e-6 * np.std(yr)) + +# %% [markdown] +# ## Summary +# +# The metrics of this notebook, the number of checks per status, and every check that +# did not pass (FAIL: error-level, fails the run; WARN: warn-level; KNOWN: listed in +# `tools/known_issues.csv`, with its issue; SKIP: feature absent from the installed version). +# All of them are also written to `results/noisy-data_Kriging-nugget-noise_python.checks.csv` and `.metrics.csv`. + +# %% +suite.summary() + +# %% +suite.finalize() diff --git a/src/objectives-LL-LOO-LMP-gradients_Kriging_python.py b/src/objectives-LL-LOO-LMP-gradients_Kriging_python.py new file mode 100644 index 0000000..626ef6e --- /dev/null +++ b/src/objectives-LL-LOO-LMP-gradients_Kriging_python.py @@ -0,0 +1,219 @@ +# %% [markdown] +# # Python · Kriging — objectives (LL, LOO, LMP), their gradients and optima +# +# **Idea.** The three estimation criteria (likelihood, leave-one-out, marginal +# posterior) must be computed and differentiated correctly, and the fitted θ̂ must be +# an optimum of the criterion it maximises. +# +# **Reference.** Finite differences of each criterion, explicit leave-one-out refits, +# and the criterion profile around θ̂. +# +# **Tags.** class: Kriging · objective: LL, LOO, LMP · topic: gradients, optimization +# +# The hyperparameters θ of `Kriging` are estimated by maximizing the +# log-likelihood (`LL`), maximizing the log marginal posterior (`LMP`), or +# minimizing the leave-one-out error (`LOO`). This notebook checks the machinery +# behind those fits rather than the predictions: +# +# * **analytic gradients** of `logLikelihoodFun`, `leaveOneOutFun`, +# `logMargPostFun` against central finite differences, at random θ, for +# every kernel; +# * **optimality** of the fitted θ: the objective at θ̂ is not improved by small +# perturbations of θ̂ (a local optimum was reached). A component θ_j > 5 on +# the unit cube is a nearly linear direction whose optimum sits at the upper +# bound of θ: moving it further out is allowed to improve the objective; +# * **global vs local optimum**: the gap between the default single-start fit +# and a 20-start fit (warn-level: it measures how often the default fit ends in +# a poor local optimum); +# * **consistency** between `logLikelihood()` and `logLikelihoodFun(theta())`; +# * the closed-form **leave-one-out** predictions of `leaveOneOutVec` against an +# explicit refit without each point (same θ, σ²); +# * multistart (`BFGS10`) never ends below a single `BFGS` run (warn-level: a +# multistart that does worse points to a starting-point issue). +# +# Functions: Branin (2D), Hartmann 3 (3D), Ishigami (3D), Friedman (5D). + +# %% +import matplotlib.pyplot as plt +import numpy as np +import pandas as pd +import pylibkriging as lk + +import gallery +from gallery import TESTS, design, metrics, compat + +suite = gallery.Suite("objectives-LL-LOO-LMP-gradients_Kriging_python") +print("pylibkriging", compat.version()) +CASES = {"branin": 25, "hartmann3": 40, "ishigami": 60, "friedman": 50} +KERNELS = ["gauss", "exp", "matern3_2", "matern5_2"] +rng = np.random.default_rng(0) + + +def objective_fun(k, objective): + """(value, gradient) of the objective as a function of theta.""" + if objective == "LL": + return lambda t, g=True: k.logLikelihoodFun(np.asarray(t, float), g)[:2] + if objective == "LOO": + return lambda t, g=True: k.leaveOneOutFun(np.asarray(t, float), g) + return lambda t, g=True: k.logMargPostFun(np.asarray(t, float), g) + +# %% [markdown] +# ## Analytic gradients vs finite differences + +# %% +rows = [] +for fname, n in CASES.items(): + tf = TESTS[fname] + X, y, _, _ = design.dataset(tf, n, seed=2) + for kernel in KERNELS: + k = lk.Kriging(y, X, kernel) + for objective in ["LL", "LOO", "LMP"]: + f = objective_fun(k, objective) + worst = 0.0 + for _ in range(3): + theta = np.exp(rng.uniform(np.log(0.05), np.log(1.0), tf.d)) + val, grad = f(theta) + fd = metrics.finite_diff_grad(lambda t: f(t, False)[0], theta, h=1e-5) + grad = np.asarray(grad).ravel() + err = np.max(np.abs(grad - fd)) / max(np.max(np.abs(fd)), 1e-8) + worst = max(worst, err) + rows.append((fname, kernel, objective, worst)) + suite.check_less(f"{fname}/{kernel}/{objective}: gradient vs finite diff (rel. err)", worst, 1e-3) +pd.DataFrame(rows, columns=["function", "kernel", "objective", "max rel. error"]).pivot_table( + index=["function", "kernel"], columns="objective", values="max rel. error").style.format("{:.1e}") + +# %% [markdown] +# ## Optimality of the fitted θ, objective by objective + +# %% +SIGN = {"LL": 1, "LMP": 1, "LOO": -1} # +1: maximized, -1: minimized +for fname, n in CASES.items(): + tf = TESTS[fname] + X, y, Xt, yt = design.dataset(tf, n, n_test=1000, seed=2) + for objective in ["LL", "LOO", "LMP"]: + k = lk.Kriging(y, X, "matern5_2", objective=objective) + theta = np.asarray(k.theta()).ravel() + f = objective_fun(k, objective) + best = SIGN[objective] * f(theta, False)[0] + improved = [] + for j in range(tf.d): + for fac in (0.9, 1.1): + t = theta.copy(); t[j] *= fac + at_bound = fac > 1 and theta[j] > 5 + # tolerance: the optimizer's stopping criterion (0.01 on a log-density is immaterial) + tol = 1e-4 * abs(best) + (0.0 if objective == "LOO" else 1e-2) + if SIGN[objective] * f(t, False)[0] > best + tol and not at_bound: + improved.append(f"theta{j + 1}*{fac}") + suite.check_true(f"{fname}/{objective}: theta_hat not improved by +/-10% moves", not improved, + detail=f"theta={np.round(theta, 4)}; improving moves: {improved}") + m, s, *_ = k.predict(Xt, return_stdev=True) + suite.metric(f"{fname}/{objective}", "pylibkriging", **metrics.all_metrics(yt, m, s)) + +# %% +tab = suite.table().reset_index() +tab[["function", "objective"]] = tab["case"].str.split("/", expand=True) +tab.pivot_table(index="function", columns="objective", values=["q2", "coverage95"]).round(3) + +# %% [markdown] +# ## In pictures: each criterion along θ = t · θ̂ (Branin) +# +# Each criterion fitted on its own, then evaluated along the ray t · θ̂ through its +# optimum (t = 1, dashed line): a maximum for LL and LMP, a minimum for LOO. +# The three optima are not the same θ, which is why the fitted models differ. + +# %% +tf = TESTS["branin"] +X, y, _, _ = design.dataset(tf, CASES["branin"], seed=2) +ts = np.exp(np.linspace(np.log(0.2), np.log(5), 61)) +fig, axes = plt.subplots(1, 3, figsize=(13, 3.6)) +for ax, objective in zip(axes, ["LL", "LOO", "LMP"]): + k = lk.Kriging(y, X, "matern5_2", objective=objective) + theta = np.asarray(k.theta()).ravel() + f = objective_fun(k, objective) + vals = [f(t * theta, False)[0] for t in ts] + ax.semilogx(ts, vals); ax.axvline(1, color="k", ls="--", lw=.8) + ax.set_title(f"{objective}, θ̂ = {np.round(theta, 3)}"); ax.set_xlabel("t (θ = t · θ̂)") + i = int(np.argmax(vals) if SIGN[objective] > 0 else np.argmin(vals)) + suite.check_between(f"branin/{objective}: optimum of the profile along t * theta_hat (t)", ts[i], 0.8, 1.25, + severity="warn") +axes[0].set_ylabel("criterion"); plt.tight_layout(); plt.show() + +# %% [markdown] +# ## `logLikelihood()` == `logLikelihoodFun(theta())` (and the same for LOO, LMP) + +# %% +tf = TESTS["hartmann3"] +X, y, _, _ = design.dataset(tf, 40, seed=5) +for objective, getter in [("LL", "logLikelihood"), ("LOO", "leaveOneOut"), ("LMP", "logMargPost")]: + k = lk.Kriging(y, X, "matern5_2", objective=objective) + v_fun = objective_fun(k, objective)(np.asarray(k.theta()).ravel(), False)[0] + suite.check_close(f"{getter}() == {objective} function at theta()", getattr(k, getter)(), v_fun, rtol=1e-8) + +# %% [markdown] +# ## Closed-form leave-one-out vs explicit refits +# +# `leaveOneOutVec(θ)` gives the mean and stdev of the prediction of each `y_i` +# from the n−1 others, by the closed-form (Dubrule) formulas. An explicit refit +# without point i, with θ and σ² fixed (β re-estimated, as in the formula), must +# give the same mean. + +# %% +for fname in ["branin", "hartmann3"]: + tf = TESTS[fname] + X, y, _, _ = design.dataset(tf, 20, seed=6) + for trend in ["constant", "linear"]: + k = lk.Kriging(y, X, "matern5_2", regmodel=trend) + theta, sigma2 = np.asarray(k.theta()).ravel(), float(k.sigma2()) + m_loo, s_loo = (np.asarray(a).ravel() for a in k.leaveOneOutVec(theta)) + m_ref, s_ref = np.zeros(len(y)), np.zeros(len(y)) + for i in range(len(y)): + keep = np.arange(len(y)) != i + ki = lk.Kriging(y[keep], X[keep], "matern5_2", regmodel=trend, optim="none", + parameters={"theta": theta.reshape(1, -1), "is_theta_estim": False, + "sigma2": sigma2, "is_sigma2_estim": False}) + mi, si, *_ = ki.predict(X[i:i + 1], return_stdev=True) + m_ref[i], s_ref[i] = np.ravel(mi)[0], np.ravel(si)[0] + suite.check_close(f"{fname}/{trend}: leaveOneOutVec mean == explicit refits", m_loo, m_ref, + rtol=1e-6, atol=1e-8 * np.std(y)) + suite.check_close(f"{fname}/{trend}: leaveOneOutVec stdev == explicit refits", s_loo, s_ref, + rtol=1e-4, atol=1e-8 * np.std(y), severity="warn") + # leaveOneOut() is the mean squared LOO error at the fitted theta + k_loo = lk.Kriging(y, X, "matern5_2", regmodel=trend, objective="LOO") + m2, _ = k_loo.leaveOneOutVec(np.asarray(k_loo.theta()).ravel()) + suite.check_close(f"{fname}/{trend}: leaveOneOut() == mean squared LOO error", k_loo.leaveOneOut(), + np.mean((np.ravel(m2) - y) ** 2), rtol=1e-6, severity="warn") + +# %% [markdown] +# ## Multistart +# +# Gap between the default single start and 10 / 20 starts. A large gap means the +# default fit is unreliable on this kind of problem. + +# %% +for fname, n in CASES.items(): + tf = TESTS[fname] + X, y, _, _ = design.dataset(tf, n, seed=7) + ll1 = lk.Kriging(y, X, "matern5_2", optim="BFGS").logLikelihood() + ll10 = lk.Kriging(y, X, "matern5_2", optim="BFGS10").logLikelihood() + ll20 = lk.Kriging(y, X, "matern5_2", optim="BFGS20").logLikelihood() + suite.metric(fname, "BFGS", ll=ll1) + suite.metric(fname, "BFGS10", ll=ll10) + suite.metric(fname, "BFGS20", ll=ll20) + suite.check_greater(f"{fname}: LL(BFGS10) - LL(BFGS)", ll10 - ll1, -1e-3 * abs(ll1), severity="warn") + suite.check_less(f"{fname}: LL(BFGS20) - LL(BFGS) (default fit far from the best optimum)", ll20 - ll1, 1.0, + severity="warn") +suite.table().loc[list(CASES)].unstack("package") + +# %% [markdown] +# ## Summary +# +# The metrics of this notebook, the number of checks per status, and every check that +# did not pass (FAIL: error-level, fails the run; WARN: warn-level; KNOWN: listed in +# `tools/known_issues.csv`, with its issue; SKIP: feature absent from the installed version). +# All of them are also written to `results/objectives-LL-LOO-LMP-gradients_Kriging_python.checks.csv` and `.metrics.csv`. + +# %% +suite.summary() + +# %% +suite.finalize() diff --git a/src/robustness-invalid-inputs_Kriging_python.py b/src/robustness-invalid-inputs_Kriging_python.py new file mode 100644 index 0000000..fee0a9e --- /dev/null +++ b/src/robustness-invalid-inputs_Kriging_python.py @@ -0,0 +1,192 @@ +# %% [markdown] +# # Python · Kriging — robustness: degenerate and adversarial inputs +# +# **Idea.** What happens off the happy path: invalid, degenerate or ill-conditioned +# inputs must be rejected or handled, results must not depend on the units, and the +# prediction cost must grow linearly with the number of points. +# +# **Reference.** Equivariance identities (rescaled inputs / outputs give rescaled +# results) and the O(q) cost of `predict`. +# +# **Tags.** class: Kriging · topic: robustness, invariance, cost +# +# What a user eventually feeds a surrogate model, and what must happen: +# +# | input | expected behaviour | +# |---|---| +# | NaN / Inf in X or y | rejected with an exception (never a silent NaN model) | +# | X and y with inconsistent sizes | rejected | +# | a wrong number of warping specs | rejected | +# | duplicated design points, noise-free | either rejected or a finite model (no NaN, no crash) | +# | duplicated design points with `noise="nugget"` | a finite model | +# | constant response y | a finite model predicting the constant | +# | very few points (n = 2, 3) | a finite model | +# | very clustered points (1e-8 apart) with the `gauss` kernel | finite predictions (ill-conditioning) | +# | inputs scaled by 1e6 or 1e-6 with `normalize=True` | the same predictions as on [0,1] | +# | y scaled by 1e8 / shifted by 1e6 | predictions scaled / shifted accordingly (equivariance) | +# | predicting far outside the design | the mean reverts to the trend, the stdev to (slightly above) √σ² | +# | many prediction points | cost linear in their number (mean and stdev only need O(n·q), not O(q²)) | + +# %% +import time + +import matplotlib.pyplot as plt +import numpy as np +import pylibkriging as lk + +import gallery +from gallery import TESTS, design, compat + +suite = gallery.Suite("robustness-invalid-inputs_Kriging_python") +print("pylibkriging", compat.version()) +tf = TESTS["branin"] +X, y, Xt, yt = design.dataset(tf, 20, n_test=200, seed=71) + + +def pred(k, Xn): + m, s, *_ = k.predict(Xn, return_stdev=True) + return np.ravel(m), np.ravel(s) + + +def finite_model(check, make, Xn=Xt, severity="error"): + """The model either raises (acceptable) or predicts finite values.""" + try: + k = make() + except Exception as exc: # noqa: BLE001 + suite.check_true(f"{check}: rejected or finite", True, detail=f"rejected: {type(exc).__name__}") + return None + m, s = pred(k, Xn) + suite.check_true(f"{check}: rejected or finite", np.all(np.isfinite(m)) and np.all(np.isfinite(s)), + detail=f"non-finite: {np.sum(~np.isfinite(m))} mean, {np.sum(~np.isfinite(s))} stdev", + severity=severity) + return k + +# %% [markdown] +# ## Invalid inputs are rejected + +# %% +Xn = X.copy(); Xn[3, 1] = np.nan +yn = y.copy(); yn[5] = np.nan +yi = y.copy(); yi[5] = np.inf +suite.check_raises("NaN in X is rejected", lambda: lk.Kriging(y, Xn, "matern5_2")) +suite.check_raises("NaN in y is rejected", lambda: lk.Kriging(yn, X, "matern5_2")) +suite.check_raises("Inf in y is rejected", lambda: lk.Kriging(yi, X, "matern5_2")) +suite.check_raises("len(y) != nrow(X) is rejected", lambda: lk.Kriging(y[:-1], X, "matern5_2")) +suite.check_raises("unknown kernel is rejected", lambda: lk.Kriging(y, X, "matern7_2")) +suite.check_raises("unknown trend is rejected", lambda: lk.Kriging(y, X, "matern5_2", regmodel="cubic")) +suite.check_raises("unknown objective is rejected", lambda: lk.Kriging(y, X, "matern5_2", objective="XYZ")) +suite.check_raises("noise vector of the wrong length is rejected", + lambda: lk.Kriging(y, X, "matern5_2", noise=np.ones(len(y) - 1))) +suite.check_raises("negative noise variance is rejected", + lambda: lk.Kriging(y, X, "matern5_2", noise=-np.ones(len(y))), severity="warn") +k = lk.Kriging(y, X, "matern5_2") +suite.check_raises("predict with the wrong number of columns is rejected", lambda: k.predict(np.ones((3, 3)), True)) +if compat.has("WarpKriging"): + suite.check_raises("WarpKriging with 1 spec for 2 columns is rejected", lambda: lk.WarpKriging(y, X, ["none"], "gauss")) + suite.check_raises("WarpKriging with an unknown spec is rejected", + lambda: lk.WarpKriging(y, X, ["none", "spline"], "gauss")) + +# %% [markdown] +# ## Degenerate designs and responses + +# %% +Xd = np.vstack([X, X[:3]]); yd = np.concatenate([y, y[:3]]) +finite_model("duplicated points, noise-free", lambda: lk.Kriging(yd, Xd, "matern5_2")) +if compat.has("noise"): + k = finite_model("duplicated points, nugget", lambda: lk.Kriging(yd, Xd, "matern5_2", noise="nugget")) + suite.check_true("duplicated points, nugget: a model is built", k is not None) +kc = finite_model("constant y", lambda: lk.Kriging(np.full(20, 3.7), X, "matern5_2")) +if kc is not None: + suite.check_close("constant y: predicts the constant", pred(kc, Xt)[0], np.full(len(Xt), 3.7), rtol=1e-6, atol=1e-6) +for n in (2, 3): + finite_model(f"n = {n}", lambda: lk.Kriging(y[:n], X[:n], "matern5_2")) +Xclose = np.vstack([X[:10], X[:10] + 1e-8]) +yclose = tf.unit(Xclose) +for kernel in ("gauss", "matern5_2"): + finite_model(f"points 1e-8 apart, {kernel}", lambda: lk.Kriging(yclose, Xclose, kernel)) +# a single input dimension constant over the design +Xconst = np.column_stack([X[:, 0], np.full(20, 0.5)]) +finite_model("one input constant over the design", lambda: lk.Kriging(tf.unit(Xconst), Xconst, "matern5_2")) + +# %% [markdown] +# ## Scale invariance and equivariance + +# %% +k0 = lk.Kriging(y, X, "matern5_2", normalize=True) +m0, s0 = pred(k0, Xt) +for scale in (1e-6, 1e6): + with suite.check_runs(f"X * {scale:g} with normalize=True: fit"): + ks = lk.Kriging(y, X * scale, "matern5_2", normalize=True) + suite.check_close(f"X * {scale:g}, normalize=True: same mean as on [0,1]", pred(ks, Xt * scale)[0], m0, + rtol=1e-3, atol=1e-3 * np.std(y)) +for a, b in [(1e8, 0.0), (1.0, 1e6), (1e-8, 0.0)]: + with suite.check_runs(f"y * {a:g} + {b:g}: fit"): + ka = lk.Kriging(a * y + b, X, "matern5_2", normalize=True) + ma, sa = pred(ka, Xt) + suite.check_close(f"y * {a:g} + {b:g}: mean equivariant", ma, a * m0 + b, rtol=1e-3, atol=1e-3 * a * np.std(y)) + suite.check_close(f"y * {a:g} + {b:g}: stdev scales by |a|", sa, abs(a) * s0, rtol=1e-2, + atol=1e-3 * a * np.std(y)) +# the same without normalize: the optimizer starts from a scale-dependent point, warn-level only +for a, b in [(1e6, 0.0), (1.0, 1e6)]: + ka = lk.Kriging(a * y + b, X, "matern5_2") + suite.check_close(f"y * {a:g} + {b:g} (normalize=False): mean equivariant", pred(ka, Xt)[0], + a * pred(lk.Kriging(y, X, "matern5_2"), Xt)[0] + b, rtol=1e-3, atol=1e-3 * a * np.std(y), + severity="warn") + +# %% [markdown] +# ## Far from the data + +# %% +k = lk.Kriging(y, X, "matern5_2") +far = np.array([[50.0, 50.0], [-30.0, 80.0]]) +m, s = pred(k, far) +suite.check_close("far away: mean -> beta (constant trend)", m, np.full(2, np.ravel(k.beta())[0]), rtol=1e-6, + atol=1e-8) +# universal kriging: sigma2 * (1 + 1 / (1' R^-1 1)) with the trend uncertainty, slightly above sigma2 +suite.check_between("far away: stdev / sqrt(sigma2)", np.max(s) / np.sqrt(k.sigma2()), 1.0 - 1e-6, 1.5) +suite.check_close("far away: same stdev at both points", s[0], s[1], rtol=1e-6) + +# %% [markdown] +# ## Many prediction points +# +# Mean and stdev at q points need O(n·q) operations and memory; only the full +# covariance (`return_cov=True`) is O(q²). The time for q = 1000, 2000, 4000, +# 8000 points must grow linearly (log2 of the ratio per doubling ~ 1). An +# exponent close to 2 means a q × q matrix is formed: a million points then needs +# terabytes. (Not run at 10⁶ points directly: it would exhaust the runner.) + +# %% +times = {} +for q in (1000, 2000, 4000, 8000): + Xq = design.uniform(q, 2, seed=72) + best = np.inf + for _ in range(3): + t0 = time.perf_counter() + pred(k, Xq) + best = min(best, time.perf_counter() - t0) + times[q] = best + suite.metric(f"predict_q={q}", "pylibkriging", seconds=best) +expo = np.log2(times[8000] / times[4000]) +suite.check_less("predict(stdev) time growth exponent in q (8000 vs 4000)", expo, 1.5, + detail=", ".join(f"q={q}: {t:.3f}s" for q, t in times.items())) +qs = np.array(list(times)); ts = np.array(list(times.values())) +plt.figure(figsize=(5, 3.6)) +plt.loglog(qs, ts, "o-", label="predict(return_stdev=True)") +plt.loglog(qs, ts[0] * qs / qs[0], "k--", lw=.8, label="O(q) reference") +plt.loglog(qs, ts[0] * (qs / qs[0]) ** 2, "k:", lw=.8, label="O(q²) reference") +plt.xlabel("q (prediction points)"); plt.ylabel("time (s)"); plt.legend(); plt.grid(True, which="both", alpha=.3) +plt.show() + +# %% [markdown] +# ## Summary +# +# The metrics of this notebook, the number of checks per status, and every check that +# did not pass (FAIL: error-level, fails the run; WARN: warn-level; KNOWN: listed in +# `tools/known_issues.csv`, with its issue; SKIP: feature absent from the installed version). +# All of them are also written to `results/robustness-invalid-inputs_Kriging_python.checks.csv` and `.metrics.csv`. + +# %% +suite.summary() + +# %% +suite.finalize() diff --git a/src/save-load-pickle-copy-accessors_Kriging_python.py b/src/save-load-pickle-copy-accessors_Kriging_python.py new file mode 100644 index 0000000..e6d6d04 --- /dev/null +++ b/src/save-load-pickle-copy-accessors_Kriging_python.py @@ -0,0 +1,123 @@ +# %% [markdown] +# # Python · Kriging — save/load, pickle, copy, accessors +# +# **Idea.** A model must survive a round trip (file, pickle, copy) unchanged, and +# expose its settings and hyperparameters: for every noise model, objective and fit +# method of `Kriging`. +# +# **Reference.** The original model: predictions and hyperparameters must be identical +# to the last bit (1e-12). +# +# **Tags.** class: Kriging · objective: LL, LOO, LMP · method: exact, vecchia, nystrom · +# noise: none, nugget, variances · topic: persistence +# +# Persistence and introspection of `Kriging`, for each of its variants (noise +# models, objectives, approximate fit methods): +# +# * `save` → `load` (class-specific and generic `pylibkriging.load`) gives a model +# with **identical** predictions and hyperparameters; +# * `pickle` / `copy` round trips, and a copy is independent of the original +# (updating one leaves the other untouched); +# * accessors mirror the constructor arguments (`kernel()`, `regmodel()`, +# `optim()`, `objective()`, `X()`, `y()`, ...). +# +# The same persistence checks for the other classes are in their own notebooks +# (`*_WarpKriging_python`, `*_MLPKriging_python`, `*_NestedKriging_python`); the +# scikit-learn estimators in `sklearn-estimators_Kriging_python.ipynb`. +# +# An API notebook: its results are identities (to 1e-12), so they are given as +# the table of checks of the Summary rather than as plots. + +# %% +import copy + +import numpy as np +import pylibkriging as lk + +import gallery +from gallery import TESTS, design, compat + +suite = gallery.Suite("save-load-pickle-copy-accessors_Kriging_python") +print("pylibkriging", compat.version()) + +tf = TESTS["hartmann3"] +X, y, Xt, yt = design.dataset(tf, 40, n_test=100, seed=61) + + +def mean_sd(k, Xn): + out = k.predict(Xn, return_stdev=True) + return np.ravel(out[0]), np.ravel(out[1]) + + +MODELS = { + "Kriging": lambda: lk.Kriging(y, X, "matern5_2", regmodel="linear"), + "Kriging nugget": lambda: lk.Kriging(y, X, "matern3_2", noise="nugget"), + "Kriging noise vector": lambda: lk.Kriging(y, X, "gauss", noise=np.full(len(y), 1e-4)), + "Kriging LOO": lambda: lk.Kriging(y, X, "matern5_2", objective="LOO"), + "Kriging LMP": lambda: lk.Kriging(y, X, "matern5_2", objective="LMP"), +} +if compat.has("method"): + MODELS["Kriging vecchia"] = lambda: lk.Kriging(y, X, "matern5_2", method="vecchia(10)") + MODELS["Kriging nystrom"] = lambda: lk.Kriging(y, X, "matern5_2", method="nystrom(20)") + + +def predict_any(k, Xn): + meth = k.method() if hasattr(k, "method") and isinstance(k, lk.Kriging) else None + if meth and meth != "exact": + out = k.predict(Xn, return_stdev=True, method=meth.split("(")[0]) + return np.ravel(out[0]), np.ravel(out[1]) + return mean_sd(k, Xn) + +# %% [markdown] +# ## save / load, pickle, copy + +# %% +for name, make in MODELS.items(): + with suite.check_runs(f"{name}: fit"): + gallery.persistence.check(suite, name, make(), Xt, predict_any) + +# %% [markdown] +# A copy is independent: updating it leaves the original model unchanged. + +# %% +k = lk.Kriging(y[:30], X[:30], "matern5_2") +m_before = mean_sd(k, Xt)[0] +kc = k.copy() if hasattr(k, "copy") else copy.deepcopy(k) +kc.update(y[30:], X[30:], refit=False) +suite.check_close("update on a copy leaves the original unchanged", mean_sd(k, Xt)[0], m_before, rtol=0, atol=0) +suite.check_true("the updated copy did change", not np.allclose(mean_sd(kc, Xt)[0], m_before)) + +# %% [markdown] +# ## Accessors mirror the constructor + +# %% +k = lk.Kriging(y, X, "matern3_2", regmodel="linear", normalize=False, optim="BFGS", objective="LMP") +for acc, expected in [("kernel", "matern3_2"), ("regmodel", "linear"), ("optim", "BFGS"), ("objective", "LMP"), + ("normalize", False)]: + suite.check_true(f"Kriging.{acc}() == {expected!r}", getattr(k, acc)() == expected, detail=str(getattr(k, acc)())) +suite.check_close("Kriging.X() is the design", np.asarray(k.X()), X, rtol=0, atol=0) +suite.check_close("Kriging.y() is the response", np.ravel(k.y()), y, rtol=0, atol=0) +# with normalize=True: do the accessors return the data as given, or as rescaled internally? +kn = lk.Kriging(y, X * 10, "matern3_2", normalize=True) +suite.check_true("Kriging.normalize() == True", kn.normalize() is True or kn.normalize() == 1) +suite.check_close("normalize=True: X() is the design as given", np.asarray(kn.X()), X * 10, rtol=1e-12, atol=0, + severity="warn") +suite.check_close("normalize=True: y() is the response as given", np.ravel(kn.y()), y, rtol=1e-12, atol=0, + severity="warn") +suite.check_true("Kriging.beta() has 1 + d coefficients (linear trend)", np.ravel(k.beta()).size == 1 + X.shape[1]) +suite.check_true("summary() is a non-empty string", isinstance(k.summary(), str) and len(k.summary()) > 20) +suite.check_true("is_theta_estim() after an optimized fit", bool(k.is_theta_estim())) + +# %% [markdown] +# ## Summary +# +# The metrics of this notebook, the number of checks per status, and every check that +# did not pass (FAIL: error-level, fails the run; WARN: warn-level; KNOWN: listed in +# `tools/known_issues.csv`, with its issue; SKIP: feature absent from the installed version). +# All of them are also written to `results/save-load-pickle-copy-accessors_Kriging_python.checks.csv` and `.metrics.csv`. + +# %% +suite.summary() + +# %% +suite.finalize() diff --git a/src/sklearn-estimators_Kriging_python.py b/src/sklearn-estimators_Kriging_python.py new file mode 100644 index 0000000..2e58c37 --- /dev/null +++ b/src/sklearn-estimators_Kriging_python.py @@ -0,0 +1,117 @@ +# %% [markdown] +# # Python · Kriging — scikit-learn estimators (`pylibkriging.sklearn`) +# +# **Idea.** `pylibkriging.sklearn` makes `Kriging` a scikit-learn estimator, usable in +# pipelines, grid searches and cross-validation. +# +# **Reference.** The native `Kriging` class on the same data (same predictions), and +# scikit-learn's estimator conventions. +# +# **Tags.** class: Kriging · compared to: scikit-learn · topic: scikit-learn API +# +# `KrigingRegressor` (the scikit-learn face of `Kriging`) on Hartmann 3: +# `fit`/`predict` (scikit-learn conventions), the same predictions as the native +# class, `get_params`/`set_params`/`clone`, use inside `Pipeline`, +# `GridSearchCV`, `cross_val_score`, and `sample_y`. + +# %% +import matplotlib.pyplot as plt +import numpy as np +import pandas as pd +import pylibkriging as lk + +import gallery +from gallery import TESTS, design, compat + +suite = gallery.Suite("sklearn-estimators_Kriging_python") +print("pylibkriging", compat.version()) + +tf = TESTS["hartmann3"] +X, y, Xt, yt = design.dataset(tf, 40, n_test=100, seed=61) + + +def mean_sd(k, Xn): + out = k.predict(Xn, return_stdev=True) + return np.ravel(out[0]), np.ravel(out[1]) + +# %% [markdown] +# ## Installing scikit-learn +# +# With pip, into the kernel's environment, numpy pinned to its installed version +# (a package that would downgrade it is not installed: the libKriging extension +# is built against it). A package that cannot be installed is skipped, and +# reported. + +# %% +import importlib +import subprocess +import sys + +import numpy + +for package in ["scikit-learn"]: + r = subprocess.run([sys.executable, "-m", "pip", "install", "--quiet", package, f"numpy=={numpy.__version__}"], + capture_output=True, text=True) + print(f"{package}: " + ("installed" if r.returncode == 0 else "NOT installed\n" + r.stderr.strip()[-800:])) +importlib.invalidate_caches() + +# %% [markdown] +# ## scikit-learn estimators + +# %% +if compat.has("sklearn"): + from sklearn.base import clone + from sklearn.model_selection import GridSearchCV, cross_val_score + from sklearn.pipeline import make_pipeline + from sklearn.preprocessing import MinMaxScaler + from pylibkriging.sklearn import KrigingRegressor + + est = KrigingRegressor(kernel="matern5_2").fit(X, y) + m_sk, s_sk = est.predict(Xt, return_std=True) + suite.check_true("KrigingRegressor.predict returns 1-D arrays (scikit-learn convention)", + np.ndim(m_sk) == 1 and np.ndim(s_sk) == 1, detail=f"shapes {np.shape(m_sk)}, {np.shape(s_sk)}", + severity="warn") + m_sk, s_sk = np.ravel(m_sk), np.ravel(s_sk) + m_nat, s_nat = mean_sd(lk.Kriging(y, X, "matern5_2"), Xt) + suite.check_close("KrigingRegressor == Kriging (mean)", m_sk, m_nat, rtol=1e-8, atol=1e-10) + suite.check_close("KrigingRegressor == Kriging (stdev)", s_sk, s_nat, rtol=1e-6, atol=1e-10) + est2 = clone(est).set_params(kernel="gauss") + suite.check_true("clone + set_params", est2.get_params()["kernel"] == "gauss" and est.get_params()["kernel"] == + "matern5_2") + with suite.check_runs("Pipeline(MinMaxScaler, KrigingRegressor)"): + pipe = make_pipeline(MinMaxScaler(), KrigingRegressor(kernel="matern5_2")).fit(tf.scale(X), y) + suite.check_greater("pipeline Q2 on native inputs", pipe.score(tf.scale(Xt), yt), 0.8) + with suite.check_runs("GridSearchCV over kernels"): + gs = GridSearchCV(KrigingRegressor(), {"kernel": ["gauss", "matern3_2", "matern5_2"]}, cv=3).fit(X, y) + suite.check_true("GridSearchCV picked a kernel", gs.best_params_["kernel"] in ("gauss", "matern3_2", "matern5_2"), + detail=str(gs.best_params_)) + cv = pd.DataFrame(gs.cv_results_)[["param_kernel", "mean_test_score", "std_test_score", "mean_fit_time"]] + for _, r in cv.iterrows(): + suite.metric("hartmann3/GridSearchCV", str(r["param_kernel"]), cv_r2=r["mean_test_score"], + cv_r2_sd=r["std_test_score"], fit_s=r["mean_fit_time"]) + plt.figure(figsize=(5, 3.2)) + plt.bar(cv["param_kernel"].astype(str), cv["mean_test_score"], yerr=cv["std_test_score"], capsize=4) + plt.ylabel("3-fold CV R²"); plt.title(f"GridSearchCV(KrigingRegressor): best = {gs.best_params_['kernel']}") + plt.ylim(min(0, cv["mean_test_score"].min() - .05), 1.02); plt.show() + with suite.check_runs("cross_val_score"): + sc = cross_val_score(KrigingRegressor(kernel="matern5_2"), X, y, cv=4) + suite.check_finite("cross_val_score finite", sc) + with suite.check_runs("sample_y"): + S = est.sample_y(Xt[:10], n_samples=5, random_state=1) + suite.check_true("sample_y shape (q, n_samples)", np.shape(S) == (10, 5), detail=str(np.shape(S))) +else: + suite.check_skip("sklearn estimators", "pylibkriging.sklearn or scikit-learn not available") + +# %% [markdown] +# ## Summary +# +# The metrics of this notebook, the number of checks per status, and every check that +# did not pass (FAIL: error-level, fails the run; WARN: warn-level; KNOWN: listed in +# `tools/known_issues.csv`, with its issue; SKIP: feature absent from the installed version). +# All of them are also written to `results/sklearn-estimators_Kriging_python.checks.csv` and `.metrics.csv`. + +# %% +suite.summary() + +# %% +suite.finalize() diff --git a/src/test-functions-reference-values_no-model_python.py b/src/test-functions-reference-values_no-model_python.py new file mode 100644 index 0000000..3515b58 --- /dev/null +++ b/src/test-functions-reference-values_no-model_python.py @@ -0,0 +1,109 @@ +# %% [markdown] +# # Python · no model — the test functions against their published reference values +# +# **Idea.** Every example of this gallery is only as good as its test functions: +# before fitting anything, each of the 21 functions is evaluated at points where its +# value is published, and its range and shape are plotted. +# +# **Reference.** Published optima and nominal values (Virtual Library of Simulation +# Experiments, Surjanovic & Bingham). +# +# **Tags.** class: none · topic: test functions +# +# Every notebook of this gallery relies on the test functions of +# `gallery.functions` (after the [Virtual Library of Simulation Experiments](https://www.sfu.ca/~ssurjano/)). +# A typo in one of them would silently bias every result built on it, so they are +# checked first, against the reference values published on the library pages +# (global optima, nominal points) and against the known output ranges. + +# %% +import numpy as np +import pandas as pd +import matplotlib.pyplot as plt + +import gallery +from gallery import TESTS, design + +suite = gallery.Suite("test-functions-reference-values_no-model_python") + +# %% [markdown] +# ## Reference values + +# %% +rows = [] +for name, tf in TESTS.items(): + for x, v in tf.known: + fx = float(tf(x)[0]) + suite.check_close(f"{name}: f({np.round(x, 4).tolist()})", fx, v, rtol=1e-4, atol=1e-4) + rows.append((name, tf.d, np.round(x, 4).tolist(), v, fx)) +pd.DataFrame(rows, columns=["function", "d", "x", "reference", "computed"]) + +# %% [markdown] +# ## Output ranges +# +# For the engineering functions without a closed-form optimum, the range over the +# domain is compared with the one reported in the literature (loose bounds: a +# random sample does not reach the extremes). + +# %% +EXPECTED_RANGES = { # (lower bound of min, upper bound of max) over the domain + "otl_circuit": (2.5, 10.0), + "piston": (0.1, 1.3), + "borehole": (2.0, 320.0), + "wing_weight": (120.0, 480.0), + "currin_exp": (1.0, 14.0), + "dette_pepelyshev": (0.0, 100.0), +} +summary = [] +for name, tf in TESTS.items(): + U = design.uniform(20_000, tf.d, seed=1) + y = tf.unit(U) + suite.check_finite(f"{name}: finite on the domain", y) + if name in EXPECTED_RANGES: + lo, hi = EXPECTED_RANGES[name] + suite.check_true(f"{name}: range within [{lo}, {hi}]", lo <= y.min() and y.max() <= hi, + detail=f"sampled range [{y.min():.4g}, {y.max():.4g}]") + summary.append((name, tf.d, ",".join(tf.tags), y.min(), y.max(), y.std())) +pd.DataFrame(summary, columns=["function", "d", "tags", "min", "max", "std"]) + +# %% [markdown] +# ## Designs +# +# The designs are seeded: the same seed gives the same points (reproducibility of +# every notebook), and the optimized LHS is a Latin hypercube (one point per +# stratum in each dimension). + +# %% +A, B = design.lhs(20, 3, seed=7), design.lhs(20, 3, seed=7) +suite.check_close("lhs: reproducible with a seed", A, B, rtol=0, atol=0) +strata = np.floor(A * 20).astype(int) +suite.check_true("lhs: one point per stratum", all(len(set(strata[:, j])) == 20 for j in range(3))) + +# %% +fig, axes = plt.subplots(1, 4, figsize=(16, 3.5)) +for ax, name in zip(axes, ["forrester", "gramacy_lee_2012"]): + tf = TESTS[name] + u = np.linspace(0, 1, 500)[:, None] + ax.plot(tf.scale(u)[:, 0], tf.unit(u)) + ax.set_title(name) +for ax, name in zip(axes[2:], ["branin", "gramacy_lee_2008"]): + tf = TESTS[name] + g = np.linspace(0, 1, 100) + G = np.array(np.meshgrid(g, g)).reshape(2, -1).T + ax.contourf(g, g, tf.unit(G).reshape(100, 100), 30) + ax.set_title(name) +plt.tight_layout() + +# %% [markdown] +# ## Summary +# +# The metrics of this notebook, the number of checks per status, and every check that +# did not pass (FAIL: error-level, fails the run; WARN: warn-level; KNOWN: listed in +# `tools/known_issues.csv`, with its issue; SKIP: feature absent from the installed version). +# All of them are also written to `results/test-functions-reference-values_no-model_python.checks.csv` and `.metrics.csv`. + +# %% +suite.summary() + +# %% +suite.finalize() diff --git a/src/update-simulate_Kriging_python.py b/src/update-simulate_Kriging_python.py new file mode 100644 index 0000000..80d244a --- /dev/null +++ b/src/update-simulate_Kriging_python.py @@ -0,0 +1,181 @@ +# %% [markdown] +# # Python · Kriging — `update`, `simulate`, `update_simulate` +# +# **Idea.** Adding observations (`update`) and drawing conditional paths (`simulate`, +# `update_simulate`) are shortcuts: they must give what the long way gives. +# +# **Reference.** A model fitted from scratch on all the data, and Monte Carlo moments +# against the predictive mean and covariance. +# +# **Tags.** class: Kriging · topic: update, conditional simulation +# +# Sequential use of a fitted model (sequential design, Bayesian optimization): +# +# * `update(y_u, X_u, refit=False)` adds observations through a Cholesky +# (Schur complement) update, keeping θ; β and σ² are re-estimated (profiled) +# on the enlarged data. It must match a model built from scratch on all the +# data with the same θ (`optim="none"`, σ² and β estimated). +# * `update(..., refit=True)` re-optimizes: it must match a fresh fit (up to the +# optimizer: warn-level). +# * `simulate(nsim, seed, X)` draws conditional paths: their empirical mean and +# covariance must converge to `predict`'s mean and covariance; a seed makes +# them reproducible; at the design points they reproduce y. +# * `update_simulate(y_u, X_u)` conditions paths already drawn on new data: the +# corrected paths must be distributed as fresh paths of the updated model. +# +# Functions: Branin (2D), Hartmann 3 (3D), Hartmann 6 (6D). + +# %% +import matplotlib.pyplot as plt +import numpy as np +import pylibkriging as lk + +import gallery +from gallery import TESTS, design, compat + +suite = gallery.Suite("update-simulate_Kriging_python") +print("pylibkriging", compat.version()) + + +def pred(k, X, cov=False): + m, s, c, *_ = k.predict(X, return_stdev=True, return_cov=cov) + return np.ravel(m), np.ravel(s), np.asarray(c) + + +def fixed_params(k): + return {"theta": np.asarray(k.theta()).reshape(1, -1), "is_theta_estim": False, + "sigma2": float(k.sigma2()), "is_sigma2_estim": False} + +# %% [markdown] +# ## `update` vs a fit from scratch + +# %% +for fname, n0, nu in [("branin", 20, 5), ("hartmann3", 30, 10), ("hartmann6", 60, 20)]: + tf = TESTS[fname] + X, y, Xt, yt = design.dataset(tf, n0 + nu, n_test=300, seed=21) + for trend in ["constant", "linear"]: + case = f"{fname}/{trend}" + k = lk.Kriging(y[:n0], X[:n0], "matern5_2", regmodel=trend) + p = fixed_params(k) + k.update(y[n0:], X[n0:], refit=False) + ref = lk.Kriging(y, X, "matern5_2", regmodel=trend, optim="none", + parameters={"theta": p["theta"], "is_theta_estim": False}) + m, s, _ = pred(k, Xt) + mr, sr, _ = pred(ref, Xt) + suite.check_close(f"{case}: update(refit=False) mean == scratch fit, same theta", m, mr, rtol=1e-6, + atol=1e-8 * np.std(y)) + suite.check_close(f"{case}: update(refit=False) stdev == scratch fit, same theta", s, sr, rtol=1e-5, + atol=1e-8 * np.std(y)) + suite.check_close(f"{case}: update(refit=False) sigma2 == scratch fit sigma2", k.sigma2(), ref.sigma2(), + rtol=1e-8) + suite.check_close(f"{case}: update(refit=False) keeps theta", np.ravel(k.theta()), np.ravel(p["theta"]), + rtol=0, atol=0) + suite.check_true(f"{case}: model holds n0+nu points", np.asarray(k.X()).shape[0] == n0 + nu) + # refit=True vs a fresh fit (same optimizer from the same data, possibly another start) + k2 = lk.Kriging(y[:n0], X[:n0], "matern5_2", regmodel=trend) + k2.update(y[n0:], X[n0:], refit=True) + fresh = lk.Kriging(y, X, "matern5_2", regmodel=trend) + suite.check_close(f"{case}: update(refit=True) LL == fresh fit LL", k2.logLikelihood(), fresh.logLikelihood(), + rtol=1e-3, severity="warn") + +# %% [markdown] +# Several small updates in a row equal one update with all the points. + +# %% +tf = TESTS["hartmann3"] +X, y, Xt, _ = design.dataset(tf, 40, n_test=200, seed=22) +ka = lk.Kriging(y[:20], X[:20], "matern5_2") +kb = lk.Kriging(y[:20], X[:20], "matern5_2") +for i in range(20, 40, 5): + ka.update(y[i:i + 5], X[i:i + 5], refit=False) +kb.update(y[20:], X[20:], refit=False) +suite.check_close("4 updates of 5 == 1 update of 20 (mean)", pred(ka, Xt)[0], pred(kb, Xt)[0], rtol=1e-8, atol=1e-10) +suite.check_close("4 updates of 5 == 1 update of 20 (stdev)", pred(ka, Xt)[1], pred(kb, Xt)[1], rtol=1e-6, atol=1e-10) + +# %% [markdown] +# ## `simulate`: moments, seeds, conditioning + +# %% +for fname, n in [("branin", 20), ("hartmann3", 30)]: + tf = TESTS[fname] + X, y, Xt, _ = design.dataset(tf, n, n_test=15, seed=23) + k = lk.Kriging(y, X, "matern5_2") + nsim = 5000 + S = np.asarray(k.simulate(nsim, 123, Xt)) + suite.check_true(f"{fname}: simulate shape (q, nsim)", S.shape == (len(Xt), nsim), detail=str(S.shape)) + m, s, c = pred(k, Xt, cov=True) + z = (S.mean(axis=1) - m) / (s / np.sqrt(nsim)) + suite.check_less(f"{fname}: empirical mean of paths vs predict (max |z|)", np.max(np.abs(z)), 5.0) + rel = np.linalg.norm(np.cov(S) - c) / np.linalg.norm(c) + suite.check_less(f"{fname}: empirical covariance of paths vs predict (rel. Frobenius)", rel, 0.1) + S2 = np.asarray(k.simulate(nsim, 123, Xt)) + suite.check_close(f"{fname}: same seed, same paths", S2, S, rtol=0, atol=0) + S3 = np.asarray(k.simulate(nsim, 124, Xt)) + suite.check_true(f"{fname}: another seed, other paths", not np.allclose(S3, S)) + Sx = np.asarray(k.simulate(20, 1, X[:5])) + # up to the numerical jitter added to the conditional covariance (relative to the prior sd) + suite.check_close(f"{fname}: paths reproduce y at the design points", Sx, np.tile(y[:5, None], (1, 20)), + rtol=0, atol=1e-4 * np.sqrt(k.sigma2())) + +# %% [markdown] +# ## `update_simulate`: paths conditioned on new data +# +# Paths drawn with `will_update=True`, then corrected by `update_simulate(y_u, +# X_u)`, must have the moments of the model updated with (X_u, y_u). + +# %% +tf = TESTS["branin"] +X, y, Xt, _ = design.dataset(tf, 25, n_test=12, seed=24) +n0 = 20 +k = lk.Kriging(y[:n0], X[:n0], "matern5_2") +p = fixed_params(k) +nsim = 5000 +with suite.check_runs("update_simulate runs"): + k.simulate(nsim, 7, Xt, will_update=True) + U = np.asarray(k.update_simulate(y[n0:], X[n0:])) + # paths drawn under (theta, sigma2) of the first model, then conditioned on all the data + ref = lk.Kriging(y, X, "matern5_2", optim="none", parameters=p) + m, s, c = pred(ref, Xt, cov=True) + z = (U.mean(axis=1) - m) / (s / np.sqrt(nsim)) + suite.check_less("update_simulate: mean of corrected paths vs updated model (max |z|)", np.max(np.abs(z)), 5.0) + rel = np.linalg.norm(np.cov(U) - c) / np.linalg.norm(c) + suite.check_less("update_simulate: covariance vs updated model (rel. Frobenius)", rel, 0.1) + +# %% [markdown] +# ## In pictures: Forrester, 6 points, then 2 more +# +# Conditional paths of the 6-point model (left), the same paths corrected by +# `update_simulate` with 2 new observations (right): they now go through the new +# points, without redrawing them. + +# %% +tf = TESTS["forrester"] +X = np.array([[0.0], [0.2], [0.4], [0.6], [0.8], [1.0]]); y = tf.unit(X) +Xu = np.array([[0.7], [0.9]]); yu = tf.unit(Xu) +G = np.linspace(0, 1, 201).reshape(-1, 1) +k = lk.Kriging(y, X, "matern5_2") +P = np.asarray(k.simulate(20, 3, G, will_update=True)) +PU = np.asarray(k.update_simulate(yu, Xu)) +suite.check_less("update_simulate: corrected paths go through the new points (max |path - y_u| / sd(y))", + np.max(np.abs(PU[[140, 180], :] - yu[:, None])) / np.std(y), 1e-3) +fig, axes = plt.subplots(1, 2, figsize=(11, 3.8), sharey=True) +for ax, paths, title in [(axes[0], P, "6 points: conditional paths"), (axes[1], PU, "+ 2 points: update_simulate")]: + ax.plot(G, paths, color="tab:blue", lw=.6, alpha=.6) + ax.plot(G, tf.unit(G), "k--", lw=1, label="Forrester") + ax.plot(X, y, "ko", label="design") + ax.set_title(title); ax.set_xlabel("x") +axes[1].plot(Xu, yu, "rs", label="new points"); axes[1].legend(); plt.show() + +# %% [markdown] +# ## Summary +# +# The metrics of this notebook, the number of checks per status, and every check that +# did not pass (FAIL: error-level, fails the run; WARN: warn-level; KNOWN: listed in +# `tools/known_issues.csv`, with its issue; SKIP: feature absent from the installed version). +# All of them are also written to `results/update-simulate_Kriging_python.checks.csv` and `.metrics.csv`. + +# %% +suite.summary() + +# %% +suite.finalize() diff --git a/src/vs-DiceKriging-RobustGaSP_Kriging_R.R b/src/vs-DiceKriging-RobustGaSP_Kriging_R.R new file mode 100644 index 0000000..8e2a700 --- /dev/null +++ b/src/vs-DiceKriging-RobustGaSP_Kriging_R.R @@ -0,0 +1,142 @@ +# %% [markdown] +# # R · Kriging vs DiceKriging and RobustGaSP +# +# **Idea.** libKriging's `Kriging` follows DiceKriging: at the same hyperparameters it +# must give DiceKriging's numbers, and its default fit must be as accurate as the R +# references. +# +# **Reference.** DiceKriging (exact equality) and RobustGaSP (accuracy, cost). +# +# **Tags.** class: Kriging · kernel: gauss, exp, matern3_2, matern5_2 · +# compared to: DiceKriging, RobustGaSP +# +# DiceKriging is the reference implementation libKriging's `Kriging` follows +# (same product-form kernels and range parametrization, same concentrated +# likelihood). Hence two levels of comparison: +# +# 1. **Same hyperparameters → same numbers** (error-level): the concentrated +# log-likelihood at a given θ (`DiceKriging::logLikFun` vs +# `logLikelihoodFun`), and the universal kriging mean / stdev at fixed +# (θ, σ², β) (`predict(type = "UK")`). +# 2. **Default fits** on the shared datasets of `data/` (Branin, Hartmann 3, +# Borehole), with RobustGaSP as a third package: Q², fit time +# (warn-level comparisons). + +# %% [markdown] +# ## Installing DiceKriging and RobustGaSP +# +# From CRAN (the configured repository, else cloud.r-project.org), into the +# first library of `.libPaths()`, unless already installed. DiceKriging is +# required; without RobustGaSP its comparison is skipped (warn-level). + +# %% +repos <- getOption("repos") +if (is.null(repos) || any(repos == "@CRAN@")) repos <- c(CRAN = "https://cloud.r-project.org") +for (p in c("DiceKriging", "RobustGaSP")) { + if (!requireNamespace(p, quietly = TRUE)) { + message("installing ", p) + tryCatch(install.packages(p, repos = repos, lib = .libPaths()[1]), + error = function(e) message(p, " could not be installed: ", conditionMessage(e))) + } +} + +# %% +suppressPackageStartupMessages({ + library(rlibkriging) + library(DiceKriging) +}) +has_rgasp <- requireNamespace("RobustGaSP", quietly = TRUE) +source("tools/R/gallery.R") +s <- suite("vs-DiceKriging-RobustGaSP_Kriging_R") +cat("rlibkriging", as.character(packageVersion("rlibkriging")), "| DiceKriging", as.character(packageVersion("DiceKriging")), + "| RobustGaSP", if (has_rgasp) as.character(packageVersion("RobustGaSP")) else "absent", "\n") +DATASETS <- c("branin", "hartmann3", "borehole") + +# %% [markdown] +# ## 1. Same hyperparameters, same numbers + +# %% +for (name in DATASETS) { + X <- read_data(name, "X"); y <- as.numeric(read_data(name, "y")); Xt <- read_data(name, "Xt") + d <- ncol(X) + colnames(X) <- colnames(Xt) <- paste0("x", 1:d) + for (kernel in c("gauss", "matern5_2", "matern3_2", "exp")) { + theta <- seq(0.2, 0.6, length.out = d) + k <- Kriging(y, X, kernel, optim = "none", parameters = list(theta = matrix(theta, nrow = 1), is_theta_estim = FALSE)) + km0 <- km(~1, design = data.frame(X), response = y, covtype = kernel, coef.cov = theta, + coef.var = k$sigma2(), coef.trend = as.numeric(k$beta()), control = list(trace = FALSE)) + # concentrated log-likelihood at theta (DiceKriging profiles sigma2 and beta the same way) + km_fit <- km(~1, design = data.frame(X), response = y, covtype = kernel, control = list(trace = FALSE)) + ll_dk <- DiceKriging::logLikFun(theta, km_fit) + ll_lk <- logLikelihoodFun(k, matrix(theta, nrow = 1))$logLikelihood + check_close(s, sprintf("%s/%s: logLikelihood == DiceKriging logLikFun", name, kernel), ll_lk, ll_dk, rtol = 1e-8) + p_dk <- predict(km0, newdata = data.frame(Xt), type = "UK", checkNames = FALSE) + p_lk <- predict(k, Xt, return_stdev = TRUE) + check_close(s, sprintf("%s/%s: UK mean == DiceKriging", name, kernel), p_lk$mean, p_dk$mean, rtol = 1e-6, atol = 1e-8 * sd(y)) + check_close(s, sprintf("%s/%s: UK stdev == DiceKriging", name, kernel), p_lk$stdev, p_dk$sd, rtol = 1e-5, atol = 1e-8 * sd(y)) + } +} + +# %% [markdown] +# ## 2. Default fits + +# %% +results <- list() +for (name in DATASETS) { + X <- read_data(name, "X"); y <- as.numeric(read_data(name, "y")); Xt <- read_data(name, "Xt"); yt <- as.numeric(read_data(name, "yt")) + colnames(X) <- colnames(Xt) <- paste0("x", 1:ncol(X)) + fits <- list( + rlibkriging = function() { k <- Kriging(y, X, "matern5_2"); function(Z) predict(k, Z)$mean }, + `rlibkriging BFGS10` = function() { k <- Kriging(y, X, "matern5_2", optim = "BFGS10"); function(Z) predict(k, Z)$mean }, + DiceKriging = function() { + m <- km(~1, design = data.frame(X), response = y, covtype = "matern5_2", control = list(trace = FALSE)) + function(Z) predict(m, newdata = data.frame(Z), type = "UK", checkNames = FALSE)$mean + }) + if (has_rgasp) fits$RobustGaSP <- function() { + m <- RobustGaSP::rgasp(design = X, response = y, kernel_type = "matern_5_2") + function(Z) RobustGaSP::predict(m, Z)$mean + } + for (pkg in names(fits)) { + t0 <- Sys.time() + r <- tryCatch(fits[[pkg]](), error = function(e) e) + t_fit <- as.numeric(difftime(Sys.time(), t0, units = "secs")) + if (inherits(r, "error")) { + if (startsWith(pkg, "rlibkriging")) check(s, paste0(name, "/", pkg, ": fit"), FALSE, detail = conditionMessage(r)) + next + } + m <- as.numeric(r(Xt)) + metric(s, name, pkg, q2 = q2(yt, m), rmse = rmse(yt, m), fit_s = t_fit) + results[[length(results) + 1]] <- data.frame(dataset = name, package = pkg, q2 = q2(yt, m), fit_s = t_fit) + } +} +res <- do.call(rbind, results) +print(reshape(res[, c("dataset", "package", "q2", "fit_s")], idvar = "dataset", timevar = "package", direction = "wide")) +op <- par(mfrow = c(1, 2), mar = c(4, 4, 2, 1)) +q <- tapply(res$q2, list(res$package, res$dataset), mean) +barplot(pmax(q, 0), beside = TRUE, legend.text = rownames(q), args.legend = list(x = "bottomright", cex = .7), + ylim = c(0, 1), ylab = "Q2 (test set)", main = "Default fits: accuracy") +t <- tapply(res$fit_s, list(res$package, res$dataset), mean) +barplot(t, beside = TRUE, log = "y", ylab = "fit time (s)", main = "Default fits: cost") +par(op) + +# %% +for (name in DATASETS) { + r <- res[res$dataset == name, ] + best_other <- max(r$q2[!startsWith(r$package, "rlibkriging")]) + check_greater(s, paste0(name, ": Q2(rlibkriging) - best other package"), r$q2[r$package == "rlibkriging"] - best_other, -0.02, severity = "warn") + check_greater(s, paste0(name, ": Q2(rlibkriging BFGS10) > 0.9"), r$q2[r$package == "rlibkriging BFGS10"], 0.9) +} + +# %% [markdown] +# ## Summary +# +# The metrics of this notebook, the number of checks per status, and every check that +# did not pass (FAIL: error-level, fails the run; WARN: warn-level; KNOWN: listed in +# `tools/known_issues.csv`, with its issue; SKIP: feature absent from the installed version). +# All of them are also written to `results/vs-DiceKriging-RobustGaSP_Kriging_R.checks.csv` and `.metrics.csv`. + +# %% +suite_summary(s) + +# %% +suite_finalize(s) diff --git a/src/vs-GaussianProcesses_Kriging_julia.jl b/src/vs-GaussianProcesses_Kriging_julia.jl new file mode 100644 index 0000000..46c8288 --- /dev/null +++ b/src/vs-GaussianProcesses_Kriging_julia.jl @@ -0,0 +1,89 @@ +# %% [markdown] +# # Julia · Kriging vs GaussianProcesses.jl +# +# **Idea.** JLibKriging against the main Julia GP package: default fits on the shared +# datasets. +# +# **Reference.** GaussianProcesses.jl (accuracy and cost). +# +# **Tags.** class: Kriging · kernel: matern5_2 · compared to: GaussianProcesses.jl +# +# Default fits on the shared datasets of `data/` (Branin, Hartmann 3, Borehole): +# JLibKriging (`matern5_2`, constant trend, single start and 10 restarts) against +# [GaussianProcesses.jl](https://github.com/STOR-i/GaussianProcesses.jl) +# (Matérn 5/2 ARD, constant mean, MLE), warn-level comparison of Q². +# +# The module is `JLibKriging` (the packaged binding, [JLibKriging.jl](https://github.com/libKriging/JLibKriging.jl)), +# imported under a short name (as GaussianProcesses.jl): both export `predict`, +# `optimize!`, ... so every call is qualified. + +# %% +import JLibKriging as LK +include("tools/julia/gallery.jl") +s = Suite("vs-GaussianProcesses_Kriging_julia") + +# %% [markdown] +# ## Installing GaussianProcesses.jl +# +# Added to the active Julia environment (with Optim.jl, its optimizer). Without +# it, the comparison is skipped (warn-level). + +# %% +import Pkg +have_gp = try + Pkg.add(["GaussianProcesses", "Optim"]) + @eval import GaussianProcesses as GP + @eval import Optim + true +catch e + println("GaussianProcesses.jl could not be installed: ", sprint(showerror, e)) + false +end + +# %% [markdown] +# ## Default fits + +# %% +for name in ("branin", "hartmann3", "borehole") + Xd = read_data(name, "X"); yd = vec(read_data(name, "y")); Xt = read_data(name, "Xt"); yt = vec(read_data(name, "yt")) + t = @elapsed k = LK.Kriging(yd, Xd, "matern5_2") + q_lk = q2(yt, LK.predict(k, Xt; return_stdev=false).mean) + metric!(s, name, "JLibKriging"; q2=q_lk, fit_s=t) + t = @elapsed k10 = LK.Kriging(yd, Xd, "matern5_2"; optim="BFGS10") + q_lk10 = q2(yt, LK.predict(k10, Xt; return_stdev=false).mean) + metric!(s, name, "JLibKriging BFGS10"; q2=q_lk10, fit_s=t) + check!(s, "$name: Q2(JLibKriging BFGS10) > 0.9", q_lk10 > 0.9; value=q_lk10, expected="> 0.9") + if have_gp + try + d = size(Xd, 2) + t = @elapsed begin + gp = GP.GP(permutedims(Xd), yd, GP.MeanConst(sum(yd) / length(yd)), + GP.Mat52Ard(zeros(d), log(sum((yd .- sum(yd) / length(yd)) .^ 2) / length(yd)) / 2), -8.0) + GP.optimize!(gp; noise=false) + end + mg, _ = GP.predict_f(gp, permutedims(Xt)) + q_gp = q2(yt, mg) + metric!(s, name, "GaussianProcesses.jl"; q2=q_gp, fit_s=t) + check!(s, "$name: Q2(JLibKriging) - Q2(GaussianProcesses.jl)", q_lk - q_gp > -0.02; + value=q_lk - q_gp, expected="> -0.02", severity="warn") + catch e + check!(s, "$name: GaussianProcesses.jl fit", false; value=string(typeof(e)), severity="warn") + end + end +end +have_gp || check!(s, "GaussianProcesses.jl available", false; value="absent", severity="warn") +foreach(r -> println(join(r[3:6], " ")), s.metrics) + +# %% [markdown] +# ## Summary +# +# The metrics of this notebook, the number of checks per status, and every check that +# did not pass (FAIL: error-level, fails the run; WARN: warn-level; KNOWN: listed in +# `tools/known_issues.csv`, with its issue; SKIP: feature absent from the installed version). +# All of them are also written to `results/vs-GaussianProcesses_Kriging_julia.checks.csv` and `.metrics.csv`. + +# %% +suite_summary(s) + +# %% +suite_finalize!(s) diff --git a/src/vs-STK_Kriging_octave.m b/src/vs-STK_Kriging_octave.m new file mode 100644 index 0000000..ae87951 --- /dev/null +++ b/src/vs-STK_Kriging_octave.m @@ -0,0 +1,121 @@ +% %% [markdown] +% # Octave · Kriging vs STK +% +% **Idea.** The Octave/MATLAB reference toolbox, STK: at the same hyperparameters +% (Gaussian kernel) both must give the same predictor, and their default fits are +% compared. +% +% **Reference.** STK (exact equality, then accuracy and cost). +% +% **Tags.** class: Kriging · kernel: gauss, matern5_2 · compared to: STK +% +% [STK](https://github.com/stk-kriging/stk) (Small Toolbox for Kriging) is the +% reference kriging toolbox for Octave/MATLAB. +% +% 1. **Same hyperparameters → same numbers** (error-level). STK's anisotropic +% Gaussian covariance is σ² exp(−‖(x−x')/ρ‖²) and libKriging's `gauss` is +% σ² exp(−½ Σ ((xⱼ−x'ⱼ)/θⱼ)²): the same kernel with ρ = θ √2. With a constant +% mean (ordinary/universal kriging, β estimated by GLS in both), the +% predicted mean and variance at fixed (θ, σ²) must coincide. (STK's Matérn +% covariances are radial, libKriging's are product-form: not comparable +% hyperparameter by hyperparameter, hence the Gaussian kernel here.) +% 2. **Default fits** on the shared datasets of `data/`: Matérn 5/2, each +% toolbox's own estimation (warn-level comparison of Q²). +% +% mLibKriging must be on the path (`OCTAVE_PATH` or `addpath`), see +% `tools/install/octave.sh`. + +% %% +source("tools/octave/gallery.m"); +s = suite("vs-STK_Kriging_octave"); + +% %% [markdown] +% ## Installing STK +% +% STK is fetched here (shallow clone of its repository into a temporary +% directory), unless the `STK_PATH` environment variable already points to a copy. +% Without STK, the comparison is skipped (warn-level). + +% %% +stk_path = getenv("STK_PATH"); +if isempty(stk_path) || ~exist(fullfile(stk_path, "stk_init.m"), "file") + stk_path = fullfile(tempdir(), "stk"); + if ~exist(fullfile(stk_path, "stk_init.m"), "file") + [status, out] = system(sprintf("git clone --depth 1 https://github.com/stk-kriging/stk.git %s 2>&1", stk_path)); + printf("git clone stk: status %d\n%s", status, out); + end +end +have_stk = exist(fullfile(stk_path, "stk_init.m"), "file") == 2; +if have_stk + run(fullfile(stk_path, "stk_init.m")); + printf("STK %s from %s\n", stk_version(), stk_path); +else + s = check(s, "STK available", false, "absent", stk_path, "STK could not be installed: comparison skipped", "warn"); +end +datasets = {"branin", "hartmann3", "borehole"}; + +% %% [markdown] +% ## 1. Same hyperparameters, same numbers + +% %% +if have_stk + for i = 1:numel(datasets) + name = datasets{i}; + X = read_data(name, "X"); y = read_data(name, "y"); Xt = read_data(name, "Xt"); + d = columns(X); + theta = linspace(0.2, 0.6, d); + k = Kriging(y, X, "gauss", "constant", false, "none", "LL", Params("theta", theta, "is_theta_estim", false)); + sigma2 = k.sigma2(); + [m_lk, s_lk] = k.predict(Xt, true, false, false); + model = stk_model(@stk_gausscov_aniso, d); + model.param = [log(sigma2); -log(theta(:) * sqrt(2))]; + model.lm = stk_lm_constant(); + zp = stk_predict(model, X, y, Xt); + s = check_close(s, [name ": gauss UK mean == STK"], m_lk, zp.mean, 1e-6, 1e-8 * std(y)); + s = check_close(s, [name ": gauss UK variance == STK"], s_lk.^2, zp.var, 1e-4, 1e-8 * sigma2); + end +end + +% %% [markdown] +% ## 2. Default fits + +% %% +for i = 1:numel(datasets) + name = datasets{i}; + X = read_data(name, "X"); y = read_data(name, "y"); Xt = read_data(name, "Xt"); yt = read_data(name, "yt"); + d = columns(X); + tic; k = Kriging(y, X, "matern5_2"); t = toc; + [m, ~] = k.predict(Xt, false, false, false); + q_lk = q2(yt, m); + s = metric(s, name, "mLibKriging", "q2", q_lk); s = metric(s, name, "mLibKriging", "fit_s", t); + tic; k10 = Kriging(y, X, "matern5_2", "constant", false, "BFGS10", "LL"); t = toc; + [m, ~] = k10.predict(Xt, false, false, false); + q_lk10 = q2(yt, m); + s = metric(s, name, "mLibKriging BFGS10", "q2", q_lk10); s = metric(s, name, "mLibKriging BFGS10", "fit_s", t); + s = check(s, [name ": Q2(mLibKriging BFGS10) > 0.9"], q_lk10 > 0.9, q_lk10, "> 0.9"); + if have_stk + tic; + model = stk_model(@stk_materncov52_aniso, d); + model = stk_param_estim(model, X, y); % (STK >= 2.8 returns the model) + zp = stk_predict(model, X, y, Xt); + t = toc; + q_stk = q2(yt, zp.mean); + s = metric(s, name, "STK", "q2", q_stk); s = metric(s, name, "STK", "fit_s", t); + s = check(s, [name ": Q2(mLibKriging) - Q2(STK)"], q_lk - q_stk > -0.02, q_lk - q_stk, "> -0.02", "", "warn"); + end +end +disp(s.metrics(:, 3:6)) + +% %% [markdown] +% ## Summary +% +% The metrics of this notebook, the number of checks per status, and every check that +% did not pass (FAIL: error-level, fails the run; WARN: warn-level; KNOWN: listed in +% `tools/known_issues.csv`, with its issue; SKIP: feature absent from the installed version). +% All of them are also written to `results/vs-STK_Kriging_octave.checks.csv` and `.metrics.csv`. + +% %% +suite_summary(s); + +% %% +suite_finalize(s); diff --git a/src/vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python.py b/src/vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python.py new file mode 100644 index 0000000..eca82a5 --- /dev/null +++ b/src/vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python.py @@ -0,0 +1,263 @@ +# %% [markdown] +# # Python · Kriging vs scikit-learn, SMT, OpenTURNS, GPy +# +# **Idea.** How does libKriging compare with the Python GP packages, first as an +# implementation (same hyperparameters → same numbers), then as a tool (default fits: +# accuracy, calibration, cost) on 10 functions? +# +# **Reference.** Scikit-learn, SMT, OpenTURNS and GPy. +# +# **Tags.** class: Kriging · kernel: gauss, matern5_2 · +# compared to: scikit-learn, SMT, OpenTURNS, GPy +# +# Two kinds of comparisons, which should not be confused: +# +# 1. **Same model, same hyperparameters → same numbers.** With θ, σ², β fixed, +# libKriging's predictor and likelihood are textbook formulas: another +# implementation must give the same values to rounding error. Done against +# scikit-learn (`RBF` = libKriging's `gauss`, both product-form) and +# OpenTURNS (product of 1D Matérn 5/2 = libKriging's `matern5_2`). A gap +# here is a bug in one of the packages (error-level). +# 2. **Same problem, each package's default fit → accuracy and cost.** Each +# package estimates its hyperparameters its own way (optimizer, restarts, +# bounds, mean model), so differences are expected; the table is a +# benchmark, and the checks are warn-level: libKriging should not be +# markedly less accurate than the best package. +# +# Packages (each installed below, and skipped if that fails): scikit-learn, +# SMT, OpenTURNS, GPy. + +# %% [markdown] +# ## Installing scikit-learn, SMT, OpenTURNS, GPy +# +# With pip, into the kernel's environment, numpy pinned to its installed version +# (a package that would downgrade it is not installed: the libKriging extension +# is built against it). A package that cannot be installed is skipped, and +# reported. + +# %% +import importlib +import subprocess +import sys + +import numpy + +for package in ["scikit-learn", "smt", "openturns", "GPy"]: + r = subprocess.run([sys.executable, "-m", "pip", "install", "--quiet", package, f"numpy=={numpy.__version__}"], + capture_output=True, text=True) + print(f"{package}: " + ("installed" if r.returncode == 0 else "NOT installed\n" + r.stderr.strip()[-800:])) +importlib.invalidate_caches() + +# %% +import time +import warnings + +import matplotlib.pyplot as plt +import numpy as np +import pandas as pd +import pylibkriging as lk + +import gallery +from gallery import TESTS, design, metrics, compat + +warnings.filterwarnings("ignore") +suite = gallery.Suite("vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python") +sk = compat.optional("sklearn.gaussian_process") +smt = compat.optional("smt.surrogate_models") +ot = compat.optional("openturns") +GPy = compat.optional("GPy") +print("pylibkriging", compat.version(), "| sklearn", bool(sk), "| SMT", bool(smt), "| OpenTURNS", bool(ot), + "| GPy", bool(GPy)) + + +def lk_fixed(y, X, kernel, theta, sigma2, beta): + return lk.Kriging(y, X, kernel, optim="none", + parameters={"theta": np.reshape(theta, (1, -1)), "is_theta_estim": False, + "sigma2": float(sigma2), "is_sigma2_estim": False, + "beta": np.reshape(beta, (-1, 1)), "is_beta_estim": False}) + +# %% [markdown] +# ## 1. Same hyperparameters, same numbers + +# %% +tf = TESTS["hartmann3"] +X, y, Xt, yt = design.dataset(tf, 40, n_test=300, seed=81) +kg = lk.Kriging(y, X, "gauss") +theta, sigma2, beta = np.ravel(kg.theta()), float(kg.sigma2()), float(np.ravel(kg.beta())[0]) +k_simple = lk_fixed(y, X, "gauss", theta, sigma2, beta) # simple kriging, known beta +m_lk, s_lk, *_ = (np.ravel(a) for a in k_simple.predict(Xt, return_stdev=True)[:2]) + +if sk is not None: + from sklearn.gaussian_process import GaussianProcessRegressor + from sklearn.gaussian_process.kernels import RBF, ConstantKernel + + kern = ConstantKernel(sigma2, "fixed") * RBF(theta, "fixed") + gp = GaussianProcessRegressor(kern, alpha=1e-12, optimizer=None, normalize_y=False).fit(X, y - beta) + m_sk, s_sk = gp.predict(Xt, return_std=True) + suite.check_close("sklearn RBF == libKriging gauss: simple kriging mean", m_sk + beta, m_lk, rtol=1e-6, + atol=1e-7 * np.std(y)) + suite.check_close("sklearn RBF == libKriging gauss: simple kriging stdev (beta fixed)", s_sk, s_lk, rtol=1e-4, + atol=1e-6 * np.sqrt(sigma2)) + # with beta estimated, libKriging's stdev is the universal kriging one: the simple kriging variance + # plus the trend-estimation term (f - F'C^-1 c)^2 / (F'C^-1 F), computed here by hand + C = kern(X); c = kern(X, Xt) + w = np.linalg.solve(C, c) + one = np.ones((len(y), 1)) + v_uk = s_sk ** 2 + ((1 - one.T @ w) ** 2).ravel() / (one.T @ np.linalg.solve(C, one)).item() + s_uk = np.ravel(kg.predict(Xt, return_stdev=True)[1]) + suite.check_close("libKriging gauss (beta estimated) stdev == universal kriging formula", s_uk, np.sqrt(v_uk), + rtol=1e-4, atol=1e-6 * np.sqrt(sigma2)) + # the (concentrated) log-likelihood at the MLE equals the full Gaussian log-density at (theta, sigma2, beta) + lml = gp.log_marginal_likelihood_value_ + suite.check_close("sklearn log-marginal-likelihood == libKriging logLikelihood()", lml, kg.logLikelihood(), + rtol=1e-6) +else: + suite.check_skip("sklearn exact comparison", "scikit-learn not installed") + +# %% +if ot is not None: + km = lk.Kriging(y, X, "matern5_2") + th, s2 = np.ravel(km.theta()), float(km.sigma2()) + cov = ot.ProductCovarianceModel([ot.MaternModel([t], [1.0], 2.5) for t in th]) + cov.setAmplitude([np.sqrt(s2)]) + basis = ot.ConstantBasisFactory(3).build() + algo = ot.KrigingAlgorithm(ot.Sample(X), ot.Sample(y.reshape(-1, 1)), cov, basis) + algo.setOptimizeParameters(False) + ot.ResourceMap.SetAsScalar("GeneralLinearModelAlgorithm-DefaultOptimizationLowerBound", 1e-6) + algo.run() + res = algo.getResult() + m_ot = np.ravel(res.getConditionalMean(ot.Sample(Xt))) + v_ot = np.ravel(res.getConditionalMarginalVariance(ot.Sample(Xt))) + # OpenTURNS re-estimates beta by GLS: compare with libKriging's universal kriging at the same theta, sigma2 + ku = lk.Kriging(y, X, "matern5_2", optim="none", + parameters={"theta": th.reshape(1, -1), "is_theta_estim": False, + "sigma2": s2, "is_sigma2_estim": False}) + m_u, s_u = (np.ravel(a) for a in ku.predict(Xt, return_stdev=True)[:2]) + suite.check_close("OpenTURNS product Matern 5/2 == libKriging matern5_2: mean", m_ot, m_u, rtol=1e-6, + atol=1e-6 * np.std(y)) + suite.check_close("OpenTURNS product Matern 5/2 == libKriging matern5_2: variance (universal kriging)", v_ot, s_u ** 2, + rtol=1e-4, atol=1e-8 * s2) +else: + suite.check_skip("OpenTURNS exact comparison", "openturns not installed") + +# %% [markdown] +# ## 2. Default fits on a set of test functions +# +# Matérn 5/2 (or the closest the package offers), constant mean, maximum +# likelihood with each package's default optimizer, 3 designs per case. + +# %% +def fit_libkriging(X, y): + k = lk.Kriging(y, X, "matern5_2") + return lambda Z: tuple(np.ravel(a) for a in k.predict(Z, return_stdev=True)[:2]) + + +def fit_libkriging_ms(X, y): + k = lk.Kriging(y, X, "matern5_2", optim="BFGS10") + return lambda Z: tuple(np.ravel(a) for a in k.predict(Z, return_stdev=True)[:2]) + + +PACKAGES = {"pylibkriging": fit_libkriging, "pylibkriging BFGS10": fit_libkriging_ms} + +if sk is not None: + from sklearn.gaussian_process.kernels import Matern + + def fit_sklearn(X, y): + d = X.shape[1] + kern = ConstantKernel(1.0, (1e-3, 1e3)) * Matern(np.full(d, 0.5), (1e-3, 1e3), nu=2.5) + gp = GaussianProcessRegressor(kern, alpha=1e-10, normalize_y=True, n_restarts_optimizer=5, + random_state=0).fit(X, y) + return lambda Z: gp.predict(Z, return_std=True) + PACKAGES["scikit-learn"] = fit_sklearn + +if smt is not None: + def fit_smt(X, y): + m = smt.KRG(corr="matern52", poly="constant", print_global=False) + m.set_training_values(X, y) + m.train() + return lambda Z: (np.ravel(m.predict_values(Z)), np.sqrt(np.maximum(np.ravel(m.predict_variances(Z)), 0))) + PACKAGES["SMT"] = fit_smt + +if ot is not None: + def fit_openturns(X, y): + d = X.shape[1] + cov = ot.MaternModel([0.5] * d, [float(np.std(y))], 2.5) + algo = ot.KrigingAlgorithm(ot.Sample(X), ot.Sample(y.reshape(-1, 1)), cov, ot.ConstantBasisFactory(d).build()) + algo.run() + r = algo.getResult() + return lambda Z: (np.ravel(r.getConditionalMean(ot.Sample(Z))), + np.sqrt(np.maximum(np.ravel(r.getConditionalMarginalVariance(ot.Sample(Z))), 0))) + PACKAGES["OpenTURNS"] = fit_openturns + +if GPy is not None: + def fit_gpy(X, y): + kern = GPy.kern.Matern52(X.shape[1], ARD=True) + m = GPy.models.GPRegression(X, y.reshape(-1, 1), kern, mean_function=None) + m.Gaussian_noise.variance.fix(1e-8) + m.optimize_restarts(3, verbose=False) + def p(Z): + mu, var = m.predict(Z) + return np.ravel(mu), np.sqrt(np.maximum(np.ravel(var) - 1e-8, 0)) + return p + PACKAGES["GPy"] = fit_gpy + +CASES = {"branin": 30, "six_hump_camel": 40, "hartmann3": 50, "ishigami": 100, "friedman": 80, + "otl_circuit": 80, "piston": 100, "borehole": 80, "wing_weight": 100, "hartmann6": 120} +rows = [] +for fname, n in CASES.items(): + tf = TESTS[fname] + for rep in range(3): + X, y, Xt, yt = design.dataset(tf, n, n_test=1000, seed=100 + rep) + for pkg, fit in PACKAGES.items(): + try: + t0 = time.time() + predictor = fit(X, y) + t_fit = time.time() - t0 + t0 = time.time() + m, s = predictor(Xt) + t_pred = time.time() - t0 + rows.append(dict(function=fname, n=n, rep=rep, package=pkg, fit_s=t_fit, predict_s=t_pred, + **metrics.all_metrics(yt, m, s))) + except Exception as exc: # noqa: BLE001 + rows.append(dict(function=fname, n=n, rep=rep, package=pkg, error=str(exc)[:100])) + if pkg.startswith("pylibkriging"): + suite.check_true(f"{fname}/rep{rep}/{pkg}: fit", False, detail=str(exc)[:200]) +res = pd.DataFrame(rows) +summary = res.groupby(["function", "package"], sort=False)[["q2", "nlpd", "coverage95", "fit_s"]].median() +for (fname, pkg), r in summary.iterrows(): + suite.metric(fname, pkg, **r.to_dict()) +summary.unstack("package")["q2"].round(4) + +# %% +q2tab = summary.unstack("package")["q2"].clip(lower=-0.1) +q2tab.plot.bar(figsize=(11, 3.8), width=.8) +plt.ylabel("median Q² (test set)"); plt.ylim(-0.1, 1.02); plt.legend(ncol=3, fontsize=8); plt.grid(axis="y", alpha=.3) +plt.title("Default fits: accuracy per function and package"); plt.show() + +# %% +summary.unstack("package")[["fit_s"]].round(3) + +# %% +for fname in CASES: + s = summary.loc[fname]["q2"] + others = s.drop(index=[p for p in s.index if p.startswith("pylibkriging")]) + if len(others): + suite.check_greater(f"{fname}: Q2(pylibkriging) - best other package", s["pylibkriging"] - others.max(), -0.02, + severity="warn", detail=f"best: {others.idxmax()}") + suite.check_greater(f"{fname}: Q2(pylibkriging BFGS10) - best other package", s["pylibkriging BFGS10"] - others.max(), + -0.05, detail=f"best: {others.idxmax()}") if len(others) else None + suite.check_greater(f"{fname}: Q2(pylibkriging BFGS10) > 0.5", s["pylibkriging BFGS10"], 0.5) + +# %% [markdown] +# ## Summary +# +# The metrics of this notebook, the number of checks per status, and every check that +# did not pass (FAIL: error-level, fails the run; WARN: warn-level; KNOWN: listed in +# `tools/known_issues.csv`, with its issue; SKIP: feature absent from the installed version). +# All of them are also written to `results/vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python.checks.csv` and `.metrics.csv`. + +# %% +suite.summary() + +# %% +suite.finalize() diff --git a/src/warpings-categorical-ordinal_WarpKriging_python.py b/src/warpings-categorical-ordinal_WarpKriging_python.py new file mode 100644 index 0000000..8c1080c --- /dev/null +++ b/src/warpings-categorical-ordinal_WarpKriging_python.py @@ -0,0 +1,218 @@ +# %% [markdown] +# # Python · WarpKriging — non-stationarity, categorical and ordinal inputs +# +# **Idea.** Input warpings handle non-stationarity and categorical or ordinal inputs; +# each warping must beat, or at least match, plain Kriging where it is meant to help, +# without making the predictive variance overconfident. +# +# **Reference.** Plain `Kriging` on the same data (accuracy and NLPD), and `none` +# warping = `Kriging`. +# +# **Tags.** class: WarpKriging, Kriging · +# warping: none, affine, boxcox, kumaraswamy, neural_mono, categorical, ordinal · +# topic: non-stationarity, categorical inputs +# +# `WarpKriging` learns a transform per input variable jointly with the GP +# hyperparameters. Problems where a plain stationary GP is the wrong model: +# +# | case | function | warp | +# |---|---|---| +# | 1D non-stationary | Gramacy & Lee (2012): fast oscillation on the left, flat on the right | `kumaraswamy`, `knots(5)`, `neural_mono(8)`, `boxcox`, `affine` | +# | 2D non-stationary | Gramacy & Lee (2008): a bump in a corner of [−2, 6]² | `kumaraswamy` × 2 | +# | mixed continuous / categorical | Branin + a level-dependent shift and scale | `categorical(5,2)`, `none` | +# | mixed continuous / ordinal | Hartmann 3 with x₁ observed on 6 ordered levels | `ordinal(6)` | +# +# Checks: all-`none` warping reproduces plain `Kriging`; a noise-free +# `WarpKriging` interpolates; predictions are finite; and (warn-level) the warps +# beat plain `Kriging` where they are meant to. + +# %% +import numpy as np +import pandas as pd +import matplotlib.pyplot as plt +import pylibkriging as lk + +import gallery +from gallery import TESTS, design, metrics, compat + +suite = gallery.Suite("warpings-categorical-ordinal_WarpKriging_python") +print("pylibkriging", compat.version()) +if not compat.has("WarpKriging"): + suite.check_skip("WarpKriging", "not available in this pylibkriging") + + +def pred(k, X): + m, s, *_ = k.predict(X, return_stdev=True) + return np.ravel(m), np.ravel(s) + + +BASE_NLPD = {} + + +def evaluate(case, label, k, X, y, Xt, yt): + m, s = pred(k, Xt) + res = metrics.all_metrics(yt, m, s) + # calibration: a warp must not make the stdev collapse where the error is not small + if label.startswith("Kriging"): + BASE_NLPD[case] = res["nlpd"] + else: + suite.check_less(f"{case}/{label}: NLPD - NLPD(Kriging) (overconfident stdev?)", res["nlpd"] - BASE_NLPD[case], + 2.0, severity="warn", detail=f"min stdev {np.min(s):.3g}, max |error|/stdev " + f"{np.max(np.abs(yt - m) / np.maximum(s, 1e-300)):.3g}") + suite.metric(case, label, ll=float(k.logLikelihood()), **res) + suite.check_finite(f"{case}/{label}: predictions finite", m) + mX, _ = pred(k, X) + suite.check_less(f"{case}/{label}: interpolates (max |m(X)-y| / sd(y))", np.max(np.abs(mX - y)) / np.std(y), 1e-3, + severity="warn") + return res + +# %% [markdown] +# ## `none` warping == plain Kriging + +# %% +if compat.has("WarpKriging"): + tf = TESTS["branin"] + X, y, Xt, yt = design.dataset(tf, 30, n_test=500, seed=31) + for kernel in ["gauss", "matern5_2"]: + kp = lk.Kriging(y, X, kernel) + kw = lk.WarpKriging(y, X, ["none", "none"], kernel) + suite.check_close(f"{kernel}: WarpKriging(none) LL == Kriging LL", kw.logLikelihood(), kp.logLikelihood(), + rtol=1e-3, severity="warn") + suite.check_close(f"{kernel}: WarpKriging(none) mean == Kriging mean", pred(kw, Xt)[0], pred(kp, Xt)[0], + rtol=1e-2, atol=1e-2 * np.std(y), severity="warn") + +# %% [markdown] +# ## 1D non-stationary: Gramacy & Lee (2012) + +# %% +WARPS_1D = ["affine", "boxcox", "kumaraswamy", "knots(5)", "neural_mono(8)"] +if compat.has("WarpKriging"): + tf = TESTS["gramacy_lee_2012"] + X, y, Xt, yt = design.dataset(tf, 25, n_test=1000, seed=32) + X = X * 0.98 + 0.01; Xt = Xt * 0.98 + 0.01 # strictly inside (0,1) for boxcox/kumaraswamy + y, yt = tf.unit(X), tf.unit(Xt) + base = evaluate("gl2012", "Kriging", lk.Kriging(y, X, "matern5_2"), X, y, Xt, yt) + fits = {} + for w in WARPS_1D: + with suite.check_runs(f"gl2012/{w}: fit"): + fits[w] = lk.WarpKriging(y, X, [w], "matern5_2") + evaluate("gl2012", w, fits[w], X, y, Xt, yt) + t = suite.table().loc["gl2012"] + best = t.drop(index="Kriging")["rmse"].min() + suite.check_less("gl2012: best warp RMSE / Kriging RMSE", best / base["rmse"], 1.0, severity="warn") + u = np.linspace(0.01, 0.99, 400)[:, None] + fig, ax = plt.subplots(figsize=(11, 4)) + ax.plot(u, tf.unit(u), "k--", lw=2, label="truth") + ax.plot(u, pred(lk.Kriging(y, X, "matern5_2"), u)[0], label="Kriging") + for w in ["kumaraswamy", "knots(5)", "neural_mono(8)"]: + if w in fits: + ax.plot(u, pred(fits[w], u)[0], label=w) + ax.plot(X, y, "ko"); ax.legend(); ax.set_title("Gramacy & Lee (2012), n=25") + display(t.round(4)) + +# %% [markdown] +# ## 2D non-stationary: Gramacy & Lee (2008) + +# %% +if compat.has("WarpKriging"): + tf = TESTS["gramacy_lee_2008"] + X, y, Xt, yt = design.dataset(tf, 40, n_test=2000, seed=33) + X = X * 0.98 + 0.01; Xt = Xt * 0.98 + 0.01 + y, yt = tf.unit(X), tf.unit(Xt) + base = evaluate("gl2008", "Kriging", lk.Kriging(y, X, "matern5_2"), X, y, Xt, yt) + with suite.check_runs("gl2008/kumaraswamy: fit"): + res = evaluate("gl2008", "kumaraswamy", lk.WarpKriging(y, X, ["kumaraswamy"] * 2, "matern5_2"), + X, y, Xt, yt) + suite.check_less("gl2008: RMSE(kumaraswamy) / RMSE(Kriging)", res["rmse"] / base["rmse"], 1.0, severity="warn") + +# %% [markdown] +# ## Mixed continuous / categorical +# +# y = a_c · Branin(x₁, x₂) + b_c for a level c ∈ {0..4} with level-specific +# (a_c, b_c): two pairs of levels are close, the fifth one is different. The +# categorical variable is passed as its level index 0..L−1. + +# %% +if compat.has("WarpKriging"): + tf = TESTS["branin"] + L = 5 + a = np.array([1.0, 1.05, 0.6, 0.65, -0.5]) + b = np.array([0.0, 5.0, 40.0, 45.0, 100.0]) + + def mixed(Xc): + c = Xc[:, 0].astype(int) + return a[c] * tf.unit(Xc[:, 1:]) + b[c] + + def mixed_design(n, seed): + rng = np.random.default_rng(seed) + Xc = np.column_stack([np.arange(n) % L, design.lhs(n, 2, seed=seed)]) + return Xc[rng.permutation(n)] + + X = mixed_design(60, 34); Xt = mixed_design(1000, 35) + y, yt = mixed(X), mixed(Xt) + base = evaluate("mixed", "Kriging (level as number)", lk.Kriging(y, X, "matern5_2"), X, y, Xt, yt) + with suite.check_runs("mixed/categorical: fit"): + kc = lk.WarpKriging(y, X, [f"categorical({L},2)", "none", "none"], "matern5_2") + res = evaluate("mixed", "categorical(5,2)", kc, X, y, Xt, yt) + suite.check_less("mixed: RMSE(categorical) / RMSE(level as number)", res["rmse"] / base["rmse"], 1.0, + severity="warn") + suite.check_greater("mixed/categorical: Q2", res["q2"], 0.8, severity="warn") + +# %% [markdown] +# ## Mixed continuous / ordinal +# +# Hartmann 3 with x₁ only observed on 6 non-equally spaced ordered levels; the +# model sees the level rank 0..5. + +# %% +if compat.has("WarpKriging"): + tf = TESTS["hartmann3"] + levels = np.array([0.0, 0.05, 0.15, 0.4, 0.8, 1.0]) + + def ordinal_design(n, seed): + U = design.lhs(n, 3, seed=seed) + r = np.minimum((U[:, 0] * 6).astype(int), 5) + return np.column_stack([r, U[:, 1:]]).astype(float) + + def f_ord(Xo): + return tf.unit(np.column_stack([levels[Xo[:, 0].astype(int)], Xo[:, 1:]])) + + X = ordinal_design(60, 36); Xt = ordinal_design(1000, 37) + y, yt = f_ord(X), f_ord(Xt) + base = evaluate("ordinal", "Kriging (rank as number)", lk.Kriging(y, X / [5, 1, 1], "matern5_2"), + X / [5, 1, 1], y, Xt / [5, 1, 1], yt) + with suite.check_runs("ordinal/ordinal(6): fit"): + ko = lk.WarpKriging(y, X, ["ordinal(6)", "none", "none"], "matern5_2") + res = evaluate("ordinal", "ordinal(6)", ko, X, y, Xt, yt) + suite.check_less("ordinal: RMSE(ordinal) / RMSE(rank as number)", res["rmse"] / base["rmse"], 1.1, + severity="warn") + +# %% +suite.table().round(4) + +# %% [markdown] +# ## save / load, pickle, copy +# +# On Hartmann 3 (n = 40): a model saved and loaded back (class-specific and +# generic `pylibkriging.load`), pickled, or deep-copied predicts exactly as the +# original. + +# %% +if compat.has("WarpKriging"): + X, y, Xt, yt = design.dataset(TESTS["hartmann3"], 40, n_test=100, seed=61) + with suite.check_runs("WarpKriging: fit (persistence)"): + gallery.persistence.check(suite, "WarpKriging", lk.WarpKriging(y, X, ["kumaraswamy", "none", "affine"], "matern5_2"), Xt, pred) + +# %% [markdown] +# ## Summary +# +# The metrics of this notebook, the number of checks per status, and every check that +# did not pass (FAIL: error-level, fails the run; WARN: warn-level; KNOWN: listed in +# `tools/known_issues.csv`, with its issue; SKIP: feature absent from the installed version). +# All of them are also written to `results/warpings-categorical-ordinal_WarpKriging_python.checks.csv` and `.metrics.csv`. + +# %% +suite.summary() + +# %% +suite.finalize() diff --git a/temp.txt b/temp.txt deleted file mode 100644 index 8b13789..0000000 --- a/temp.txt +++ /dev/null @@ -1 +0,0 @@ - diff --git a/tools/R/gallery.R b/tools/R/gallery.R new file mode 100644 index 0000000..1f429cd --- /dev/null +++ b/tools/R/gallery.R @@ -0,0 +1,138 @@ +# Helpers of the libKriging gallery notebooks (R), in one file: source("tools/R/gallery.R"). +# +# The same functions, under the same names, as tools/python/gallery.py (where they are +# methods of Suite: suite.check_close(...) is check_close(s, ...) here), and the same +# CSV outputs (results/.checks.csv, .metrics.csv), aggregated with the other +# languages by tools/report.py. +# +# suite(name) a suite of checks, named after the notebook +# check(s, check, ok, value, expected, ...) record a check (severity "error" or "warn") +# check_true, check_less, check_greater, check_between, check_close, check_finite, +# check_runs, check_skip the usual checks +# metric(s, case, package, name = value, ...) record metrics +# suite_summary(s) end-of-notebook summary +# suite_finalize(s) write the CSV files, stop if an error-level check failed +# read_data(name, stem), rmse, q2, nlpd, coverage +# +# Statuses: PASS, FAIL (error-level), WARN (warn-level), SKIP (feature absent from the +# installed version), KNOWN (failing, listed in tools/known_issues.csv with its issue). +# Paths are relative to the repository root ($LKGALLERY_ROOT, else the working directory, +# where the notebooks are executed). + +.gallery_root <- Sys.getenv("LKGALLERY_ROOT", ".") +.known_file <- file.path(.gallery_root, "tools", "known_issues.csv") +if (!file.exists(.known_file)) stop("run the notebooks from the repository root (", .known_file, " not found)") +.known <- read.csv(.known_file, stringsAsFactors = FALSE) +results_dir <- Sys.getenv("LKGALLERY_RESULTS", file.path(.gallery_root, "results")) +dir.create(results_dir, showWarnings = FALSE, recursive = TRUE) +.STATUSES <- c("PASS", "FAIL", "WARN", "KNOWN", "SKIP") + +# ---- suite and checks ----------------------------------------------------------------- + +suite <- function(name) { + s <- new.env() + s$name <- name + s$checks <- list() + s$metrics <- list() + s$t0 <- Sys.time() + s +} + +.fmt <- function(v) if (is.null(v) || length(v) == 0) "" else if (is.numeric(v)) format(signif(v[1], 6)) else as.character(v[1]) + +.record <- function(s, check, severity, status, value, expected, detail) { + s$checks[[length(s$checks) + 1]] <- data.frame(suite = s$name, lang = "R", check = check, severity = severity, + status = status, value = .fmt(value), expected = .fmt(expected), detail = detail, stringsAsFactors = FALSE) +} + +check <- function(s, check, ok, value = NULL, expected = NULL, detail = "", severity = "error") { + ok <- isTRUE(ok) + status <- if (ok) "PASS" else if (severity == "error") "FAIL" else "WARN" + k <- .known[.known$suite == s$name & .known$check == check, ] + if (!ok && nrow(k) > 0) { status <- "KNOWN"; detail <- paste(k$issue[1], detail, sep = " | ") } + if (!ok) message(sprintf("[%s] %s: value=%s expected %s %s", status, check, .fmt(value), .fmt(expected), detail)) + .record(s, check, severity, status, value, expected, detail) + invisible(ok) +} + +check_true <- function(s, check, cond, detail = "", severity = "error") + check(s, check, isTRUE(cond), cond, TRUE, detail, severity) + +check_less <- function(s, check, value, bound, detail = "", severity = "error") + check(s, check, is.finite(value) && value < bound, value, paste("<", bound), detail, severity) + +check_greater <- function(s, check, value, bound, detail = "", severity = "error") + check(s, check, is.finite(value) && value > bound, value, paste(">", bound), detail, severity) + +check_between <- function(s, check, value, lo, hi, detail = "", severity = "error") + check(s, check, is.finite(value) && value >= lo && value <= hi, value, sprintf("in [%g, %g]", lo, hi), detail, severity) + +check_close <- function(s, check, a, b, rtol = 1e-6, atol = 0, detail = "", severity = "error") { + a <- as.numeric(a); b <- as.numeric(b) + if (length(a) != length(b)) + return(check(s, check, FALSE, paste("length", length(a)), paste("length", length(b)), detail, severity)) + gap <- abs(a - b) + ok <- all(is.finite(a)) && all(gap <= atol + rtol * abs(b)) + rel <- max(gap / pmax(abs(b), 1e-300)) + check(s, check, ok, rel, sprintf("rtol %g, atol %g", rtol, atol), detail, severity) +} + +check_finite <- function(s, check, a, detail = "", severity = "error") + check(s, check, length(a) > 0 && all(is.finite(a)), paste(sum(!is.finite(a)), "non-finite"), "all finite", detail, severity) + +check_runs <- function(s, check, expr, severity = "error") { + # check_runs(s, "fit", { ... }): an error becomes a failed check instead of stopping the notebook + r <- tryCatch({ force(expr); NULL }, error = function(e) conditionMessage(e)) + check(s, check, is.null(r), if (is.null(r)) "ok" else "error", "no error", if (is.null(r)) "" else substr(r, 1, 200), severity) +} + +check_skip <- function(s, check, reason) { + .record(s, check, "info", "SKIP", "", "", reason) + message("[SKIP] ", check, ": ", reason) +} + +# ---- metrics and results -------------------------------------------------------------- + +metric <- function(s, case, package, ...) { + v <- list(...) + for (n in names(v)) s$metrics[[length(s$metrics) + 1]] <- data.frame(suite = s$name, lang = "R", case = case, + package = package, metric = n, value = .fmt(v[[n]]), stringsAsFactors = FALSE) +} + +suite_summary <- function(s) { + # end-of-notebook summary: metrics, count per status, every check that did not pass + if (length(s$metrics) > 0) { + me <- do.call(rbind, s$metrics) + me$value <- as.numeric(me$value) + print(reshape(me[, c("case", "package", "metric", "value")], idvar = c("case", "package"), + timevar = "metric", direction = "wide"), row.names = FALSE) + } + ch <- do.call(rbind, s$checks) + print(table(factor(ch$status, levels = .STATUSES))) + other <- ch[ch$status != "PASS", c("status", "severity", "check", "value", "expected", "detail")] + if (nrow(other) > 0) print(other, row.names = FALSE) + invisible(ch) +} + +suite_finalize <- function(s) { + metric(s, "_suite", "_", elapsed_s = as.numeric(difftime(Sys.time(), s$t0, units = "secs"))) + ch <- do.call(rbind, s$checks) + me <- do.call(rbind, s$metrics) + write.csv(ch, file.path(results_dir, paste0(s$name, ".checks.csv")), row.names = FALSE) + write.csv(me, file.path(results_dir, paste0(s$name, ".metrics.csv")), row.names = FALSE) + print(table(factor(ch$status, levels = .STATUSES))) + failed <- ch$check[ch$status == "FAIL"] + if (length(failed) > 0) + stop(sprintf("%d check(s) failed in %s: %s", length(failed), s$name, paste(head(failed, 20), collapse = "; "))) + invisible(ch) +} + +# ---- data and metrics ----------------------------------------------------------------- + +read_data <- function(name, stem) # data/_.csv (no header) as a matrix + as.matrix(read.csv(file.path(.gallery_root, "data", paste0(name, "_", stem, ".csv")), header = FALSE)) + +rmse <- function(y, m) sqrt(mean((y - m)^2)) +q2 <- function(y, m) 1 - sum((y - m)^2) / sum((y - mean(y))^2) +nlpd <- function(y, m, s) { s <- pmax(s, 1e-12); mean(0.5 * log(2 * pi * s^2) + 0.5 * ((y - m) / s)^2) } +coverage <- function(y, m, s, level = 0.95) mean(abs(y - m) <= qnorm(0.5 + level / 2) * s) diff --git a/tools/baseline/metrics-master.csv b/tools/baseline/metrics-master.csv new file mode 100644 index 0000000..4f13d43 --- /dev/null +++ b/tools/baseline/metrics-master.csv @@ -0,0 +1,956 @@ +suite,lang,case,package,metric,value +kernels-trends-interpolation_Kriging_python,python,forrester/gauss/constant,pylibkriging,fit_s,0.00161386 +kernels-trends-interpolation_Kriging_python,python,forrester/gauss/constant,pylibkriging,rmse,0.0848271 +kernels-trends-interpolation_Kriging_python,python,forrester/gauss/constant,pylibkriging,q2,0.999634 +kernels-trends-interpolation_Kriging_python,python,forrester/gauss/constant,pylibkriging,nlpd,-3.06486 +kernels-trends-interpolation_Kriging_python,python,forrester/gauss/linear,pylibkriging,fit_s,0.00126195 +kernels-trends-interpolation_Kriging_python,python,forrester/gauss/linear,pylibkriging,rmse,0.0855468 +kernels-trends-interpolation_Kriging_python,python,forrester/gauss/linear,pylibkriging,q2,0.999628 +kernels-trends-interpolation_Kriging_python,python,forrester/gauss/linear,pylibkriging,nlpd,-3.11029 +kernels-trends-interpolation_Kriging_python,python,forrester/gauss/interactive,pylibkriging,fit_s,0.00143647 +kernels-trends-interpolation_Kriging_python,python,forrester/gauss/interactive,pylibkriging,rmse,0.0855468 +kernels-trends-interpolation_Kriging_python,python,forrester/gauss/interactive,pylibkriging,q2,0.999628 +kernels-trends-interpolation_Kriging_python,python,forrester/gauss/interactive,pylibkriging,nlpd,-3.11029 +kernels-trends-interpolation_Kriging_python,python,forrester/gauss/quadratic,pylibkriging,fit_s,0.000566483 +kernels-trends-interpolation_Kriging_python,python,forrester/gauss/quadratic,pylibkriging,rmse,1.57249 +kernels-trends-interpolation_Kriging_python,python,forrester/gauss/quadratic,pylibkriging,q2,0.874394 +kernels-trends-interpolation_Kriging_python,python,forrester/gauss/quadratic,pylibkriging,nlpd,5.33879 +kernels-trends-interpolation_Kriging_python,python,forrester/exp/constant,pylibkriging,fit_s,0.00041604 +kernels-trends-interpolation_Kriging_python,python,forrester/exp/constant,pylibkriging,rmse,0.721582 +kernels-trends-interpolation_Kriging_python,python,forrester/exp/constant,pylibkriging,q2,0.973551 +kernels-trends-interpolation_Kriging_python,python,forrester/exp/constant,pylibkriging,nlpd,1.54551 +kernels-trends-interpolation_Kriging_python,python,forrester/exp/linear,pylibkriging,fit_s,0.000438213 +kernels-trends-interpolation_Kriging_python,python,forrester/exp/linear,pylibkriging,rmse,0.793413 +kernels-trends-interpolation_Kriging_python,python,forrester/exp/linear,pylibkriging,q2,0.968024 +kernels-trends-interpolation_Kriging_python,python,forrester/exp/linear,pylibkriging,nlpd,1.50639 +kernels-trends-interpolation_Kriging_python,python,forrester/exp/interactive,pylibkriging,fit_s,0.000480413 +kernels-trends-interpolation_Kriging_python,python,forrester/exp/interactive,pylibkriging,rmse,0.793413 +kernels-trends-interpolation_Kriging_python,python,forrester/exp/interactive,pylibkriging,q2,0.968024 +kernels-trends-interpolation_Kriging_python,python,forrester/exp/interactive,pylibkriging,nlpd,1.50639 +kernels-trends-interpolation_Kriging_python,python,forrester/exp/quadratic,pylibkriging,fit_s,0.000730038 +kernels-trends-interpolation_Kriging_python,python,forrester/exp/quadratic,pylibkriging,rmse,0.723267 +kernels-trends-interpolation_Kriging_python,python,forrester/exp/quadratic,pylibkriging,q2,0.973428 +kernels-trends-interpolation_Kriging_python,python,forrester/exp/quadratic,pylibkriging,nlpd,1.28782 +kernels-trends-interpolation_Kriging_python,python,forrester/matern3_2/constant,pylibkriging,fit_s,0.00060606 +kernels-trends-interpolation_Kriging_python,python,forrester/matern3_2/constant,pylibkriging,rmse,0.334007 +kernels-trends-interpolation_Kriging_python,python,forrester/matern3_2/constant,pylibkriging,q2,0.994333 +kernels-trends-interpolation_Kriging_python,python,forrester/matern3_2/constant,pylibkriging,nlpd,-0.227715 +kernels-trends-interpolation_Kriging_python,python,forrester/matern3_2/linear,pylibkriging,fit_s,0.000588894 +kernels-trends-interpolation_Kriging_python,python,forrester/matern3_2/linear,pylibkriging,rmse,0.364477 +kernels-trends-interpolation_Kriging_python,python,forrester/matern3_2/linear,pylibkriging,q2,0.993252 +kernels-trends-interpolation_Kriging_python,python,forrester/matern3_2/linear,pylibkriging,nlpd,-0.187262 +kernels-trends-interpolation_Kriging_python,python,forrester/matern3_2/interactive,pylibkriging,fit_s,0.000905514 +kernels-trends-interpolation_Kriging_python,python,forrester/matern3_2/interactive,pylibkriging,rmse,0.364477 +kernels-trends-interpolation_Kriging_python,python,forrester/matern3_2/interactive,pylibkriging,q2,0.993252 +kernels-trends-interpolation_Kriging_python,python,forrester/matern3_2/interactive,pylibkriging,nlpd,-0.187262 +kernels-trends-interpolation_Kriging_python,python,forrester/matern3_2/quadratic,pylibkriging,fit_s,0.000975609 +kernels-trends-interpolation_Kriging_python,python,forrester/matern3_2/quadratic,pylibkriging,rmse,0.435609 +kernels-trends-interpolation_Kriging_python,python,forrester/matern3_2/quadratic,pylibkriging,q2,0.990361 +kernels-trends-interpolation_Kriging_python,python,forrester/matern3_2/quadratic,pylibkriging,nlpd,-0.183076 +kernels-trends-interpolation_Kriging_python,python,forrester/matern5_2/constant,pylibkriging,fit_s,0.000726223 +kernels-trends-interpolation_Kriging_python,python,forrester/matern5_2/constant,pylibkriging,rmse,0.281945 +kernels-trends-interpolation_Kriging_python,python,forrester/matern5_2/constant,pylibkriging,q2,0.995962 +kernels-trends-interpolation_Kriging_python,python,forrester/matern5_2/constant,pylibkriging,nlpd,-1.01945 +kernels-trends-interpolation_Kriging_python,python,forrester/matern5_2/constant,pylibkriging BFGS10,rmse,0.281945 +kernels-trends-interpolation_Kriging_python,python,forrester/matern5_2/constant,pylibkriging BFGS10,q2,0.995962 +kernels-trends-interpolation_Kriging_python,python,forrester/matern5_2/constant,pylibkriging BFGS10,nlpd,-1.01945 +kernels-trends-interpolation_Kriging_python,python,forrester/matern5_2/linear,pylibkriging,fit_s,0.000932932 +kernels-trends-interpolation_Kriging_python,python,forrester/matern5_2/linear,pylibkriging,rmse,0.298959 +kernels-trends-interpolation_Kriging_python,python,forrester/matern5_2/linear,pylibkriging,q2,0.99546 +kernels-trends-interpolation_Kriging_python,python,forrester/matern5_2/linear,pylibkriging,nlpd,-0.95551 +kernels-trends-interpolation_Kriging_python,python,forrester/matern5_2/interactive,pylibkriging,fit_s,0.000725269 +kernels-trends-interpolation_Kriging_python,python,forrester/matern5_2/interactive,pylibkriging,rmse,0.298959 +kernels-trends-interpolation_Kriging_python,python,forrester/matern5_2/interactive,pylibkriging,q2,0.99546 +kernels-trends-interpolation_Kriging_python,python,forrester/matern5_2/interactive,pylibkriging,nlpd,-0.95551 +kernels-trends-interpolation_Kriging_python,python,forrester/matern5_2/quadratic,pylibkriging,fit_s,0.00131941 +kernels-trends-interpolation_Kriging_python,python,forrester/matern5_2/quadratic,pylibkriging,rmse,0.365039 +kernels-trends-interpolation_Kriging_python,python,forrester/matern5_2/quadratic,pylibkriging,q2,0.993231 +kernels-trends-interpolation_Kriging_python,python,forrester/matern5_2/quadratic,pylibkriging,nlpd,-0.790032 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/gauss/constant,pylibkriging,fit_s,0.0016458 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/gauss/constant,pylibkriging,rmse,0.191274 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/gauss/constant,pylibkriging,q2,0.978742 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/gauss/constant,pylibkriging,nlpd,-0.704948 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/gauss/linear,pylibkriging,fit_s,0.00319505 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/gauss/linear,pylibkriging,rmse,0.236028 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/gauss/linear,pylibkriging,q2,0.96763 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/gauss/linear,pylibkriging,nlpd,-0.580993 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/gauss/interactive,pylibkriging,fit_s,0.00250673 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/gauss/interactive,pylibkriging,rmse,0.236028 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/gauss/interactive,pylibkriging,q2,0.96763 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/gauss/interactive,pylibkriging,nlpd,-0.580993 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/gauss/quadratic,pylibkriging,fit_s,0.00243235 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/gauss/quadratic,pylibkriging,rmse,0.138869 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/gauss/quadratic,pylibkriging,q2,0.988795 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/gauss/quadratic,pylibkriging,nlpd,-0.530437 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/exp/constant,pylibkriging,fit_s,0.000752449 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/exp/constant,pylibkriging,rmse,0.196586 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/exp/constant,pylibkriging,q2,0.977545 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/exp/constant,pylibkriging,nlpd,-0.0621192 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/exp/linear,pylibkriging,fit_s,0.0010047 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/exp/linear,pylibkriging,rmse,0.20979 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/exp/linear,pylibkriging,q2,0.974427 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/exp/linear,pylibkriging,nlpd,-0.0466568 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/exp/interactive,pylibkriging,fit_s,0.000931501 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/exp/interactive,pylibkriging,rmse,0.20979 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/exp/interactive,pylibkriging,q2,0.974427 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/exp/interactive,pylibkriging,nlpd,-0.0466568 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/exp/quadratic,pylibkriging,fit_s,0.0025785 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/exp/quadratic,pylibkriging,rmse,0.220579 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/exp/quadratic,pylibkriging,q2,0.971729 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/exp/quadratic,pylibkriging,nlpd,0.136521 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern3_2/constant,pylibkriging,fit_s,0.000761509 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern3_2/constant,pylibkriging,rmse,0.177253 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern3_2/constant,pylibkriging,q2,0.981744 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern3_2/constant,pylibkriging,nlpd,-0.482212 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern3_2/linear,pylibkriging,fit_s,0.00132823 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern3_2/linear,pylibkriging,rmse,0.211499 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern3_2/linear,pylibkriging,q2,0.974009 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern3_2/linear,pylibkriging,nlpd,-0.375844 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern3_2/interactive,pylibkriging,fit_s,0.000978708 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern3_2/interactive,pylibkriging,rmse,0.211499 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern3_2/interactive,pylibkriging,q2,0.974009 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern3_2/interactive,pylibkriging,nlpd,-0.375844 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern3_2/quadratic,pylibkriging,fit_s,0.00228167 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern3_2/quadratic,pylibkriging,rmse,0.173012 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern3_2/quadratic,pylibkriging,q2,0.982607 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern3_2/quadratic,pylibkriging,nlpd,-0.223308 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern5_2/constant,pylibkriging,fit_s,0.00131416 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern5_2/constant,pylibkriging,rmse,0.177638 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern5_2/constant,pylibkriging,q2,0.981665 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern5_2/constant,pylibkriging,nlpd,-0.575421 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern5_2/constant,pylibkriging BFGS10,rmse,0.177638 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern5_2/constant,pylibkriging BFGS10,q2,0.981665 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern5_2/constant,pylibkriging BFGS10,nlpd,-0.575422 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern5_2/linear,pylibkriging,fit_s,0.00130415 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern5_2/linear,pylibkriging,rmse,0.219759 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern5_2/linear,pylibkriging,q2,0.971939 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern5_2/linear,pylibkriging,nlpd,-0.458933 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern5_2/interactive,pylibkriging,fit_s,0.00119758 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern5_2/interactive,pylibkriging,rmse,0.219759 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern5_2/interactive,pylibkriging,q2,0.971939 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern5_2/interactive,pylibkriging,nlpd,-0.458933 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern5_2/quadratic,pylibkriging,fit_s,0.00169826 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern5_2/quadratic,pylibkriging,rmse,0.160506 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern5_2/quadratic,pylibkriging,q2,0.985031 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern5_2/quadratic,pylibkriging,nlpd,-0.337276 +kernels-trends-interpolation_Kriging_python,python,branin/gauss/constant,pylibkriging,fit_s,0.00104284 +kernels-trends-interpolation_Kriging_python,python,branin/gauss/constant,pylibkriging,rmse,7.2629 +kernels-trends-interpolation_Kriging_python,python,branin/gauss/constant,pylibkriging,q2,0.980655 +kernels-trends-interpolation_Kriging_python,python,branin/gauss/constant,pylibkriging,nlpd,4.3152 +kernels-trends-interpolation_Kriging_python,python,branin/gauss/linear,pylibkriging,fit_s,0.00129008 +kernels-trends-interpolation_Kriging_python,python,branin/gauss/linear,pylibkriging,rmse,7.25733 +kernels-trends-interpolation_Kriging_python,python,branin/gauss/linear,pylibkriging,q2,0.980685 +kernels-trends-interpolation_Kriging_python,python,branin/gauss/linear,pylibkriging,nlpd,4.30813 +kernels-trends-interpolation_Kriging_python,python,branin/gauss/interactive,pylibkriging,fit_s,0.00280094 +kernels-trends-interpolation_Kriging_python,python,branin/gauss/interactive,pylibkriging,rmse,7.2576 +kernels-trends-interpolation_Kriging_python,python,branin/gauss/interactive,pylibkriging,q2,0.980683 +kernels-trends-interpolation_Kriging_python,python,branin/gauss/interactive,pylibkriging,nlpd,4.30176 +kernels-trends-interpolation_Kriging_python,python,branin/gauss/quadratic,pylibkriging,fit_s,0.00112319 +kernels-trends-interpolation_Kriging_python,python,branin/gauss/quadratic,pylibkriging,rmse,7.27808 +kernels-trends-interpolation_Kriging_python,python,branin/gauss/quadratic,pylibkriging,q2,0.980574 +kernels-trends-interpolation_Kriging_python,python,branin/gauss/quadratic,pylibkriging,nlpd,4.29399 +kernels-trends-interpolation_Kriging_python,python,branin/exp/constant,pylibkriging,fit_s,0.00222707 +kernels-trends-interpolation_Kriging_python,python,branin/exp/constant,pylibkriging,rmse,14.5887 +kernels-trends-interpolation_Kriging_python,python,branin/exp/constant,pylibkriging,q2,0.921947 +kernels-trends-interpolation_Kriging_python,python,branin/exp/constant,pylibkriging,nlpd,4.09457 +kernels-trends-interpolation_Kriging_python,python,branin/exp/linear,pylibkriging,fit_s,0.00213099 +kernels-trends-interpolation_Kriging_python,python,branin/exp/linear,pylibkriging,rmse,14.9622 +kernels-trends-interpolation_Kriging_python,python,branin/exp/linear,pylibkriging,q2,0.9179 +kernels-trends-interpolation_Kriging_python,python,branin/exp/linear,pylibkriging,nlpd,4.0909 +kernels-trends-interpolation_Kriging_python,python,branin/exp/interactive,pylibkriging,fit_s,0.00196409 +kernels-trends-interpolation_Kriging_python,python,branin/exp/interactive,pylibkriging,rmse,11.6963 +kernels-trends-interpolation_Kriging_python,python,branin/exp/interactive,pylibkriging,q2,0.94983 +kernels-trends-interpolation_Kriging_python,python,branin/exp/interactive,pylibkriging,nlpd,3.79671 +kernels-trends-interpolation_Kriging_python,python,branin/exp/quadratic,pylibkriging,fit_s,0.00227737 +kernels-trends-interpolation_Kriging_python,python,branin/exp/quadratic,pylibkriging,rmse,9.33909 +kernels-trends-interpolation_Kriging_python,python,branin/exp/quadratic,pylibkriging,q2,0.968014 +kernels-trends-interpolation_Kriging_python,python,branin/exp/quadratic,pylibkriging,nlpd,3.62362 +kernels-trends-interpolation_Kriging_python,python,branin/matern3_2/constant,pylibkriging,fit_s,0.00244212 +kernels-trends-interpolation_Kriging_python,python,branin/matern3_2/constant,pylibkriging,rmse,3.24796 +kernels-trends-interpolation_Kriging_python,python,branin/matern3_2/constant,pylibkriging,q2,0.996131 +kernels-trends-interpolation_Kriging_python,python,branin/matern3_2/constant,pylibkriging,nlpd,2.30444 +kernels-trends-interpolation_Kriging_python,python,branin/matern3_2/linear,pylibkriging,fit_s,0.00244641 +kernels-trends-interpolation_Kriging_python,python,branin/matern3_2/linear,pylibkriging,rmse,3.14402 +kernels-trends-interpolation_Kriging_python,python,branin/matern3_2/linear,pylibkriging,q2,0.996375 +kernels-trends-interpolation_Kriging_python,python,branin/matern3_2/linear,pylibkriging,nlpd,2.28067 +kernels-trends-interpolation_Kriging_python,python,branin/matern3_2/interactive,pylibkriging,fit_s,0.00703263 +kernels-trends-interpolation_Kriging_python,python,branin/matern3_2/interactive,pylibkriging,rmse,3.19226 +kernels-trends-interpolation_Kriging_python,python,branin/matern3_2/interactive,pylibkriging,q2,0.996263 +kernels-trends-interpolation_Kriging_python,python,branin/matern3_2/interactive,pylibkriging,nlpd,2.20359 +kernels-trends-interpolation_Kriging_python,python,branin/matern3_2/quadratic,pylibkriging,fit_s,0.00404716 +kernels-trends-interpolation_Kriging_python,python,branin/matern3_2/quadratic,pylibkriging,rmse,3.01232 +kernels-trends-interpolation_Kriging_python,python,branin/matern3_2/quadratic,pylibkriging,q2,0.996672 +kernels-trends-interpolation_Kriging_python,python,branin/matern3_2/quadratic,pylibkriging,nlpd,2.11627 +kernels-trends-interpolation_Kriging_python,python,branin/matern5_2/constant,pylibkriging,fit_s,0.0029068 +kernels-trends-interpolation_Kriging_python,python,branin/matern5_2/constant,pylibkriging,rmse,0.52012 +kernels-trends-interpolation_Kriging_python,python,branin/matern5_2/constant,pylibkriging,q2,0.999901 +kernels-trends-interpolation_Kriging_python,python,branin/matern5_2/constant,pylibkriging,nlpd,0.283399 +kernels-trends-interpolation_Kriging_python,python,branin/matern5_2/constant,pylibkriging BFGS10,rmse,0.52012 +kernels-trends-interpolation_Kriging_python,python,branin/matern5_2/constant,pylibkriging BFGS10,q2,0.999901 +kernels-trends-interpolation_Kriging_python,python,branin/matern5_2/constant,pylibkriging BFGS10,nlpd,0.283395 +kernels-trends-interpolation_Kriging_python,python,branin/matern5_2/linear,pylibkriging,fit_s,0.00212407 +kernels-trends-interpolation_Kriging_python,python,branin/matern5_2/linear,pylibkriging,rmse,0.460868 +kernels-trends-interpolation_Kriging_python,python,branin/matern5_2/linear,pylibkriging,q2,0.999922 +kernels-trends-interpolation_Kriging_python,python,branin/matern5_2/linear,pylibkriging,nlpd,0.209741 +kernels-trends-interpolation_Kriging_python,python,branin/matern5_2/interactive,pylibkriging,fit_s,0.0018971 +kernels-trends-interpolation_Kriging_python,python,branin/matern5_2/interactive,pylibkriging,rmse,0.498484 +kernels-trends-interpolation_Kriging_python,python,branin/matern5_2/interactive,pylibkriging,q2,0.999909 +kernels-trends-interpolation_Kriging_python,python,branin/matern5_2/interactive,pylibkriging,nlpd,0.239079 +kernels-trends-interpolation_Kriging_python,python,branin/matern5_2/quadratic,pylibkriging,fit_s,0.00570393 +kernels-trends-interpolation_Kriging_python,python,branin/matern5_2/quadratic,pylibkriging,rmse,0.67472 +kernels-trends-interpolation_Kriging_python,python,branin/matern5_2/quadratic,pylibkriging,q2,0.999833 +kernels-trends-interpolation_Kriging_python,python,branin/matern5_2/quadratic,pylibkriging,nlpd,0.419266 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/gauss/constant,pylibkriging,fit_s,0.00197244 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/gauss/constant,pylibkriging,rmse,18.9501 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/gauss/constant,pylibkriging,q2,0.536354 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/gauss/constant,pylibkriging,nlpd,84.151 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/gauss/linear,pylibkriging,fit_s,0.00170588 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/gauss/linear,pylibkriging,rmse,18.9579 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/gauss/linear,pylibkriging,q2,0.535976 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/gauss/linear,pylibkriging,nlpd,44.8696 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/gauss/interactive,pylibkriging,fit_s,0.0020082 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/gauss/interactive,pylibkriging,rmse,18.9819 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/gauss/interactive,pylibkriging,q2,0.5348 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/gauss/interactive,pylibkriging,nlpd,35.2765 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/gauss/quadratic,pylibkriging,fit_s,0.00248289 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/gauss/quadratic,pylibkriging,rmse,10.9058 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/gauss/quadratic,pylibkriging,q2,0.846441 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/gauss/quadratic,pylibkriging,nlpd,4.92017 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/exp/constant,pylibkriging,fit_s,0.00100708 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/exp/constant,pylibkriging,rmse,16.9653 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/exp/constant,pylibkriging,q2,0.628393 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/exp/constant,pylibkriging,nlpd,4.48595 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/exp/linear,pylibkriging,fit_s,0.00118208 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/exp/linear,pylibkriging,rmse,17.1781 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/exp/linear,pylibkriging,q2,0.619012 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/exp/linear,pylibkriging,nlpd,4.47061 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/exp/interactive,pylibkriging,fit_s,0.00123763 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/exp/interactive,pylibkriging,rmse,17.1856 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/exp/interactive,pylibkriging,q2,0.618679 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/exp/interactive,pylibkriging,nlpd,4.43133 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/exp/quadratic,pylibkriging,fit_s,0.0013876 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/exp/quadratic,pylibkriging,rmse,7.99484 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/exp/quadratic,pylibkriging,q2,0.917476 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/exp/quadratic,pylibkriging,nlpd,3.35612 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern3_2/constant,pylibkriging,fit_s,0.00175476 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern3_2/constant,pylibkriging,rmse,18.4958 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern3_2/constant,pylibkriging,q2,0.558321 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern3_2/constant,pylibkriging,nlpd,18.8349 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern3_2/linear,pylibkriging,fit_s,0.00206542 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern3_2/linear,pylibkriging,rmse,18.5562 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern3_2/linear,pylibkriging,q2,0.555433 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern3_2/linear,pylibkriging,nlpd,14.9494 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern3_2/interactive,pylibkriging,fit_s,0.00178266 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern3_2/interactive,pylibkriging,rmse,18.561 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern3_2/interactive,pylibkriging,q2,0.555202 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern3_2/interactive,pylibkriging,nlpd,12.9923 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern3_2/quadratic,pylibkriging,fit_s,0.00338221 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern3_2/quadratic,pylibkriging,rmse,8.91645 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern3_2/quadratic,pylibkriging,q2,0.897353 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern3_2/quadratic,pylibkriging,nlpd,3.94939 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern5_2/constant,pylibkriging,fit_s,0.00228882 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern5_2/constant,pylibkriging,rmse,18.3236 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern5_2/constant,pylibkriging,q2,0.566507 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern5_2/constant,pylibkriging,nlpd,18.4322 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern5_2/constant,pylibkriging BFGS10,rmse,1.36208 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern5_2/constant,pylibkriging BFGS10,q2,0.997605 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern5_2/constant,pylibkriging BFGS10,nlpd,1.03215 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern5_2/linear,pylibkriging,fit_s,0.00242186 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern5_2/linear,pylibkriging,rmse,18.7127 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern5_2/linear,pylibkriging,q2,0.547901 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern5_2/linear,pylibkriging,nlpd,24.5771 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern5_2/interactive,pylibkriging,fit_s,0.00234485 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern5_2/interactive,pylibkriging,rmse,18.7194 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern5_2/interactive,pylibkriging,q2,0.547577 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern5_2/interactive,pylibkriging,nlpd,20.4255 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern5_2/quadratic,pylibkriging,fit_s,0.00251341 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern5_2/quadratic,pylibkriging,rmse,10.2878 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern5_2/quadratic,pylibkriging,q2,0.863351 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern5_2/quadratic,pylibkriging,nlpd,4.46344 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/gauss/constant,pylibkriging,fit_s,0.00588727 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/gauss/constant,pylibkriging,rmse,0.295934 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/gauss/constant,pylibkriging,q2,0.920537 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/gauss/constant,pylibkriging,nlpd,1.58666 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/gauss/linear,pylibkriging,fit_s,0.00581551 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/gauss/linear,pylibkriging,rmse,0.296355 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/gauss/linear,pylibkriging,q2,0.92031 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/gauss/linear,pylibkriging,nlpd,1.2962 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/gauss/interactive,pylibkriging,fit_s,0.00644922 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/gauss/interactive,pylibkriging,rmse,0.285136 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/gauss/interactive,pylibkriging,q2,0.926229 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/gauss/interactive,pylibkriging,nlpd,1.04122 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/gauss/quadratic,pylibkriging,fit_s,0.00578475 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/gauss/quadratic,pylibkriging,rmse,0.237308 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/gauss/quadratic,pylibkriging,q2,0.948902 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/gauss/quadratic,pylibkriging,nlpd,0.146484 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/exp/constant,pylibkriging,fit_s,0.00258231 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/exp/constant,pylibkriging,rmse,0.334876 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/exp/constant,pylibkriging,q2,0.898247 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/exp/constant,pylibkriging,nlpd,0.339235 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/exp/linear,pylibkriging,fit_s,0.00211 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/exp/linear,pylibkriging,rmse,0.333447 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/exp/linear,pylibkriging,q2,0.899114 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/exp/linear,pylibkriging,nlpd,0.336515 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/exp/interactive,pylibkriging,fit_s,0.0025239 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/exp/interactive,pylibkriging,rmse,0.336819 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/exp/interactive,pylibkriging,q2,0.897063 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/exp/interactive,pylibkriging,nlpd,0.329566 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/exp/quadratic,pylibkriging,fit_s,0.00309515 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/exp/quadratic,pylibkriging,rmse,0.327816 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/exp/quadratic,pylibkriging,q2,0.902492 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/exp/quadratic,pylibkriging,nlpd,0.322056 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern3_2/constant,pylibkriging,fit_s,0.00425816 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern3_2/constant,pylibkriging,rmse,0.684362 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern3_2/constant,pylibkriging,q2,0.575038 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern3_2/constant,pylibkriging,nlpd,0.972803 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern3_2/linear,pylibkriging,fit_s,0.00413442 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern3_2/linear,pylibkriging,rmse,0.650201 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern3_2/linear,pylibkriging,q2,0.616404 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern3_2/linear,pylibkriging,nlpd,0.921455 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern3_2/interactive,pylibkriging,fit_s,0.00704145 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern3_2/interactive,pylibkriging,rmse,0.274326 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern3_2/interactive,pylibkriging,q2,0.931717 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern3_2/interactive,pylibkriging,nlpd,0.0549679 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern3_2/quadratic,pylibkriging,fit_s,0.00996184 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern3_2/quadratic,pylibkriging,rmse,0.263745 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern3_2/quadratic,pylibkriging,q2,0.936883 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern3_2/quadratic,pylibkriging,nlpd,0.0446267 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern5_2/constant,pylibkriging,fit_s,0.00410438 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern5_2/constant,pylibkriging,rmse,0.674391 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern5_2/constant,pylibkriging,q2,0.58733 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern5_2/constant,pylibkriging,nlpd,0.944765 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern5_2/constant,pylibkriging BFGS10,rmse,0.268216 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern5_2/constant,pylibkriging BFGS10,q2,0.934725 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern5_2/constant,pylibkriging BFGS10,nlpd,0.169585 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern5_2/linear,pylibkriging,fit_s,0.00728607 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern5_2/linear,pylibkriging,rmse,0.268808 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern5_2/linear,pylibkriging,q2,0.934436 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern5_2/linear,pylibkriging,nlpd,0.142382 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern5_2/interactive,pylibkriging,fit_s,0.00524259 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern5_2/interactive,pylibkriging,rmse,0.271088 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern5_2/interactive,pylibkriging,q2,0.933319 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern5_2/interactive,pylibkriging,nlpd,0.128788 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern5_2/quadratic,pylibkriging,fit_s,0.011641 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern5_2/quadratic,pylibkriging,rmse,0.253093 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern5_2/quadratic,pylibkriging,q2,0.941878 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern5_2/quadratic,pylibkriging,nlpd,0.025469 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/gauss/constant,pylibkriging,fit_s,0.00113583 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/gauss/constant,pylibkriging,rmse,0.026127 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/gauss/constant,pylibkriging,q2,0.999737 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/gauss/constant,pylibkriging,nlpd,-2.69817 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/gauss/linear,pylibkriging,fit_s,0.00152946 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/gauss/linear,pylibkriging,rmse,1.06243 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/gauss/linear,pylibkriging,q2,0.564691 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/gauss/linear,pylibkriging,nlpd,2.7394 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/gauss/interactive,pylibkriging,fit_s,0.00153446 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/gauss/interactive,pylibkriging,rmse,1.06504 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/gauss/interactive,pylibkriging,q2,0.562551 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/gauss/interactive,pylibkriging,nlpd,2.33575 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/gauss/quadratic,pylibkriging,fit_s,0.00196862 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/gauss/quadratic,pylibkriging,rmse,0.281816 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/gauss/quadratic,pylibkriging,q2,0.969371 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/gauss/quadratic,pylibkriging,nlpd,0.71682 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/exp/constant,pylibkriging,fit_s,0.000828028 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/exp/constant,pylibkriging,rmse,0.249023 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/exp/constant,pylibkriging,q2,0.976085 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/exp/constant,pylibkriging,nlpd,0.394159 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/exp/linear,pylibkriging,fit_s,0.000962973 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/exp/linear,pylibkriging,rmse,0.245558 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/exp/linear,pylibkriging,q2,0.976746 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/exp/linear,pylibkriging,nlpd,0.224593 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/exp/interactive,pylibkriging,fit_s,0.00143671 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/exp/interactive,pylibkriging,rmse,0.239032 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/exp/interactive,pylibkriging,q2,0.977965 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/exp/interactive,pylibkriging,nlpd,0.226133 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/exp/quadratic,pylibkriging,fit_s,0.00274634 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/exp/quadratic,pylibkriging,rmse,0.237713 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/exp/quadratic,pylibkriging,q2,0.978208 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/exp/quadratic,pylibkriging,nlpd,-0.139582 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern3_2/constant,pylibkriging,fit_s,0.0011251 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern3_2/constant,pylibkriging,rmse,0.0600911 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern3_2/constant,pylibkriging,q2,0.998607 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern3_2/constant,pylibkriging,nlpd,-1.34658 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern3_2/linear,pylibkriging,fit_s,0.000877857 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern3_2/linear,pylibkriging,rmse,0.0584376 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern3_2/linear,pylibkriging,q2,0.998683 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern3_2/linear,pylibkriging,nlpd,-1.34912 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern3_2/interactive,pylibkriging,fit_s,0.00140738 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern3_2/interactive,pylibkriging,rmse,0.0576096 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern3_2/interactive,pylibkriging,q2,0.99872 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern3_2/interactive,pylibkriging,nlpd,-1.35376 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern3_2/quadratic,pylibkriging,fit_s,0.00383472 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern3_2/quadratic,pylibkriging,rmse,0.215336 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern3_2/quadratic,pylibkriging,q2,0.982117 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern3_2/quadratic,pylibkriging,nlpd,-0.350837 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern5_2/constant,pylibkriging,fit_s,0.000718832 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern5_2/constant,pylibkriging,rmse,0.0246348 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern5_2/constant,pylibkriging,q2,0.999766 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern5_2/constant,pylibkriging,nlpd,-2.59598 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern5_2/constant,pylibkriging BFGS10,rmse,0.0246338 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern5_2/constant,pylibkriging BFGS10,q2,0.999766 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern5_2/constant,pylibkriging BFGS10,nlpd,-2.59598 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern5_2/linear,pylibkriging,fit_s,0.0013113 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern5_2/linear,pylibkriging,rmse,0.0232483 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern5_2/linear,pylibkriging,q2,0.999792 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern5_2/linear,pylibkriging,nlpd,-2.58843 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern5_2/interactive,pylibkriging,fit_s,0.00150442 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern5_2/interactive,pylibkriging,rmse,0.0230248 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern5_2/interactive,pylibkriging,q2,0.999796 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern5_2/interactive,pylibkriging,nlpd,-2.58839 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern5_2/quadratic,pylibkriging,fit_s,0.00300908 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern5_2/quadratic,pylibkriging,rmse,0.209802 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern5_2/quadratic,pylibkriging,q2,0.983025 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern5_2/quadratic,pylibkriging,nlpd,-0.403998 +kernels-trends-interpolation_Kriging_python,python,ishigami/gauss/constant,pylibkriging,fit_s,0.0101857 +kernels-trends-interpolation_Kriging_python,python,ishigami/gauss/constant,pylibkriging,rmse,2.07486 +kernels-trends-interpolation_Kriging_python,python,ishigami/gauss/constant,pylibkriging,q2,0.668191 +kernels-trends-interpolation_Kriging_python,python,ishigami/gauss/constant,pylibkriging,nlpd,2.80326 +kernels-trends-interpolation_Kriging_python,python,ishigami/gauss/linear,pylibkriging,fit_s,0.0193923 +kernels-trends-interpolation_Kriging_python,python,ishigami/gauss/linear,pylibkriging,rmse,2.8459 +kernels-trends-interpolation_Kriging_python,python,ishigami/gauss/linear,pylibkriging,q2,0.375762 +kernels-trends-interpolation_Kriging_python,python,ishigami/gauss/linear,pylibkriging,nlpd,3.41919 +kernels-trends-interpolation_Kriging_python,python,ishigami/gauss/interactive,pylibkriging,fit_s,0.0174267 +kernels-trends-interpolation_Kriging_python,python,ishigami/gauss/interactive,pylibkriging,rmse,2.85366 +kernels-trends-interpolation_Kriging_python,python,ishigami/gauss/interactive,pylibkriging,q2,0.372353 +kernels-trends-interpolation_Kriging_python,python,ishigami/gauss/interactive,pylibkriging,nlpd,3.28643 +kernels-trends-interpolation_Kriging_python,python,ishigami/gauss/quadratic,pylibkriging,fit_s,0.00865173 +kernels-trends-interpolation_Kriging_python,python,ishigami/gauss/quadratic,pylibkriging,rmse,2.08981 +kernels-trends-interpolation_Kriging_python,python,ishigami/gauss/quadratic,pylibkriging,q2,0.663392 +kernels-trends-interpolation_Kriging_python,python,ishigami/gauss/quadratic,pylibkriging,nlpd,2.80097 +kernels-trends-interpolation_Kriging_python,python,ishigami/exp/constant,pylibkriging,fit_s,0.00899458 +kernels-trends-interpolation_Kriging_python,python,ishigami/exp/constant,pylibkriging,rmse,1.41632 +kernels-trends-interpolation_Kriging_python,python,ishigami/exp/constant,pylibkriging,q2,0.845391 +kernels-trends-interpolation_Kriging_python,python,ishigami/exp/constant,pylibkriging,nlpd,1.75239 +kernels-trends-interpolation_Kriging_python,python,ishigami/exp/linear,pylibkriging,fit_s,0.0115979 +kernels-trends-interpolation_Kriging_python,python,ishigami/exp/linear,pylibkriging,rmse,1.4095 +kernels-trends-interpolation_Kriging_python,python,ishigami/exp/linear,pylibkriging,q2,0.846877 +kernels-trends-interpolation_Kriging_python,python,ishigami/exp/linear,pylibkriging,nlpd,1.74779 +kernels-trends-interpolation_Kriging_python,python,ishigami/exp/interactive,pylibkriging,fit_s,0.00745058 +kernels-trends-interpolation_Kriging_python,python,ishigami/exp/interactive,pylibkriging,rmse,1.41511 +kernels-trends-interpolation_Kriging_python,python,ishigami/exp/interactive,pylibkriging,q2,0.845657 +kernels-trends-interpolation_Kriging_python,python,ishigami/exp/interactive,pylibkriging,nlpd,1.75024 +kernels-trends-interpolation_Kriging_python,python,ishigami/exp/quadratic,pylibkriging,fit_s,0.00843048 +kernels-trends-interpolation_Kriging_python,python,ishigami/exp/quadratic,pylibkriging,rmse,1.40329 +kernels-trends-interpolation_Kriging_python,python,ishigami/exp/quadratic,pylibkriging,q2,0.848223 +kernels-trends-interpolation_Kriging_python,python,ishigami/exp/quadratic,pylibkriging,nlpd,1.73254 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern3_2/constant,pylibkriging,fit_s,0.0480943 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern3_2/constant,pylibkriging,rmse,1.32971 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern3_2/constant,pylibkriging,q2,0.863723 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern3_2/constant,pylibkriging,nlpd,1.50265 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern3_2/linear,pylibkriging,fit_s,0.047653 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern3_2/linear,pylibkriging,rmse,1.30244 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern3_2/linear,pylibkriging,q2,0.869255 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern3_2/linear,pylibkriging,nlpd,1.47906 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern3_2/interactive,pylibkriging,fit_s,0.060349 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern3_2/interactive,pylibkriging,rmse,1.30591 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern3_2/interactive,pylibkriging,q2,0.868558 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern3_2/interactive,pylibkriging,nlpd,1.47384 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern3_2/quadratic,pylibkriging,fit_s,0.0471144 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern3_2/quadratic,pylibkriging,rmse,1.32197 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern3_2/quadratic,pylibkriging,q2,0.865305 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern3_2/quadratic,pylibkriging,nlpd,1.46827 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern5_2/constant,pylibkriging,fit_s,0.0145204 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern5_2/constant,pylibkriging,rmse,3.00566 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern5_2/constant,pylibkriging,q2,0.303709 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern5_2/constant,pylibkriging,nlpd,3.42085 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern5_2/constant,pylibkriging BFGS10,rmse,1.33903 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern5_2/constant,pylibkriging BFGS10,q2,0.861806 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern5_2/constant,pylibkriging BFGS10,nlpd,1.66132 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern5_2/linear,pylibkriging,fit_s,0.0518982 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern5_2/linear,pylibkriging,rmse,1.3434 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern5_2/linear,pylibkriging,q2,0.860901 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern5_2/linear,pylibkriging,nlpd,1.65103 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern5_2/interactive,pylibkriging,fit_s,0.0539804 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern5_2/interactive,pylibkriging,rmse,1.34479 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern5_2/interactive,pylibkriging,q2,0.860615 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern5_2/interactive,pylibkriging,nlpd,1.64153 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern5_2/quadratic,pylibkriging,fit_s,0.052948 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern5_2/quadratic,pylibkriging,rmse,1.37082 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern5_2/quadratic,pylibkriging,q2,0.855166 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern5_2/quadratic,pylibkriging,nlpd,1.62228 +kernels-trends-interpolation_Kriging_python,python,hartmann3/gauss/constant,pylibkriging,fit_s,0.00805306 +kernels-trends-interpolation_Kriging_python,python,hartmann3/gauss/constant,pylibkriging,rmse,0.156226 +kernels-trends-interpolation_Kriging_python,python,hartmann3/gauss/constant,pylibkriging,q2,0.971116 +kernels-trends-interpolation_Kriging_python,python,hartmann3/gauss/constant,pylibkriging,nlpd,-0.555623 +kernels-trends-interpolation_Kriging_python,python,hartmann3/gauss/linear,pylibkriging,fit_s,0.0129044 +kernels-trends-interpolation_Kriging_python,python,hartmann3/gauss/linear,pylibkriging,rmse,0.153297 +kernels-trends-interpolation_Kriging_python,python,hartmann3/gauss/linear,pylibkriging,q2,0.972189 +kernels-trends-interpolation_Kriging_python,python,hartmann3/gauss/linear,pylibkriging,nlpd,-0.568404 +kernels-trends-interpolation_Kriging_python,python,hartmann3/gauss/interactive,pylibkriging,fit_s,0.00773382 +kernels-trends-interpolation_Kriging_python,python,hartmann3/gauss/interactive,pylibkriging,rmse,0.712948 +kernels-trends-interpolation_Kriging_python,python,hartmann3/gauss/interactive,pylibkriging,q2,0.398455 +kernels-trends-interpolation_Kriging_python,python,hartmann3/gauss/interactive,pylibkriging,nlpd,1.74554 +kernels-trends-interpolation_Kriging_python,python,hartmann3/gauss/quadratic,pylibkriging,fit_s,0.00593877 +kernels-trends-interpolation_Kriging_python,python,hartmann3/gauss/quadratic,pylibkriging,rmse,0.73238 +kernels-trends-interpolation_Kriging_python,python,hartmann3/gauss/quadratic,pylibkriging,q2,0.365217 +kernels-trends-interpolation_Kriging_python,python,hartmann3/gauss/quadratic,pylibkriging,nlpd,2.14587 +kernels-trends-interpolation_Kriging_python,python,hartmann3/exp/constant,pylibkriging,fit_s,0.00374722 +kernels-trends-interpolation_Kriging_python,python,hartmann3/exp/constant,pylibkriging,rmse,0.347348 +kernels-trends-interpolation_Kriging_python,python,hartmann3/exp/constant,pylibkriging,q2,0.857215 +kernels-trends-interpolation_Kriging_python,python,hartmann3/exp/constant,pylibkriging,nlpd,0.352683 +kernels-trends-interpolation_Kriging_python,python,hartmann3/exp/linear,pylibkriging,fit_s,0.00216174 +kernels-trends-interpolation_Kriging_python,python,hartmann3/exp/linear,pylibkriging,rmse,0.347273 +kernels-trends-interpolation_Kriging_python,python,hartmann3/exp/linear,pylibkriging,q2,0.857276 +kernels-trends-interpolation_Kriging_python,python,hartmann3/exp/linear,pylibkriging,nlpd,0.343631 +kernels-trends-interpolation_Kriging_python,python,hartmann3/exp/interactive,pylibkriging,fit_s,0.0024302 +kernels-trends-interpolation_Kriging_python,python,hartmann3/exp/interactive,pylibkriging,rmse,0.378349 +kernels-trends-interpolation_Kriging_python,python,hartmann3/exp/interactive,pylibkriging,q2,0.83059 +kernels-trends-interpolation_Kriging_python,python,hartmann3/exp/interactive,pylibkriging,nlpd,0.385842 +kernels-trends-interpolation_Kriging_python,python,hartmann3/exp/quadratic,pylibkriging,fit_s,0.0039556 +kernels-trends-interpolation_Kriging_python,python,hartmann3/exp/quadratic,pylibkriging,rmse,0.355119 +kernels-trends-interpolation_Kriging_python,python,hartmann3/exp/quadratic,pylibkriging,q2,0.850755 +kernels-trends-interpolation_Kriging_python,python,hartmann3/exp/quadratic,pylibkriging,nlpd,0.334371 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern3_2/constant,pylibkriging,fit_s,0.0110524 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern3_2/constant,pylibkriging,rmse,0.227521 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern3_2/constant,pylibkriging,q2,0.938737 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern3_2/constant,pylibkriging,nlpd,-0.181207 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern3_2/linear,pylibkriging,fit_s,0.00763154 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern3_2/linear,pylibkriging,rmse,0.23038 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern3_2/linear,pylibkriging,q2,0.937188 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern3_2/linear,pylibkriging,nlpd,-0.162203 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern3_2/interactive,pylibkriging,fit_s,0.00412679 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern3_2/interactive,pylibkriging,rmse,0.315752 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern3_2/interactive,pylibkriging,q2,0.88201 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern3_2/interactive,pylibkriging,nlpd,0.898253 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern3_2/quadratic,pylibkriging,fit_s,0.00620675 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern3_2/quadratic,pylibkriging,rmse,0.291393 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern3_2/quadratic,pylibkriging,q2,0.899513 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern3_2/quadratic,pylibkriging,nlpd,0.53743 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern5_2/constant,pylibkriging,fit_s,0.00997186 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern5_2/constant,pylibkriging,rmse,0.190287 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern5_2/constant,pylibkriging,q2,0.957148 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern5_2/constant,pylibkriging,nlpd,-0.403054 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern5_2/constant,pylibkriging BFGS10,rmse,0.190286 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern5_2/constant,pylibkriging BFGS10,q2,0.957148 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern5_2/constant,pylibkriging BFGS10,nlpd,-0.40306 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern5_2/linear,pylibkriging,fit_s,0.0127335 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern5_2/linear,pylibkriging,rmse,0.189633 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern5_2/linear,pylibkriging,q2,0.957442 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern5_2/linear,pylibkriging,nlpd,-0.410075 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern5_2/interactive,pylibkriging,fit_s,0.0101018 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern5_2/interactive,pylibkriging,rmse,0.307114 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern5_2/interactive,pylibkriging,q2,0.888377 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern5_2/interactive,pylibkriging,nlpd,1.50063 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern5_2/quadratic,pylibkriging,fit_s,0.0101326 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern5_2/quadratic,pylibkriging,rmse,0.285355 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern5_2/quadratic,pylibkriging,q2,0.903634 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern5_2/quadratic,pylibkriging,nlpd,0.898128 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,branin/LL,pylibkriging,rmse,0.833715 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,branin/LL,pylibkriging,q2,0.999729 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,branin/LL,pylibkriging,nlpd,0.697034 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,branin/LOO,pylibkriging,rmse,0.550995 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,branin/LOO,pylibkriging,q2,0.999882 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,branin/LOO,pylibkriging,nlpd,0.471613 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,branin/LMP,pylibkriging,rmse,0.66967 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,branin/LMP,pylibkriging,q2,0.999825 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,branin/LMP,pylibkriging,nlpd,0.617527 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,hartmann3/LL,pylibkriging,rmse,0.157276 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,hartmann3/LL,pylibkriging,q2,0.970945 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,hartmann3/LL,pylibkriging,nlpd,-0.497548 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,hartmann3/LOO,pylibkriging,rmse,0.144982 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,hartmann3/LOO,pylibkriging,q2,0.97531 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,hartmann3/LOO,pylibkriging,nlpd,-0.584885 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,hartmann3/LMP,pylibkriging,rmse,0.150075 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,hartmann3/LMP,pylibkriging,q2,0.973544 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,hartmann3/LMP,pylibkriging,nlpd,-0.528379 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,ishigami/LL,pylibkriging,rmse,3.41501 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,ishigami/LL,pylibkriging,q2,0.203357 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,ishigami/LL,pylibkriging,nlpd,4.86258 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,ishigami/LOO,pylibkriging,rmse,2.45853 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,ishigami/LOO,pylibkriging,q2,0.587112 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,ishigami/LOO,pylibkriging,nlpd,2.72077 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,ishigami/LMP,pylibkriging,rmse,3.47385 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,ishigami/LMP,pylibkriging,q2,0.175667 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,ishigami/LMP,pylibkriging,nlpd,5.27033 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,friedman/LL,pylibkriging,rmse,0.273164 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,friedman/LL,pylibkriging,q2,0.996855 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,friedman/LL,pylibkriging,nlpd,-0.118408 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,friedman/LOO,pylibkriging,rmse,0.233921 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,friedman/LOO,pylibkriging,q2,0.997694 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,friedman/LOO,pylibkriging,nlpd,-0.169369 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,friedman/LMP,pylibkriging,rmse,0.269913 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,friedman/LMP,pylibkriging,q2,0.996929 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,friedman/LMP,pylibkriging,nlpd,-0.122709 +noisy-data_Kriging-nugget-noise_python,python,forrester,nugget,rmse,0.198091 +noisy-data_Kriging-nugget-noise_python,python,forrester,nugget,q2,0.998087 +noisy-data_Kriging-nugget-noise_python,python,forrester,nugget,nlpd,0.286631 +noisy-data_Kriging-nugget-noise_python,python,forrester,noise-free fit,rmse,0.487916 +noisy-data_Kriging-nugget-noise_python,python,forrester,noise-free fit,q2,0.988392 +noisy-data_Kriging-nugget-noise_python,python,forrester,noise-free fit,nlpd,862.67 +noisy-data_Kriging-nugget-noise_python,python,friedman,nugget,rmse,0.467335 +noisy-data_Kriging-nugget-noise_python,python,friedman,nugget,q2,0.99089 +noisy-data_Kriging-nugget-noise_python,python,friedman,nugget,nlpd,1.07073 +noisy-data_Kriging-nugget-noise_python,python,friedman,noise-free fit,rmse,1.27157 +noisy-data_Kriging-nugget-noise_python,python,friedman,noise-free fit,q2,0.932558 +noisy-data_Kriging-nugget-noise_python,python,friedman,noise-free fit,nlpd,1.64497 +noisy-data_Kriging-nugget-noise_python,python,gramacy_lee_2012/hetero,known variances,rmse,0.168741 +noisy-data_Kriging-nugget-noise_python,python,gramacy_lee_2012/hetero,known variances,q2,0.982515 +noisy-data_Kriging-nugget-noise_python,python,gramacy_lee_2012/hetero,known variances,nlpd,-0.570574 +noisy-data_Kriging-nugget-noise_python,python,gramacy_lee_2012/hetero,nugget,rmse,0.170945 +noisy-data_Kriging-nugget-noise_python,python,gramacy_lee_2012/hetero,nugget,q2,0.982055 +noisy-data_Kriging-nugget-noise_python,python,gramacy_lee_2012/hetero,nugget,nlpd,-0.221568 +warpings-categorical-ordinal_WarpKriging_python,python,gl2012,Kriging,rmse,0.167263 +warpings-categorical-ordinal_WarpKriging_python,python,gl2012,Kriging,q2,0.981176 +warpings-categorical-ordinal_WarpKriging_python,python,gl2012,Kriging,nlpd,-1.08917 +warpings-categorical-ordinal_WarpKriging_python,python,gl2012,affine,rmse,0.167263 +warpings-categorical-ordinal_WarpKriging_python,python,gl2012,affine,q2,0.981176 +warpings-categorical-ordinal_WarpKriging_python,python,gl2012,affine,nlpd,-1.08917 +warpings-categorical-ordinal_WarpKriging_python,python,gl2012,boxcox,rmse,0.16759 +warpings-categorical-ordinal_WarpKriging_python,python,gl2012,boxcox,q2,0.981103 +warpings-categorical-ordinal_WarpKriging_python,python,gl2012,boxcox,nlpd,282.422 +warpings-categorical-ordinal_WarpKriging_python,python,gl2012,kumaraswamy,rmse,0.168286 +warpings-categorical-ordinal_WarpKriging_python,python,gl2012,kumaraswamy,q2,0.980945 +warpings-categorical-ordinal_WarpKriging_python,python,gl2012,kumaraswamy,nlpd,2.04577e+06 +warpings-categorical-ordinal_WarpKriging_python,python,gl2012,knots(5),rmse,0.16791 +warpings-categorical-ordinal_WarpKriging_python,python,gl2012,knots(5),q2,0.98103 +warpings-categorical-ordinal_WarpKriging_python,python,gl2012,knots(5),nlpd,2.04376e+06 +warpings-categorical-ordinal_WarpKriging_python,python,gl2012,neural_mono(8),rmse,0.170824 +warpings-categorical-ordinal_WarpKriging_python,python,gl2012,neural_mono(8),q2,0.980366 +warpings-categorical-ordinal_WarpKriging_python,python,gl2012,neural_mono(8),nlpd,-0.265141 +warpings-categorical-ordinal_WarpKriging_python,python,gl2008,Kriging,rmse,0.0534649 +warpings-categorical-ordinal_WarpKriging_python,python,gl2008,Kriging,q2,0.585636 +warpings-categorical-ordinal_WarpKriging_python,python,gl2008,Kriging,nlpd,-1.74569 +warpings-categorical-ordinal_WarpKriging_python,python,gl2008,kumaraswamy,rmse,0.0535298 +warpings-categorical-ordinal_WarpKriging_python,python,gl2008,kumaraswamy,q2,0.58463 +warpings-categorical-ordinal_WarpKriging_python,python,gl2008,kumaraswamy,nlpd,-1.74552 +warpings-categorical-ordinal_WarpKriging_python,python,mixed,Kriging (level as number),rmse,16.7693 +warpings-categorical-ordinal_WarpKriging_python,python,mixed,Kriging (level as number),q2,0.838452 +warpings-categorical-ordinal_WarpKriging_python,python,mixed,Kriging (level as number),nlpd,3.81063 +warpings-categorical-ordinal_WarpKriging_python,python,mixed,"categorical(5,2)",rmse,16.1438 +warpings-categorical-ordinal_WarpKriging_python,python,mixed,"categorical(5,2)",q2,0.85028 +warpings-categorical-ordinal_WarpKriging_python,python,mixed,"categorical(5,2)",nlpd,3.3206 +warpings-categorical-ordinal_WarpKriging_python,python,ordinal,Kriging (rank as number),rmse,0.213845 +warpings-categorical-ordinal_WarpKriging_python,python,ordinal,Kriging (rank as number),q2,0.953177 +warpings-categorical-ordinal_WarpKriging_python,python,ordinal,Kriging (rank as number),nlpd,0.810969 +warpings-categorical-ordinal_WarpKriging_python,python,ordinal,ordinal(6),rmse,0.213043 +warpings-categorical-ordinal_WarpKriging_python,python,ordinal,ordinal(6),q2,0.953528 +warpings-categorical-ordinal_WarpKriging_python,python,ordinal,ordinal(6),nlpd,0.78888 +high-dimension-normalize-screening_Kriging_python,python,otl_circuit,unit cube,fit_s,0.00789523 +high-dimension-normalize-screening_Kriging_python,python,otl_circuit,unit cube,rmse,0.0150514 +high-dimension-normalize-screening_Kriging_python,python,otl_circuit,unit cube,q2,0.999836 +high-dimension-normalize-screening_Kriging_python,python,otl_circuit,unit cube,nlpd,-2.90452 +high-dimension-normalize-screening_Kriging_python,python,otl_circuit,native + normalize,rmse,0.0150514 +high-dimension-normalize-screening_Kriging_python,python,otl_circuit,native + normalize,q2,0.999836 +high-dimension-normalize-screening_Kriging_python,python,otl_circuit,native + normalize,nlpd,-2.90452 +high-dimension-normalize-screening_Kriging_python,python,piston,unit cube,fit_s,0.0197392 +high-dimension-normalize-screening_Kriging_python,python,piston,unit cube,rmse,0.0104258 +high-dimension-normalize-screening_Kriging_python,python,piston,unit cube,q2,0.994462 +high-dimension-normalize-screening_Kriging_python,python,piston,unit cube,nlpd,-3.49038 +high-dimension-normalize-screening_Kriging_python,python,piston,native + normalize,rmse,0.0104258 +high-dimension-normalize-screening_Kriging_python,python,piston,native + normalize,q2,0.994462 +high-dimension-normalize-screening_Kriging_python,python,piston,native + normalize,nlpd,-3.49038 +high-dimension-normalize-screening_Kriging_python,python,borehole,unit cube,fit_s,0.0140877 +high-dimension-normalize-screening_Kriging_python,python,borehole,unit cube,rmse,0.893341 +high-dimension-normalize-screening_Kriging_python,python,borehole,unit cube,q2,0.999599 +high-dimension-normalize-screening_Kriging_python,python,borehole,unit cube,nlpd,1.08433 +high-dimension-normalize-screening_Kriging_python,python,borehole,native + normalize,rmse,0.893341 +high-dimension-normalize-screening_Kriging_python,python,borehole,native + normalize,q2,0.999599 +high-dimension-normalize-screening_Kriging_python,python,borehole,native + normalize,nlpd,1.08433 +high-dimension-normalize-screening_Kriging_python,python,robot_arm,unit cube,fit_s,0.167482 +high-dimension-normalize-screening_Kriging_python,python,robot_arm,unit cube,rmse,0.358507 +high-dimension-normalize-screening_Kriging_python,python,robot_arm,unit cube,q2,0.571154 +high-dimension-normalize-screening_Kriging_python,python,robot_arm,unit cube,nlpd,0.355106 +high-dimension-normalize-screening_Kriging_python,python,robot_arm,native + normalize,rmse,0.358507 +high-dimension-normalize-screening_Kriging_python,python,robot_arm,native + normalize,q2,0.571154 +high-dimension-normalize-screening_Kriging_python,python,robot_arm,native + normalize,nlpd,0.355106 +high-dimension-normalize-screening_Kriging_python,python,dette_pepelyshev,unit cube,fit_s,0.0246282 +high-dimension-normalize-screening_Kriging_python,python,dette_pepelyshev,unit cube,rmse,0.424908 +high-dimension-normalize-screening_Kriging_python,python,dette_pepelyshev,unit cube,q2,0.997757 +high-dimension-normalize-screening_Kriging_python,python,dette_pepelyshev,unit cube,nlpd,0.510276 +high-dimension-normalize-screening_Kriging_python,python,dette_pepelyshev,native + normalize,rmse,0.424908 +high-dimension-normalize-screening_Kriging_python,python,dette_pepelyshev,native + normalize,q2,0.997757 +high-dimension-normalize-screening_Kriging_python,python,dette_pepelyshev,native + normalize,nlpd,0.510276 +high-dimension-normalize-screening_Kriging_python,python,wing_weight,unit cube,fit_s,0.0245481 +high-dimension-normalize-screening_Kriging_python,python,wing_weight,unit cube,rmse,0.809743 +high-dimension-normalize-screening_Kriging_python,python,wing_weight,unit cube,q2,0.999716 +high-dimension-normalize-screening_Kriging_python,python,wing_weight,unit cube,nlpd,1.07161 +high-dimension-normalize-screening_Kriging_python,python,wing_weight,native + normalize,rmse,0.809743 +high-dimension-normalize-screening_Kriging_python,python,wing_weight,native + normalize,q2,0.999716 +high-dimension-normalize-screening_Kriging_python,python,wing_weight,native + normalize,nlpd,1.07161 +high-dimension-normalize-screening_Kriging_python,python,welch,unit cube,fit_s,0.564758 +high-dimension-normalize-screening_Kriging_python,python,welch,unit cube,rmse,0.167729 +high-dimension-normalize-screening_Kriging_python,python,welch,unit cube,q2,0.993452 +high-dimension-normalize-screening_Kriging_python,python,welch,unit cube,nlpd,-0.429138 +high-dimension-normalize-screening_Kriging_python,python,welch,native + normalize,rmse,0.167692 +high-dimension-normalize-screening_Kriging_python,python,welch,native + normalize,q2,0.993454 +high-dimension-normalize-screening_Kriging_python,python,welch,native + normalize,nlpd,-0.429291 +deep-kernel-projected-branin_MLPKriging_python,python,projected_branin_10d,Kriging,rmse,4.42284 +deep-kernel-projected-branin_MLPKriging_python,python,projected_branin_10d,Kriging,q2,0.959122 +deep-kernel-projected-branin_MLPKriging_python,python,projected_branin_10d,Kriging,nlpd,2.89997 +deep-kernel-projected-branin_MLPKriging_python,python,projected_branin_10d,MLPKriging tanh BFGS,fit_s,0.130867 +deep-kernel-projected-branin_MLPKriging_python,python,projected_branin_10d,MLPKriging tanh BFGS,rmse,57.402 +deep-kernel-projected-branin_MLPKriging_python,python,projected_branin_10d,MLPKriging tanh BFGS,q2,-5.88553 +deep-kernel-projected-branin_MLPKriging_python,python,projected_branin_10d,MLPKriging tanh BFGS,nlpd,136.771 +deep-kernel-projected-branin_MLPKriging_python,python,projected_branin_10d,MLPKriging tanh BFGS10,fit_s,0.442791 +deep-kernel-projected-branin_MLPKriging_python,python,projected_branin_10d,MLPKriging tanh BFGS10,rmse,4.28606 +deep-kernel-projected-branin_MLPKriging_python,python,projected_branin_10d,MLPKriging tanh BFGS10,q2,0.961612 +deep-kernel-projected-branin_MLPKriging_python,python,projected_branin_10d,MLPKriging tanh BFGS10,nlpd,29.0854 +deep-kernel-projected-branin_MLPKriging_python,python,projected_branin_10d,MLPKriging selu BFGS,fit_s,0.139126 +deep-kernel-projected-branin_MLPKriging_python,python,projected_branin_10d,MLPKriging selu BFGS,rmse,58.9913 +deep-kernel-projected-branin_MLPKriging_python,python,projected_branin_10d,MLPKriging selu BFGS,q2,-6.2721 +deep-kernel-projected-branin_MLPKriging_python,python,projected_branin_10d,MLPKriging selu BFGS,nlpd,46.9788 +deep-kernel-projected-branin_MLPKriging_python,python,projected_branin_10d,MLPKriging selu BFGS10,fit_s,0.440179 +deep-kernel-projected-branin_MLPKriging_python,python,projected_branin_10d,MLPKriging selu BFGS10,rmse,4.39195 +deep-kernel-projected-branin_MLPKriging_python,python,projected_branin_10d,MLPKriging selu BFGS10,q2,0.959691 +deep-kernel-projected-branin_MLPKriging_python,python,projected_branin_10d,MLPKriging selu BFGS10,nlpd,10.6053 +large-n_Kriging-vecchia-nystrom_python,python,hartmann3/n=1500,exact,fit_s,11.1133 +aggregations-large-n_NestedKriging_python,python,hartmann3/n=1500,exact,fit_s,11.1133 +large-n_Kriging-vecchia-nystrom_python,python,hartmann3/n=1500,exact,rmse,0.000259233 +aggregations-large-n_NestedKriging_python,python,hartmann3/n=1500,exact,rmse,0.000259233 +large-n_Kriging-vecchia-nystrom_python,python,hartmann3/n=1500,exact,q2,1 +aggregations-large-n_NestedKriging_python,python,hartmann3/n=1500,exact,q2,1 +large-n_Kriging-vecchia-nystrom_python,python,hartmann3/n=1500,exact,nlpd,-6.92184 +aggregations-large-n_NestedKriging_python,python,hartmann3/n=1500,exact,nlpd,-6.92184 +large-n_Kriging-vecchia-nystrom_python,python,hartmann3/n=1500,vecchia(30),fit_s,5.43768 +large-n_Kriging-vecchia-nystrom_python,python,hartmann3/n=1500,vecchia(30),rmse,0.00353225 +large-n_Kriging-vecchia-nystrom_python,python,hartmann3/n=1500,vecchia(30),q2,0.999987 +large-n_Kriging-vecchia-nystrom_python,python,hartmann3/n=1500,vecchia(30),nlpd,-4.6266 +large-n_Kriging-vecchia-nystrom_python,python,hartmann3/n=5000,vecchia(30),fit_s,16.2077 +large-n_Kriging-vecchia-nystrom_python,python,hartmann3/n=5000,vecchia(30),rmse,0.000851738 +large-n_Kriging-vecchia-nystrom_python,python,hartmann3/n=5000,vecchia(30),q2,0.999999 +large-n_Kriging-vecchia-nystrom_python,python,hartmann3/n=5000,vecchia(30),nlpd,-5.95078 +large-n_Kriging-vecchia-nystrom_python,python,hartmann3/n=1500,nystrom(100),fit_s,2.04779 +large-n_Kriging-vecchia-nystrom_python,python,hartmann3/n=1500,nystrom(100),rmse,1.51127 +large-n_Kriging-vecchia-nystrom_python,python,hartmann3/n=1500,nystrom(100),q2,-1.39237 +large-n_Kriging-vecchia-nystrom_python,python,hartmann3/n=1500,nystrom(100),nlpd,1.14197e+24 +large-n_Kriging-vecchia-nystrom_python,python,hartmann3/n=5000,nystrom(100),fit_s,3.25622 +large-n_Kriging-vecchia-nystrom_python,python,hartmann3/n=5000,nystrom(100),rmse,9.95398 +large-n_Kriging-vecchia-nystrom_python,python,hartmann3/n=5000,nystrom(100),q2,-102.786 +large-n_Kriging-vecchia-nystrom_python,python,hartmann3/n=5000,nystrom(100),nlpd,4.95409e+25 +aggregations-large-n_NestedKriging_python,python,hartmann3/n=1500,nested NK (20 groups),fit_s,0.269932 +aggregations-large-n_NestedKriging_python,python,hartmann3/n=1500,nested NK (20 groups),rmse,0.0029908 +aggregations-large-n_NestedKriging_python,python,hartmann3/n=1500,nested NK (20 groups),q2,0.999991 +aggregations-large-n_NestedKriging_python,python,hartmann3/n=1500,nested NK (20 groups),nlpd,-4.85946 +aggregations-large-n_NestedKriging_python,python,hartmann3/n=5000,nested NK (20 groups),fit_s,2.87469 +aggregations-large-n_NestedKriging_python,python,hartmann3/n=5000,nested NK (20 groups),rmse,0.000222655 +aggregations-large-n_NestedKriging_python,python,hartmann3/n=5000,nested NK (20 groups),q2,1 +aggregations-large-n_NestedKriging_python,python,hartmann3/n=5000,nested NK (20 groups),nlpd,-7.44453 +aggregations-large-n_NestedKriging_python,python,hartmann3/n=1500,nested rBCM (20 groups),fit_s,0.25767 +aggregations-large-n_NestedKriging_python,python,hartmann3/n=1500,nested rBCM (20 groups),rmse,0.00304431 +aggregations-large-n_NestedKriging_python,python,hartmann3/n=1500,nested rBCM (20 groups),q2,0.99999 +aggregations-large-n_NestedKriging_python,python,hartmann3/n=1500,nested rBCM (20 groups),nlpd,-4.50201 +aggregations-large-n_NestedKriging_python,python,hartmann3/n=5000,nested rBCM (20 groups),fit_s,2.5369 +aggregations-large-n_NestedKriging_python,python,hartmann3/n=5000,nested rBCM (20 groups),rmse,0.000224517 +aggregations-large-n_NestedKriging_python,python,hartmann3/n=5000,nested rBCM (20 groups),q2,1 +aggregations-large-n_NestedKriging_python,python,hartmann3/n=5000,nested rBCM (20 groups),nlpd,-7.18182 +large-n_Kriging-vecchia-nystrom_python,python,hartmann3/n=1500,subset of data (500),fit_s,0.712767 +large-n_Kriging-vecchia-nystrom_python,python,hartmann3/n=1500,subset of data (500),rmse,0.00269153 +large-n_Kriging-vecchia-nystrom_python,python,hartmann3/n=1500,subset of data (500),q2,0.999992 +large-n_Kriging-vecchia-nystrom_python,python,hartmann3/n=1500,subset of data (500),nlpd,-4.68641 +large-n_Kriging-vecchia-nystrom_python,python,hartmann3/n=5000,subset of data (500),fit_s,0.950298 +large-n_Kriging-vecchia-nystrom_python,python,hartmann3/n=5000,subset of data (500),rmse,0.00247552 +large-n_Kriging-vecchia-nystrom_python,python,hartmann3/n=5000,subset of data (500),q2,0.999994 +large-n_Kriging-vecchia-nystrom_python,python,hartmann3/n=5000,subset of data (500),nlpd,-4.71359 +large-n_Kriging-vecchia-nystrom_python,python,otl_circuit/n=1500,exact,fit_s,10.6103 +aggregations-large-n_NestedKriging_python,python,otl_circuit/n=1500,exact,fit_s,10.6103 +large-n_Kriging-vecchia-nystrom_python,python,otl_circuit/n=1500,exact,rmse,9.81598e-05 +aggregations-large-n_NestedKriging_python,python,otl_circuit/n=1500,exact,rmse,9.81598e-05 +large-n_Kriging-vecchia-nystrom_python,python,otl_circuit/n=1500,exact,q2,1 +aggregations-large-n_NestedKriging_python,python,otl_circuit/n=1500,exact,q2,1 +large-n_Kriging-vecchia-nystrom_python,python,otl_circuit/n=1500,exact,nlpd,-8.21962 +aggregations-large-n_NestedKriging_python,python,otl_circuit/n=1500,exact,nlpd,-8.21962 +large-n_Kriging-vecchia-nystrom_python,python,otl_circuit/n=1500,vecchia(30),fit_s,4.73217 +large-n_Kriging-vecchia-nystrom_python,python,otl_circuit/n=1500,vecchia(30),rmse,0.00629979 +large-n_Kriging-vecchia-nystrom_python,python,otl_circuit/n=1500,vecchia(30),q2,0.999969 +large-n_Kriging-vecchia-nystrom_python,python,otl_circuit/n=1500,vecchia(30),nlpd,-3.95617 +large-n_Kriging-vecchia-nystrom_python,python,otl_circuit/n=5000,vecchia(30),fit_s,14.5549 +large-n_Kriging-vecchia-nystrom_python,python,otl_circuit/n=5000,vecchia(30),rmse,0.00312888 +large-n_Kriging-vecchia-nystrom_python,python,otl_circuit/n=5000,vecchia(30),q2,0.999992 +large-n_Kriging-vecchia-nystrom_python,python,otl_circuit/n=5000,vecchia(30),nlpd,-4.57612 +large-n_Kriging-vecchia-nystrom_python,python,otl_circuit/n=1500,nystrom(100),fit_s,1.47345 +large-n_Kriging-vecchia-nystrom_python,python,otl_circuit/n=1500,nystrom(100),rmse,0.392811 +large-n_Kriging-vecchia-nystrom_python,python,otl_circuit/n=1500,nystrom(100),q2,0.880908 +large-n_Kriging-vecchia-nystrom_python,python,otl_circuit/n=1500,nystrom(100),nlpd,7.06156e+22 +large-n_Kriging-vecchia-nystrom_python,python,otl_circuit/n=5000,nystrom(100),fit_s,4.69579 +large-n_Kriging-vecchia-nystrom_python,python,otl_circuit/n=5000,nystrom(100),rmse,1.11526 +large-n_Kriging-vecchia-nystrom_python,python,otl_circuit/n=5000,nystrom(100),q2,0.0400057 +large-n_Kriging-vecchia-nystrom_python,python,otl_circuit/n=5000,nystrom(100),nlpd,5.31511e+23 +aggregations-large-n_NestedKriging_python,python,otl_circuit/n=1500,nested NK (20 groups),fit_s,0.278129 +aggregations-large-n_NestedKriging_python,python,otl_circuit/n=1500,nested NK (20 groups),rmse,0.00226873 +aggregations-large-n_NestedKriging_python,python,otl_circuit/n=1500,nested NK (20 groups),q2,0.999996 +aggregations-large-n_NestedKriging_python,python,otl_circuit/n=1500,nested NK (20 groups),nlpd,-4.90542 +aggregations-large-n_NestedKriging_python,python,otl_circuit/n=5000,nested NK (20 groups),fit_s,3.62576 +aggregations-large-n_NestedKriging_python,python,otl_circuit/n=5000,nested NK (20 groups),rmse,0.000261686 +aggregations-large-n_NestedKriging_python,python,otl_circuit/n=5000,nested NK (20 groups),q2,1 +aggregations-large-n_NestedKriging_python,python,otl_circuit/n=5000,nested NK (20 groups),nlpd,-7.20623 +aggregations-large-n_NestedKriging_python,python,otl_circuit/n=1500,nested rBCM (20 groups),fit_s,0.315619 +aggregations-large-n_NestedKriging_python,python,otl_circuit/n=1500,nested rBCM (20 groups),rmse,0.00291869 +aggregations-large-n_NestedKriging_python,python,otl_circuit/n=1500,nested rBCM (20 groups),q2,0.999993 +aggregations-large-n_NestedKriging_python,python,otl_circuit/n=1500,nested rBCM (20 groups),nlpd,-2.55512 +aggregations-large-n_NestedKriging_python,python,otl_circuit/n=5000,nested rBCM (20 groups),fit_s,3.39037 +aggregations-large-n_NestedKriging_python,python,otl_circuit/n=5000,nested rBCM (20 groups),rmse,0.000278149 +aggregations-large-n_NestedKriging_python,python,otl_circuit/n=5000,nested rBCM (20 groups),q2,1 +aggregations-large-n_NestedKriging_python,python,otl_circuit/n=5000,nested rBCM (20 groups),nlpd,-4.34927 +large-n_Kriging-vecchia-nystrom_python,python,otl_circuit/n=1500,subset of data (500),fit_s,0.672431 +large-n_Kriging-vecchia-nystrom_python,python,otl_circuit/n=1500,subset of data (500),rmse,0.0004286 +large-n_Kriging-vecchia-nystrom_python,python,otl_circuit/n=1500,subset of data (500),q2,1 +large-n_Kriging-vecchia-nystrom_python,python,otl_circuit/n=1500,subset of data (500),nlpd,-6.72678 +large-n_Kriging-vecchia-nystrom_python,python,otl_circuit/n=5000,subset of data (500),fit_s,0.692672 +large-n_Kriging-vecchia-nystrom_python,python,otl_circuit/n=5000,subset of data (500),rmse,0.000446879 +large-n_Kriging-vecchia-nystrom_python,python,otl_circuit/n=5000,subset of data (500),q2,1 +large-n_Kriging-vecchia-nystrom_python,python,otl_circuit/n=5000,subset of data (500),nlpd,-6.78844 +aggregations-large-n_NestedKriging_python,python,branin/nested,NK,rmse,0.0343984 +aggregations-large-n_NestedKriging_python,python,branin/nested,NK,q2,1 +aggregations-large-n_NestedKriging_python,python,branin/nested,NK,nlpd,2.00082e+14 +aggregations-large-n_NestedKriging_python,python,branin/nested,PoE,rmse,0.0354265 +aggregations-large-n_NestedKriging_python,python,branin/nested,PoE,q2,1 +aggregations-large-n_NestedKriging_python,python,branin/nested,PoE,nlpd,-1.64533 +aggregations-large-n_NestedKriging_python,python,branin/nested,gPoE,rmse,0.0354265 +aggregations-large-n_NestedKriging_python,python,branin/nested,gPoE,q2,1 +aggregations-large-n_NestedKriging_python,python,branin/nested,gPoE,nlpd,-0.621836 +aggregations-large-n_NestedKriging_python,python,branin/nested,BCM,rmse,0.0354196 +aggregations-large-n_NestedKriging_python,python,branin/nested,BCM,q2,1 +aggregations-large-n_NestedKriging_python,python,branin/nested,BCM,nlpd,-1.64533 +aggregations-large-n_NestedKriging_python,python,branin/nested,rBCM,rmse,0.035287 +aggregations-large-n_NestedKriging_python,python,branin/nested,rBCM,q2,1 +aggregations-large-n_NestedKriging_python,python,branin/nested,rBCM,nlpd,-2.61369 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,branin,pylibkriging,q2,0.999944 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,branin,pylibkriging,nlpd,0.137225 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,branin,pylibkriging,fit_s,0.00261593 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,branin,pylibkriging BFGS10,q2,0.999945 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,branin,pylibkriging BFGS10,nlpd,0.137254 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,branin,pylibkriging BFGS10,fit_s,0.02301 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,branin,scikit-learn,q2,0.999597 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,branin,scikit-learn,nlpd,0.928508 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,branin,scikit-learn,fit_s,0.169693 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,branin,SMT,q2,0.999944 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,branin,SMT,nlpd,0.137259 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,branin,SMT,fit_s,0.833295 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,branin,OpenTURNS,q2,0.998221 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,branin,OpenTURNS,nlpd,1.55599 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,branin,OpenTURNS,fit_s,0.00490117 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,six_hump_camel,pylibkriging,q2,0.999418 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,six_hump_camel,pylibkriging,nlpd,0.0429342 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,six_hump_camel,pylibkriging,fit_s,0.00324678 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,six_hump_camel,pylibkriging BFGS10,q2,0.999418 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,six_hump_camel,pylibkriging BFGS10,nlpd,0.0429386 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,six_hump_camel,pylibkriging BFGS10,fit_s,0.060586 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,six_hump_camel,scikit-learn,q2,0.0163311 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,six_hump_camel,scikit-learn,nlpd,4.63763 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,six_hump_camel,scikit-learn,fit_s,0.206083 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,six_hump_camel,SMT,q2,0.999594 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,six_hump_camel,SMT,nlpd,-0.121007 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,six_hump_camel,SMT,fit_s,1.03758 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,six_hump_camel,OpenTURNS,q2,0.956713 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,six_hump_camel,OpenTURNS,nlpd,2.55446 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,six_hump_camel,OpenTURNS,fit_s,0.00741315 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann3,pylibkriging,q2,0.986148 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann3,pylibkriging,nlpd,-0.847247 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann3,pylibkriging,fit_s,0.0169713 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann3,pylibkriging BFGS10,q2,0.986148 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann3,pylibkriging BFGS10,nlpd,-0.847249 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann3,pylibkriging BFGS10,fit_s,0.139588 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann3,scikit-learn,q2,0.983889 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann3,scikit-learn,nlpd,-0.782697 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann3,scikit-learn,fit_s,0.188796 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann3,SMT,q2,0.986148 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann3,SMT,nlpd,-0.847249 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann3,SMT,fit_s,1.02907 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann3,OpenTURNS,q2,0.984712 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann3,OpenTURNS,nlpd,-0.809851 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann3,OpenTURNS,fit_s,0.0153861 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,ishigami,pylibkriging,q2,0.259556 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,ishigami,pylibkriging,nlpd,4.38991 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,ishigami,pylibkriging,fit_s,0.0256422 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,ishigami,pylibkriging BFGS10,q2,0.935722 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,ishigami,pylibkriging BFGS10,nlpd,0.857443 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,ishigami,pylibkriging BFGS10,fit_s,0.308839 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,ishigami,scikit-learn,q2,0.831142 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,ishigami,scikit-learn,nlpd,1.5455 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,ishigami,scikit-learn,fit_s,0.51733 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,ishigami,SMT,q2,0.51196 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,ishigami,SMT,nlpd,3.31349 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,ishigami,SMT,fit_s,2.77225 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,ishigami,OpenTURNS,q2,0.887251 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,ishigami,OpenTURNS,nlpd,1.39348 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,ishigami,OpenTURNS,fit_s,0.0499659 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,friedman,pylibkriging,q2,0.999771 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,friedman,pylibkriging,nlpd,-1.29726 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,friedman,pylibkriging,fit_s,0.0181558 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,friedman,pylibkriging BFGS10,q2,0.999771 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,friedman,pylibkriging BFGS10,nlpd,-1.29727 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,friedman,pylibkriging BFGS10,fit_s,0.215106 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,friedman,scikit-learn,q2,0.999754 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,friedman,scikit-learn,nlpd,-1.37702 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,friedman,scikit-learn,fit_s,0.283978 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,friedman,SMT,q2,0.999899 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,friedman,SMT,nlpd,-1.871 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,friedman,SMT,fit_s,4.3249 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,friedman,OpenTURNS,q2,0.990862 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,friedman,OpenTURNS,nlpd,0.469828 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,friedman,OpenTURNS,fit_s,0.0293689 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,otl_circuit,pylibkriging,q2,0.999922 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,otl_circuit,pylibkriging,nlpd,-3.46228 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,otl_circuit,pylibkriging,fit_s,0.013788 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,otl_circuit,pylibkriging BFGS10,q2,0.999922 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,otl_circuit,pylibkriging BFGS10,nlpd,-3.46227 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,otl_circuit,pylibkriging BFGS10,fit_s,0.141071 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,otl_circuit,scikit-learn,q2,0.999906 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,otl_circuit,scikit-learn,nlpd,-3.43062 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,otl_circuit,scikit-learn,fit_s,0.51218 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,otl_circuit,SMT,q2,0.999954 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,otl_circuit,SMT,nlpd,-3.35016 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,otl_circuit,SMT,fit_s,7.1224 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,otl_circuit,OpenTURNS,q2,0.99777 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,otl_circuit,OpenTURNS,nlpd,-1.61723 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,otl_circuit,OpenTURNS,fit_s,0.0603108 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,piston,pylibkriging,q2,0.997498 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,piston,pylibkriging,nlpd,-3.80667 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,piston,pylibkriging,fit_s,0.0395648 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,piston,pylibkriging BFGS10,q2,0.997498 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,piston,pylibkriging BFGS10,nlpd,-3.8067 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,piston,pylibkriging BFGS10,fit_s,0.327989 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,piston,scikit-learn,q2,0.998684 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,piston,scikit-learn,nlpd,-4.10497 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,piston,scikit-learn,fit_s,1.11386 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,piston,SMT,q2,0.997511 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,piston,SMT,nlpd,-3.86618 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,piston,SMT,fit_s,12.649 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,piston,OpenTURNS,q2,0.990631 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,piston,OpenTURNS,nlpd,-2.9906 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,piston,OpenTURNS,fit_s,0.11631 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,borehole,pylibkriging,q2,0.999694 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,borehole,pylibkriging,nlpd,1.00118 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,borehole,pylibkriging,fit_s,0.0149446 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,borehole,pylibkriging BFGS10,q2,0.999694 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,borehole,pylibkriging BFGS10,nlpd,1.00121 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,borehole,pylibkriging BFGS10,fit_s,0.164608 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,borehole,scikit-learn,q2,0.999929 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,borehole,scikit-learn,nlpd,0.194455 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,borehole,scikit-learn,fit_s,1.04563 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,borehole,SMT,q2,0.999925 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,borehole,SMT,nlpd,0.279122 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,borehole,SMT,fit_s,11.8531 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,borehole,OpenTURNS,q2,0.996206 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,borehole,OpenTURNS,nlpd,2.55431 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,borehole,OpenTURNS,fit_s,0.12395 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,wing_weight,pylibkriging,q2,0.99972 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,wing_weight,pylibkriging,nlpd,1.11517 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,wing_weight,pylibkriging,fit_s,0.0290489 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,wing_weight,pylibkriging BFGS10,q2,0.99972 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,wing_weight,pylibkriging BFGS10,nlpd,1.11512 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,wing_weight,pylibkriging BFGS10,fit_s,0.314707 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,wing_weight,scikit-learn,q2,0.998048 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,wing_weight,scikit-learn,nlpd,2.57304 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,wing_weight,scikit-learn,fit_s,0.729278 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,wing_weight,SMT,q2,0.999898 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,wing_weight,SMT,nlpd,0.604023 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,wing_weight,SMT,fit_s,25.8262 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,wing_weight,OpenTURNS,q2,0.993145 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,wing_weight,OpenTURNS,nlpd,2.83627 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,wing_weight,OpenTURNS,fit_s,0.280434 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann6,pylibkriging,q2,0.422938 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann6,pylibkriging,nlpd,0.0965167 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann6,pylibkriging,fit_s,0.0562823 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann6,pylibkriging BFGS10,q2,0.72352 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann6,pylibkriging BFGS10,nlpd,-0.156898 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann6,pylibkriging BFGS10,fit_s,0.822745 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann6,scikit-learn,q2,0.69049 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann6,scikit-learn,nlpd,-0.137845 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann6,scikit-learn,fit_s,1.3155 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann6,SMT,q2,0.708463 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann6,SMT,nlpd,-0.0177081 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann6,SMT,fit_s,12.0578 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann6,OpenTURNS,q2,0.696177 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann6,OpenTURNS,nlpd,-0.132838 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann6,OpenTURNS,fit_s,0.133332 diff --git a/tools/baseline/metrics-release.csv b/tools/baseline/metrics-release.csv new file mode 100644 index 0000000..d9414f2 --- /dev/null +++ b/tools/baseline/metrics-release.csv @@ -0,0 +1,924 @@ +suite,lang,case,package,metric,value +kernels-trends-interpolation_Kriging_python,python,forrester/gauss/constant,pylibkriging,fit_s,0.0039537 +kernels-trends-interpolation_Kriging_python,python,forrester/gauss/constant,pylibkriging,rmse,0.0848271 +kernels-trends-interpolation_Kriging_python,python,forrester/gauss/constant,pylibkriging,q2,0.999634 +kernels-trends-interpolation_Kriging_python,python,forrester/gauss/constant,pylibkriging,nlpd,-3.06486 +kernels-trends-interpolation_Kriging_python,python,forrester/gauss/linear,pylibkriging,fit_s,0.0008564 +kernels-trends-interpolation_Kriging_python,python,forrester/gauss/linear,pylibkriging,rmse,0.0855468 +kernels-trends-interpolation_Kriging_python,python,forrester/gauss/linear,pylibkriging,q2,0.999628 +kernels-trends-interpolation_Kriging_python,python,forrester/gauss/linear,pylibkriging,nlpd,-3.11029 +kernels-trends-interpolation_Kriging_python,python,forrester/gauss/interactive,pylibkriging,fit_s,0.000818729 +kernels-trends-interpolation_Kriging_python,python,forrester/gauss/interactive,pylibkriging,rmse,0.0855468 +kernels-trends-interpolation_Kriging_python,python,forrester/gauss/interactive,pylibkriging,q2,0.999628 +kernels-trends-interpolation_Kriging_python,python,forrester/gauss/interactive,pylibkriging,nlpd,-3.11029 +kernels-trends-interpolation_Kriging_python,python,forrester/gauss/quadratic,pylibkriging,fit_s,0.000473261 +kernels-trends-interpolation_Kriging_python,python,forrester/gauss/quadratic,pylibkriging,rmse,1.57249 +kernels-trends-interpolation_Kriging_python,python,forrester/gauss/quadratic,pylibkriging,q2,0.874394 +kernels-trends-interpolation_Kriging_python,python,forrester/gauss/quadratic,pylibkriging,nlpd,5.33879 +kernels-trends-interpolation_Kriging_python,python,forrester/exp/constant,pylibkriging,fit_s,0.00042963 +kernels-trends-interpolation_Kriging_python,python,forrester/exp/constant,pylibkriging,rmse,0.721582 +kernels-trends-interpolation_Kriging_python,python,forrester/exp/constant,pylibkriging,q2,0.973551 +kernels-trends-interpolation_Kriging_python,python,forrester/exp/constant,pylibkriging,nlpd,1.54551 +kernels-trends-interpolation_Kriging_python,python,forrester/exp/linear,pylibkriging,fit_s,0.000402451 +kernels-trends-interpolation_Kriging_python,python,forrester/exp/linear,pylibkriging,rmse,0.793413 +kernels-trends-interpolation_Kriging_python,python,forrester/exp/linear,pylibkriging,q2,0.968024 +kernels-trends-interpolation_Kriging_python,python,forrester/exp/linear,pylibkriging,nlpd,1.50639 +kernels-trends-interpolation_Kriging_python,python,forrester/exp/interactive,pylibkriging,fit_s,0.000428438 +kernels-trends-interpolation_Kriging_python,python,forrester/exp/interactive,pylibkriging,rmse,0.793413 +kernels-trends-interpolation_Kriging_python,python,forrester/exp/interactive,pylibkriging,q2,0.968024 +kernels-trends-interpolation_Kriging_python,python,forrester/exp/interactive,pylibkriging,nlpd,1.50639 +kernels-trends-interpolation_Kriging_python,python,forrester/exp/quadratic,pylibkriging,fit_s,0.000418425 +kernels-trends-interpolation_Kriging_python,python,forrester/exp/quadratic,pylibkriging,rmse,0.723267 +kernels-trends-interpolation_Kriging_python,python,forrester/exp/quadratic,pylibkriging,q2,0.973428 +kernels-trends-interpolation_Kriging_python,python,forrester/exp/quadratic,pylibkriging,nlpd,1.28782 +kernels-trends-interpolation_Kriging_python,python,forrester/matern3_2/constant,pylibkriging,fit_s,0.000519753 +kernels-trends-interpolation_Kriging_python,python,forrester/matern3_2/constant,pylibkriging,rmse,0.334007 +kernels-trends-interpolation_Kriging_python,python,forrester/matern3_2/constant,pylibkriging,q2,0.994333 +kernels-trends-interpolation_Kriging_python,python,forrester/matern3_2/constant,pylibkriging,nlpd,-0.227715 +kernels-trends-interpolation_Kriging_python,python,forrester/matern3_2/linear,pylibkriging,fit_s,0.000559092 +kernels-trends-interpolation_Kriging_python,python,forrester/matern3_2/linear,pylibkriging,rmse,0.364477 +kernels-trends-interpolation_Kriging_python,python,forrester/matern3_2/linear,pylibkriging,q2,0.993252 +kernels-trends-interpolation_Kriging_python,python,forrester/matern3_2/linear,pylibkriging,nlpd,-0.187262 +kernels-trends-interpolation_Kriging_python,python,forrester/matern3_2/interactive,pylibkriging,fit_s,0.00060463 +kernels-trends-interpolation_Kriging_python,python,forrester/matern3_2/interactive,pylibkriging,rmse,0.364477 +kernels-trends-interpolation_Kriging_python,python,forrester/matern3_2/interactive,pylibkriging,q2,0.993252 +kernels-trends-interpolation_Kriging_python,python,forrester/matern3_2/interactive,pylibkriging,nlpd,-0.187262 +kernels-trends-interpolation_Kriging_python,python,forrester/matern3_2/quadratic,pylibkriging,fit_s,0.000493765 +kernels-trends-interpolation_Kriging_python,python,forrester/matern3_2/quadratic,pylibkriging,rmse,0.435609 +kernels-trends-interpolation_Kriging_python,python,forrester/matern3_2/quadratic,pylibkriging,q2,0.990361 +kernels-trends-interpolation_Kriging_python,python,forrester/matern3_2/quadratic,pylibkriging,nlpd,-0.183076 +kernels-trends-interpolation_Kriging_python,python,forrester/matern5_2/constant,pylibkriging,fit_s,0.000758886 +kernels-trends-interpolation_Kriging_python,python,forrester/matern5_2/constant,pylibkriging,rmse,0.281945 +kernels-trends-interpolation_Kriging_python,python,forrester/matern5_2/constant,pylibkriging,q2,0.995962 +kernels-trends-interpolation_Kriging_python,python,forrester/matern5_2/constant,pylibkriging,nlpd,-1.01945 +kernels-trends-interpolation_Kriging_python,python,forrester/matern5_2/constant,pylibkriging BFGS10,rmse,0.281945 +kernels-trends-interpolation_Kriging_python,python,forrester/matern5_2/constant,pylibkriging BFGS10,q2,0.995962 +kernels-trends-interpolation_Kriging_python,python,forrester/matern5_2/constant,pylibkriging BFGS10,nlpd,-1.01945 +kernels-trends-interpolation_Kriging_python,python,forrester/matern5_2/linear,pylibkriging,fit_s,0.000576973 +kernels-trends-interpolation_Kriging_python,python,forrester/matern5_2/linear,pylibkriging,rmse,0.298959 +kernels-trends-interpolation_Kriging_python,python,forrester/matern5_2/linear,pylibkriging,q2,0.99546 +kernels-trends-interpolation_Kriging_python,python,forrester/matern5_2/linear,pylibkriging,nlpd,-0.95551 +kernels-trends-interpolation_Kriging_python,python,forrester/matern5_2/interactive,pylibkriging,fit_s,0.00054884 +kernels-trends-interpolation_Kriging_python,python,forrester/matern5_2/interactive,pylibkriging,rmse,0.298959 +kernels-trends-interpolation_Kriging_python,python,forrester/matern5_2/interactive,pylibkriging,q2,0.99546 +kernels-trends-interpolation_Kriging_python,python,forrester/matern5_2/interactive,pylibkriging,nlpd,-0.95551 +kernels-trends-interpolation_Kriging_python,python,forrester/matern5_2/quadratic,pylibkriging,fit_s,0.000477076 +kernels-trends-interpolation_Kriging_python,python,forrester/matern5_2/quadratic,pylibkriging,rmse,0.365039 +kernels-trends-interpolation_Kriging_python,python,forrester/matern5_2/quadratic,pylibkriging,q2,0.993231 +kernels-trends-interpolation_Kriging_python,python,forrester/matern5_2/quadratic,pylibkriging,nlpd,-0.790032 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/gauss/constant,pylibkriging,fit_s,0.0016222 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/gauss/constant,pylibkriging,rmse,0.191274 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/gauss/constant,pylibkriging,q2,0.978742 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/gauss/constant,pylibkriging,nlpd,-0.704948 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/gauss/linear,pylibkriging,fit_s,0.00240469 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/gauss/linear,pylibkriging,rmse,0.236028 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/gauss/linear,pylibkriging,q2,0.96763 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/gauss/linear,pylibkriging,nlpd,-0.580993 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/gauss/interactive,pylibkriging,fit_s,0.00205922 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/gauss/interactive,pylibkriging,rmse,0.236028 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/gauss/interactive,pylibkriging,q2,0.96763 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/gauss/interactive,pylibkriging,nlpd,-0.580993 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/gauss/quadratic,pylibkriging,fit_s,0.00200701 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/gauss/quadratic,pylibkriging,rmse,0.150914 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/gauss/quadratic,pylibkriging,q2,0.986767 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/gauss/quadratic,pylibkriging,nlpd,-0.458431 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/exp/constant,pylibkriging,fit_s,0.00123 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/exp/constant,pylibkriging,rmse,0.196586 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/exp/constant,pylibkriging,q2,0.977545 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/exp/constant,pylibkriging,nlpd,-0.0621192 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/exp/linear,pylibkriging,fit_s,0.00172782 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/exp/linear,pylibkriging,rmse,0.20979 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/exp/linear,pylibkriging,q2,0.974427 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/exp/linear,pylibkriging,nlpd,-0.0466568 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/exp/interactive,pylibkriging,fit_s,0.0017736 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/exp/interactive,pylibkriging,rmse,0.20979 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/exp/interactive,pylibkriging,q2,0.974427 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/exp/interactive,pylibkriging,nlpd,-0.0466568 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/exp/quadratic,pylibkriging,fit_s,0.00394821 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/exp/quadratic,pylibkriging,rmse,0.252565 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/exp/quadratic,pylibkriging,q2,0.962935 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/exp/quadratic,pylibkriging,nlpd,0.220609 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern3_2/constant,pylibkriging,fit_s,0.000825167 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern3_2/constant,pylibkriging,rmse,0.177253 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern3_2/constant,pylibkriging,q2,0.981744 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern3_2/constant,pylibkriging,nlpd,-0.482212 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern3_2/linear,pylibkriging,fit_s,0.000857115 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern3_2/linear,pylibkriging,rmse,0.211499 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern3_2/linear,pylibkriging,q2,0.974009 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern3_2/linear,pylibkriging,nlpd,-0.375844 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern3_2/interactive,pylibkriging,fit_s,0.00114012 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern3_2/interactive,pylibkriging,rmse,0.211499 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern3_2/interactive,pylibkriging,q2,0.974009 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern3_2/interactive,pylibkriging,nlpd,-0.375844 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern3_2/quadratic,pylibkriging,fit_s,0.00237894 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern3_2/quadratic,pylibkriging,rmse,0.184528 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern3_2/quadratic,pylibkriging,q2,0.980215 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern3_2/quadratic,pylibkriging,nlpd,-0.171795 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern5_2/constant,pylibkriging,fit_s,0.000836849 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern5_2/constant,pylibkriging,rmse,0.177638 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern5_2/constant,pylibkriging,q2,0.981665 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern5_2/constant,pylibkriging,nlpd,-0.575421 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern5_2/constant,pylibkriging BFGS10,rmse,0.177638 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern5_2/constant,pylibkriging BFGS10,q2,0.981665 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern5_2/constant,pylibkriging BFGS10,nlpd,-0.575422 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern5_2/linear,pylibkriging,fit_s,0.000859499 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern5_2/linear,pylibkriging,rmse,0.219759 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern5_2/linear,pylibkriging,q2,0.971939 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern5_2/linear,pylibkriging,nlpd,-0.458933 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern5_2/interactive,pylibkriging,fit_s,0.00080061 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern5_2/interactive,pylibkriging,rmse,0.219759 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern5_2/interactive,pylibkriging,q2,0.971939 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern5_2/interactive,pylibkriging,nlpd,-0.458933 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern5_2/quadratic,pylibkriging,fit_s,0.00249147 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern5_2/quadratic,pylibkriging,rmse,0.169941 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern5_2/quadratic,pylibkriging,q2,0.983219 +kernels-trends-interpolation_Kriging_python,python,gramacy_lee_2012/matern5_2/quadratic,pylibkriging,nlpd,-0.28938 +kernels-trends-interpolation_Kriging_python,python,branin/gauss/constant,pylibkriging,fit_s,0.00172019 +kernels-trends-interpolation_Kriging_python,python,branin/gauss/constant,pylibkriging,rmse,7.2629 +kernels-trends-interpolation_Kriging_python,python,branin/gauss/constant,pylibkriging,q2,0.980655 +kernels-trends-interpolation_Kriging_python,python,branin/gauss/constant,pylibkriging,nlpd,4.3152 +kernels-trends-interpolation_Kriging_python,python,branin/gauss/linear,pylibkriging,fit_s,0.00188136 +kernels-trends-interpolation_Kriging_python,python,branin/gauss/linear,pylibkriging,rmse,7.25733 +kernels-trends-interpolation_Kriging_python,python,branin/gauss/linear,pylibkriging,q2,0.980685 +kernels-trends-interpolation_Kriging_python,python,branin/gauss/linear,pylibkriging,nlpd,4.30813 +kernels-trends-interpolation_Kriging_python,python,branin/gauss/interactive,pylibkriging,fit_s,0.00456238 +kernels-trends-interpolation_Kriging_python,python,branin/gauss/interactive,pylibkriging,rmse,7.2576 +kernels-trends-interpolation_Kriging_python,python,branin/gauss/interactive,pylibkriging,q2,0.980683 +kernels-trends-interpolation_Kriging_python,python,branin/gauss/interactive,pylibkriging,nlpd,4.30176 +kernels-trends-interpolation_Kriging_python,python,branin/gauss/quadratic,pylibkriging,fit_s,0.00182772 +kernels-trends-interpolation_Kriging_python,python,branin/gauss/quadratic,pylibkriging,rmse,7.27808 +kernels-trends-interpolation_Kriging_python,python,branin/gauss/quadratic,pylibkriging,q2,0.980574 +kernels-trends-interpolation_Kriging_python,python,branin/gauss/quadratic,pylibkriging,nlpd,4.29399 +kernels-trends-interpolation_Kriging_python,python,branin/exp/constant,pylibkriging,fit_s,0.00215006 +kernels-trends-interpolation_Kriging_python,python,branin/exp/constant,pylibkriging,rmse,14.5887 +kernels-trends-interpolation_Kriging_python,python,branin/exp/constant,pylibkriging,q2,0.921947 +kernels-trends-interpolation_Kriging_python,python,branin/exp/constant,pylibkriging,nlpd,4.09457 +kernels-trends-interpolation_Kriging_python,python,branin/exp/linear,pylibkriging,fit_s,0.00210071 +kernels-trends-interpolation_Kriging_python,python,branin/exp/linear,pylibkriging,rmse,14.9622 +kernels-trends-interpolation_Kriging_python,python,branin/exp/linear,pylibkriging,q2,0.9179 +kernels-trends-interpolation_Kriging_python,python,branin/exp/linear,pylibkriging,nlpd,4.0909 +kernels-trends-interpolation_Kriging_python,python,branin/exp/interactive,pylibkriging,fit_s,0.00220799 +kernels-trends-interpolation_Kriging_python,python,branin/exp/interactive,pylibkriging,rmse,11.6963 +kernels-trends-interpolation_Kriging_python,python,branin/exp/interactive,pylibkriging,q2,0.94983 +kernels-trends-interpolation_Kriging_python,python,branin/exp/interactive,pylibkriging,nlpd,3.79671 +kernels-trends-interpolation_Kriging_python,python,branin/exp/quadratic,pylibkriging,fit_s,0.00237751 +kernels-trends-interpolation_Kriging_python,python,branin/exp/quadratic,pylibkriging,rmse,9.33909 +kernels-trends-interpolation_Kriging_python,python,branin/exp/quadratic,pylibkriging,q2,0.968014 +kernels-trends-interpolation_Kriging_python,python,branin/exp/quadratic,pylibkriging,nlpd,3.62362 +kernels-trends-interpolation_Kriging_python,python,branin/matern3_2/constant,pylibkriging,fit_s,0.00252843 +kernels-trends-interpolation_Kriging_python,python,branin/matern3_2/constant,pylibkriging,rmse,3.24796 +kernels-trends-interpolation_Kriging_python,python,branin/matern3_2/constant,pylibkriging,q2,0.996131 +kernels-trends-interpolation_Kriging_python,python,branin/matern3_2/constant,pylibkriging,nlpd,2.30444 +kernels-trends-interpolation_Kriging_python,python,branin/matern3_2/linear,pylibkriging,fit_s,0.00200725 +kernels-trends-interpolation_Kriging_python,python,branin/matern3_2/linear,pylibkriging,rmse,3.14402 +kernels-trends-interpolation_Kriging_python,python,branin/matern3_2/linear,pylibkriging,q2,0.996375 +kernels-trends-interpolation_Kriging_python,python,branin/matern3_2/linear,pylibkriging,nlpd,2.28067 +kernels-trends-interpolation_Kriging_python,python,branin/matern3_2/interactive,pylibkriging,fit_s,0.00414944 +kernels-trends-interpolation_Kriging_python,python,branin/matern3_2/interactive,pylibkriging,rmse,3.19226 +kernels-trends-interpolation_Kriging_python,python,branin/matern3_2/interactive,pylibkriging,q2,0.996263 +kernels-trends-interpolation_Kriging_python,python,branin/matern3_2/interactive,pylibkriging,nlpd,2.20359 +kernels-trends-interpolation_Kriging_python,python,branin/matern3_2/quadratic,pylibkriging,fit_s,0.00253224 +kernels-trends-interpolation_Kriging_python,python,branin/matern3_2/quadratic,pylibkriging,rmse,3.01232 +kernels-trends-interpolation_Kriging_python,python,branin/matern3_2/quadratic,pylibkriging,q2,0.996672 +kernels-trends-interpolation_Kriging_python,python,branin/matern3_2/quadratic,pylibkriging,nlpd,2.11627 +kernels-trends-interpolation_Kriging_python,python,branin/matern5_2/constant,pylibkriging,fit_s,0.00171685 +kernels-trends-interpolation_Kriging_python,python,branin/matern5_2/constant,pylibkriging,rmse,0.52012 +kernels-trends-interpolation_Kriging_python,python,branin/matern5_2/constant,pylibkriging,q2,0.999901 +kernels-trends-interpolation_Kriging_python,python,branin/matern5_2/constant,pylibkriging,nlpd,0.283399 +kernels-trends-interpolation_Kriging_python,python,branin/matern5_2/constant,pylibkriging BFGS10,rmse,0.52012 +kernels-trends-interpolation_Kriging_python,python,branin/matern5_2/constant,pylibkriging BFGS10,q2,0.999901 +kernels-trends-interpolation_Kriging_python,python,branin/matern5_2/constant,pylibkriging BFGS10,nlpd,0.283395 +kernels-trends-interpolation_Kriging_python,python,branin/matern5_2/linear,pylibkriging,fit_s,0.0021255 +kernels-trends-interpolation_Kriging_python,python,branin/matern5_2/linear,pylibkriging,rmse,0.460868 +kernels-trends-interpolation_Kriging_python,python,branin/matern5_2/linear,pylibkriging,q2,0.999922 +kernels-trends-interpolation_Kriging_python,python,branin/matern5_2/linear,pylibkriging,nlpd,0.209741 +kernels-trends-interpolation_Kriging_python,python,branin/matern5_2/interactive,pylibkriging,fit_s,0.00307012 +kernels-trends-interpolation_Kriging_python,python,branin/matern5_2/interactive,pylibkriging,rmse,0.498484 +kernels-trends-interpolation_Kriging_python,python,branin/matern5_2/interactive,pylibkriging,q2,0.999909 +kernels-trends-interpolation_Kriging_python,python,branin/matern5_2/interactive,pylibkriging,nlpd,0.239079 +kernels-trends-interpolation_Kriging_python,python,branin/matern5_2/quadratic,pylibkriging,fit_s,0.0092895 +kernels-trends-interpolation_Kriging_python,python,branin/matern5_2/quadratic,pylibkriging,rmse,0.67472 +kernels-trends-interpolation_Kriging_python,python,branin/matern5_2/quadratic,pylibkriging,q2,0.999833 +kernels-trends-interpolation_Kriging_python,python,branin/matern5_2/quadratic,pylibkriging,nlpd,0.419266 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/gauss/constant,pylibkriging,fit_s,0.00194049 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/gauss/constant,pylibkriging,rmse,18.9501 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/gauss/constant,pylibkriging,q2,0.536354 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/gauss/constant,pylibkriging,nlpd,84.151 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/gauss/linear,pylibkriging,fit_s,0.00170755 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/gauss/linear,pylibkriging,rmse,18.9579 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/gauss/linear,pylibkriging,q2,0.535976 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/gauss/linear,pylibkriging,nlpd,44.8696 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/gauss/interactive,pylibkriging,fit_s,0.00180864 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/gauss/interactive,pylibkriging,rmse,18.9819 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/gauss/interactive,pylibkriging,q2,0.5348 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/gauss/interactive,pylibkriging,nlpd,35.2765 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/gauss/quadratic,pylibkriging,fit_s,0.00178218 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/gauss/quadratic,pylibkriging,rmse,10.9058 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/gauss/quadratic,pylibkriging,q2,0.846441 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/gauss/quadratic,pylibkriging,nlpd,4.92017 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/exp/constant,pylibkriging,fit_s,0.000977039 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/exp/constant,pylibkriging,rmse,16.9653 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/exp/constant,pylibkriging,q2,0.628393 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/exp/constant,pylibkriging,nlpd,4.48595 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/exp/linear,pylibkriging,fit_s,0.00107837 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/exp/linear,pylibkriging,rmse,17.1781 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/exp/linear,pylibkriging,q2,0.619012 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/exp/linear,pylibkriging,nlpd,4.47061 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/exp/interactive,pylibkriging,fit_s,0.00123405 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/exp/interactive,pylibkriging,rmse,17.1856 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/exp/interactive,pylibkriging,q2,0.618679 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/exp/interactive,pylibkriging,nlpd,4.43133 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/exp/quadratic,pylibkriging,fit_s,0.00120115 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/exp/quadratic,pylibkriging,rmse,7.99484 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/exp/quadratic,pylibkriging,q2,0.917476 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/exp/quadratic,pylibkriging,nlpd,3.35612 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern3_2/constant,pylibkriging,fit_s,0.00264788 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern3_2/constant,pylibkriging,rmse,18.4958 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern3_2/constant,pylibkriging,q2,0.558321 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern3_2/constant,pylibkriging,nlpd,18.8349 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern3_2/linear,pylibkriging,fit_s,0.00251079 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern3_2/linear,pylibkriging,rmse,18.5562 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern3_2/linear,pylibkriging,q2,0.555433 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern3_2/linear,pylibkriging,nlpd,14.9494 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern3_2/interactive,pylibkriging,fit_s,0.00148797 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern3_2/interactive,pylibkriging,rmse,18.561 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern3_2/interactive,pylibkriging,q2,0.555202 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern3_2/interactive,pylibkriging,nlpd,12.9923 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern3_2/quadratic,pylibkriging,fit_s,0.00289869 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern3_2/quadratic,pylibkriging,rmse,8.91645 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern3_2/quadratic,pylibkriging,q2,0.897353 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern3_2/quadratic,pylibkriging,nlpd,3.94939 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern5_2/constant,pylibkriging,fit_s,0.00238967 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern5_2/constant,pylibkriging,rmse,18.3236 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern5_2/constant,pylibkriging,q2,0.566507 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern5_2/constant,pylibkriging,nlpd,18.4322 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern5_2/constant,pylibkriging BFGS10,rmse,1.36208 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern5_2/constant,pylibkriging BFGS10,q2,0.997605 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern5_2/constant,pylibkriging BFGS10,nlpd,1.03215 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern5_2/linear,pylibkriging,fit_s,0.00207353 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern5_2/linear,pylibkriging,rmse,18.7127 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern5_2/linear,pylibkriging,q2,0.547901 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern5_2/linear,pylibkriging,nlpd,24.5771 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern5_2/interactive,pylibkriging,fit_s,0.0020268 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern5_2/interactive,pylibkriging,rmse,18.7194 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern5_2/interactive,pylibkriging,q2,0.547577 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern5_2/interactive,pylibkriging,nlpd,20.4255 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern5_2/quadratic,pylibkriging,fit_s,0.00144219 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern5_2/quadratic,pylibkriging,rmse,10.2878 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern5_2/quadratic,pylibkriging,q2,0.863351 +kernels-trends-interpolation_Kriging_python,python,six_hump_camel/matern5_2/quadratic,pylibkriging,nlpd,4.46344 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/gauss/constant,pylibkriging,fit_s,0.00526643 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/gauss/constant,pylibkriging,rmse,0.295934 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/gauss/constant,pylibkriging,q2,0.920537 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/gauss/constant,pylibkriging,nlpd,1.58666 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/gauss/linear,pylibkriging,fit_s,0.00606179 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/gauss/linear,pylibkriging,rmse,0.296355 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/gauss/linear,pylibkriging,q2,0.92031 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/gauss/linear,pylibkriging,nlpd,1.2962 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/gauss/interactive,pylibkriging,fit_s,0.00633311 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/gauss/interactive,pylibkriging,rmse,0.285136 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/gauss/interactive,pylibkriging,q2,0.926229 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/gauss/interactive,pylibkriging,nlpd,1.04122 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/gauss/quadratic,pylibkriging,fit_s,0.00615239 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/gauss/quadratic,pylibkriging,rmse,0.237308 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/gauss/quadratic,pylibkriging,q2,0.948902 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/gauss/quadratic,pylibkriging,nlpd,0.146484 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/exp/constant,pylibkriging,fit_s,0.00271893 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/exp/constant,pylibkriging,rmse,0.334876 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/exp/constant,pylibkriging,q2,0.898247 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/exp/constant,pylibkriging,nlpd,0.339235 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/exp/linear,pylibkriging,fit_s,0.00223827 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/exp/linear,pylibkriging,rmse,0.333447 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/exp/linear,pylibkriging,q2,0.899114 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/exp/linear,pylibkriging,nlpd,0.336515 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/exp/interactive,pylibkriging,fit_s,0.00192547 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/exp/interactive,pylibkriging,rmse,0.336819 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/exp/interactive,pylibkriging,q2,0.897063 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/exp/interactive,pylibkriging,nlpd,0.329566 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/exp/quadratic,pylibkriging,fit_s,0.00333118 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/exp/quadratic,pylibkriging,rmse,0.327816 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/exp/quadratic,pylibkriging,q2,0.902492 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/exp/quadratic,pylibkriging,nlpd,0.322056 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern3_2/constant,pylibkriging,fit_s,0.00606227 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern3_2/constant,pylibkriging,rmse,0.684362 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern3_2/constant,pylibkriging,q2,0.575038 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern3_2/constant,pylibkriging,nlpd,0.972803 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern3_2/linear,pylibkriging,fit_s,0.00382805 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern3_2/linear,pylibkriging,rmse,0.650201 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern3_2/linear,pylibkriging,q2,0.616404 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern3_2/linear,pylibkriging,nlpd,0.921455 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern3_2/interactive,pylibkriging,fit_s,0.00707507 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern3_2/interactive,pylibkriging,rmse,0.274326 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern3_2/interactive,pylibkriging,q2,0.931717 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern3_2/interactive,pylibkriging,nlpd,0.0549679 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern3_2/quadratic,pylibkriging,fit_s,0.00617123 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern3_2/quadratic,pylibkriging,rmse,0.263745 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern3_2/quadratic,pylibkriging,q2,0.936883 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern3_2/quadratic,pylibkriging,nlpd,0.0446267 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern5_2/constant,pylibkriging,fit_s,0.00426936 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern5_2/constant,pylibkriging,rmse,0.674391 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern5_2/constant,pylibkriging,q2,0.58733 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern5_2/constant,pylibkriging,nlpd,0.944765 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern5_2/constant,pylibkriging BFGS10,rmse,0.268216 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern5_2/constant,pylibkriging BFGS10,q2,0.934725 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern5_2/constant,pylibkriging BFGS10,nlpd,0.169585 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern5_2/linear,pylibkriging,fit_s,0.00730562 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern5_2/linear,pylibkriging,rmse,0.268808 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern5_2/linear,pylibkriging,q2,0.934436 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern5_2/linear,pylibkriging,nlpd,0.142382 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern5_2/interactive,pylibkriging,fit_s,0.00520349 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern5_2/interactive,pylibkriging,rmse,0.271088 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern5_2/interactive,pylibkriging,q2,0.933319 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern5_2/interactive,pylibkriging,nlpd,0.128788 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern5_2/quadratic,pylibkriging,fit_s,0.00757623 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern5_2/quadratic,pylibkriging,rmse,0.253093 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern5_2/quadratic,pylibkriging,q2,0.941878 +kernels-trends-interpolation_Kriging_python,python,log_goldstein_price/matern5_2/quadratic,pylibkriging,nlpd,0.025469 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/gauss/constant,pylibkriging,fit_s,0.000713825 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/gauss/constant,pylibkriging,rmse,0.026127 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/gauss/constant,pylibkriging,q2,0.999737 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/gauss/constant,pylibkriging,nlpd,-2.69817 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/gauss/linear,pylibkriging,fit_s,0.000786066 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/gauss/linear,pylibkriging,rmse,1.06243 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/gauss/linear,pylibkriging,q2,0.564691 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/gauss/linear,pylibkriging,nlpd,2.7394 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/gauss/interactive,pylibkriging,fit_s,0.000851393 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/gauss/interactive,pylibkriging,rmse,1.06504 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/gauss/interactive,pylibkriging,q2,0.562551 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/gauss/interactive,pylibkriging,nlpd,2.33575 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/gauss/quadratic,pylibkriging,fit_s,0.0012238 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/gauss/quadratic,pylibkriging,rmse,0.281816 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/gauss/quadratic,pylibkriging,q2,0.969371 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/gauss/quadratic,pylibkriging,nlpd,0.71682 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/exp/constant,pylibkriging,fit_s,0.000771999 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/exp/constant,pylibkriging,rmse,0.249023 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/exp/constant,pylibkriging,q2,0.976085 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/exp/constant,pylibkriging,nlpd,0.394159 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/exp/linear,pylibkriging,fit_s,0.0006814 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/exp/linear,pylibkriging,rmse,0.245558 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/exp/linear,pylibkriging,q2,0.976746 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/exp/linear,pylibkriging,nlpd,0.224593 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/exp/interactive,pylibkriging,fit_s,0.000779867 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/exp/interactive,pylibkriging,rmse,0.239032 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/exp/interactive,pylibkriging,q2,0.977965 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/exp/interactive,pylibkriging,nlpd,0.226133 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/exp/quadratic,pylibkriging,fit_s,0.00164485 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/exp/quadratic,pylibkriging,rmse,0.237713 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/exp/quadratic,pylibkriging,q2,0.978208 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/exp/quadratic,pylibkriging,nlpd,-0.139582 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern3_2/constant,pylibkriging,fit_s,0.000581741 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern3_2/constant,pylibkriging,rmse,0.0600911 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern3_2/constant,pylibkriging,q2,0.998607 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern3_2/constant,pylibkriging,nlpd,-1.34658 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern3_2/linear,pylibkriging,fit_s,0.000698566 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern3_2/linear,pylibkriging,rmse,0.0584376 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern3_2/linear,pylibkriging,q2,0.998683 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern3_2/linear,pylibkriging,nlpd,-1.34912 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern3_2/interactive,pylibkriging,fit_s,0.000763416 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern3_2/interactive,pylibkriging,rmse,0.0576096 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern3_2/interactive,pylibkriging,q2,0.99872 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern3_2/interactive,pylibkriging,nlpd,-1.35376 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern3_2/quadratic,pylibkriging,fit_s,0.00313497 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern3_2/quadratic,pylibkriging,rmse,0.215336 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern3_2/quadratic,pylibkriging,q2,0.982117 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern3_2/quadratic,pylibkriging,nlpd,-0.350837 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern5_2/constant,pylibkriging,fit_s,0.000642776 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern5_2/constant,pylibkriging,rmse,0.0246348 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern5_2/constant,pylibkriging,q2,0.999766 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern5_2/constant,pylibkriging,nlpd,-2.59598 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern5_2/constant,pylibkriging BFGS10,rmse,0.0246338 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern5_2/constant,pylibkriging BFGS10,q2,0.999766 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern5_2/constant,pylibkriging BFGS10,nlpd,-2.59598 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern5_2/linear,pylibkriging,fit_s,0.000878334 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern5_2/linear,pylibkriging,rmse,0.0232483 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern5_2/linear,pylibkriging,q2,0.999792 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern5_2/linear,pylibkriging,nlpd,-2.58843 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern5_2/interactive,pylibkriging,fit_s,0.000845909 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern5_2/interactive,pylibkriging,rmse,0.0230248 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern5_2/interactive,pylibkriging,q2,0.999796 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern5_2/interactive,pylibkriging,nlpd,-2.58839 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern5_2/quadratic,pylibkriging,fit_s,0.00176644 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern5_2/quadratic,pylibkriging,rmse,0.209802 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern5_2/quadratic,pylibkriging,q2,0.983025 +kernels-trends-interpolation_Kriging_python,python,lim_polynomial/matern5_2/quadratic,pylibkriging,nlpd,-0.403998 +kernels-trends-interpolation_Kriging_python,python,ishigami/gauss/constant,pylibkriging,fit_s,0.0820026 +kernels-trends-interpolation_Kriging_python,python,ishigami/gauss/constant,pylibkriging,rmse,1.41975 +kernels-trends-interpolation_Kriging_python,python,ishigami/gauss/constant,pylibkriging,q2,0.844642 +kernels-trends-interpolation_Kriging_python,python,ishigami/gauss/constant,pylibkriging,nlpd,2.25894 +kernels-trends-interpolation_Kriging_python,python,ishigami/gauss/linear,pylibkriging,fit_s,0.0247335 +kernels-trends-interpolation_Kriging_python,python,ishigami/gauss/linear,pylibkriging,rmse,2.84579 +kernels-trends-interpolation_Kriging_python,python,ishigami/gauss/linear,pylibkriging,q2,0.375811 +kernels-trends-interpolation_Kriging_python,python,ishigami/gauss/linear,pylibkriging,nlpd,3.41805 +kernels-trends-interpolation_Kriging_python,python,ishigami/gauss/interactive,pylibkriging,fit_s,0.0169995 +kernels-trends-interpolation_Kriging_python,python,ishigami/gauss/interactive,pylibkriging,rmse,2.85366 +kernels-trends-interpolation_Kriging_python,python,ishigami/gauss/interactive,pylibkriging,q2,0.372353 +kernels-trends-interpolation_Kriging_python,python,ishigami/gauss/interactive,pylibkriging,nlpd,3.28643 +kernels-trends-interpolation_Kriging_python,python,ishigami/gauss/quadratic,pylibkriging,fit_s,0.00751019 +kernels-trends-interpolation_Kriging_python,python,ishigami/gauss/quadratic,pylibkriging,rmse,2.08982 +kernels-trends-interpolation_Kriging_python,python,ishigami/gauss/quadratic,pylibkriging,q2,0.66339 +kernels-trends-interpolation_Kriging_python,python,ishigami/gauss/quadratic,pylibkriging,nlpd,2.80099 +kernels-trends-interpolation_Kriging_python,python,ishigami/exp/constant,pylibkriging,fit_s,0.00825953 +kernels-trends-interpolation_Kriging_python,python,ishigami/exp/constant,pylibkriging,rmse,1.41632 +kernels-trends-interpolation_Kriging_python,python,ishigami/exp/constant,pylibkriging,q2,0.845391 +kernels-trends-interpolation_Kriging_python,python,ishigami/exp/constant,pylibkriging,nlpd,1.75239 +kernels-trends-interpolation_Kriging_python,python,ishigami/exp/linear,pylibkriging,fit_s,0.00724149 +kernels-trends-interpolation_Kriging_python,python,ishigami/exp/linear,pylibkriging,rmse,1.4095 +kernels-trends-interpolation_Kriging_python,python,ishigami/exp/linear,pylibkriging,q2,0.846877 +kernels-trends-interpolation_Kriging_python,python,ishigami/exp/linear,pylibkriging,nlpd,1.74779 +kernels-trends-interpolation_Kriging_python,python,ishigami/exp/interactive,pylibkriging,fit_s,0.00750446 +kernels-trends-interpolation_Kriging_python,python,ishigami/exp/interactive,pylibkriging,rmse,1.41511 +kernels-trends-interpolation_Kriging_python,python,ishigami/exp/interactive,pylibkriging,q2,0.845657 +kernels-trends-interpolation_Kriging_python,python,ishigami/exp/interactive,pylibkriging,nlpd,1.75024 +kernels-trends-interpolation_Kriging_python,python,ishigami/exp/quadratic,pylibkriging,fit_s,0.00750661 +kernels-trends-interpolation_Kriging_python,python,ishigami/exp/quadratic,pylibkriging,rmse,1.40329 +kernels-trends-interpolation_Kriging_python,python,ishigami/exp/quadratic,pylibkriging,q2,0.848223 +kernels-trends-interpolation_Kriging_python,python,ishigami/exp/quadratic,pylibkriging,nlpd,1.73254 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern3_2/constant,pylibkriging,fit_s,0.0483243 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern3_2/constant,pylibkriging,rmse,1.32974 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern3_2/constant,pylibkriging,q2,0.863717 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern3_2/constant,pylibkriging,nlpd,1.50266 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern3_2/linear,pylibkriging,fit_s,0.0462224 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern3_2/linear,pylibkriging,rmse,1.30244 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern3_2/linear,pylibkriging,q2,0.869255 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern3_2/linear,pylibkriging,nlpd,1.47906 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern3_2/interactive,pylibkriging,fit_s,0.0462408 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern3_2/interactive,pylibkriging,rmse,1.30591 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern3_2/interactive,pylibkriging,q2,0.868558 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern3_2/interactive,pylibkriging,nlpd,1.47384 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern3_2/quadratic,pylibkriging,fit_s,0.0412879 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern3_2/quadratic,pylibkriging,rmse,1.32197 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern3_2/quadratic,pylibkriging,q2,0.865305 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern3_2/quadratic,pylibkriging,nlpd,1.46827 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern5_2/constant,pylibkriging,fit_s,0.02212 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern5_2/constant,pylibkriging,rmse,3.00566 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern5_2/constant,pylibkriging,q2,0.303709 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern5_2/constant,pylibkriging,nlpd,3.42085 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern5_2/constant,pylibkriging BFGS10,rmse,1.33903 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern5_2/constant,pylibkriging BFGS10,q2,0.861806 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern5_2/constant,pylibkriging BFGS10,nlpd,1.66132 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern5_2/linear,pylibkriging,fit_s,0.0472796 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern5_2/linear,pylibkriging,rmse,1.3434 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern5_2/linear,pylibkriging,q2,0.860901 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern5_2/linear,pylibkriging,nlpd,1.65103 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern5_2/interactive,pylibkriging,fit_s,0.0477624 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern5_2/interactive,pylibkriging,rmse,1.34479 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern5_2/interactive,pylibkriging,q2,0.860615 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern5_2/interactive,pylibkriging,nlpd,1.64153 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern5_2/quadratic,pylibkriging,fit_s,0.0483804 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern5_2/quadratic,pylibkriging,rmse,1.37082 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern5_2/quadratic,pylibkriging,q2,0.855166 +kernels-trends-interpolation_Kriging_python,python,ishigami/matern5_2/quadratic,pylibkriging,nlpd,1.62228 +kernels-trends-interpolation_Kriging_python,python,hartmann3/gauss/constant,pylibkriging,fit_s,0.00703788 +kernels-trends-interpolation_Kriging_python,python,hartmann3/gauss/constant,pylibkriging,rmse,0.156226 +kernels-trends-interpolation_Kriging_python,python,hartmann3/gauss/constant,pylibkriging,q2,0.971116 +kernels-trends-interpolation_Kriging_python,python,hartmann3/gauss/constant,pylibkriging,nlpd,-0.555623 +kernels-trends-interpolation_Kriging_python,python,hartmann3/gauss/linear,pylibkriging,fit_s,0.0116901 +kernels-trends-interpolation_Kriging_python,python,hartmann3/gauss/linear,pylibkriging,rmse,0.153297 +kernels-trends-interpolation_Kriging_python,python,hartmann3/gauss/linear,pylibkriging,q2,0.972189 +kernels-trends-interpolation_Kriging_python,python,hartmann3/gauss/linear,pylibkriging,nlpd,-0.568404 +kernels-trends-interpolation_Kriging_python,python,hartmann3/gauss/interactive,pylibkriging,fit_s,0.00962472 +kernels-trends-interpolation_Kriging_python,python,hartmann3/gauss/interactive,pylibkriging,rmse,0.712947 +kernels-trends-interpolation_Kriging_python,python,hartmann3/gauss/interactive,pylibkriging,q2,0.398455 +kernels-trends-interpolation_Kriging_python,python,hartmann3/gauss/interactive,pylibkriging,nlpd,1.74554 +kernels-trends-interpolation_Kriging_python,python,hartmann3/gauss/quadratic,pylibkriging,fit_s,0.00845146 +kernels-trends-interpolation_Kriging_python,python,hartmann3/gauss/quadratic,pylibkriging,rmse,0.73238 +kernels-trends-interpolation_Kriging_python,python,hartmann3/gauss/quadratic,pylibkriging,q2,0.365216 +kernels-trends-interpolation_Kriging_python,python,hartmann3/gauss/quadratic,pylibkriging,nlpd,2.14588 +kernels-trends-interpolation_Kriging_python,python,hartmann3/exp/constant,pylibkriging,fit_s,0.00484872 +kernels-trends-interpolation_Kriging_python,python,hartmann3/exp/constant,pylibkriging,rmse,0.347348 +kernels-trends-interpolation_Kriging_python,python,hartmann3/exp/constant,pylibkriging,q2,0.857215 +kernels-trends-interpolation_Kriging_python,python,hartmann3/exp/constant,pylibkriging,nlpd,0.352683 +kernels-trends-interpolation_Kriging_python,python,hartmann3/exp/linear,pylibkriging,fit_s,0.00318956 +kernels-trends-interpolation_Kriging_python,python,hartmann3/exp/linear,pylibkriging,rmse,0.347273 +kernels-trends-interpolation_Kriging_python,python,hartmann3/exp/linear,pylibkriging,q2,0.857276 +kernels-trends-interpolation_Kriging_python,python,hartmann3/exp/linear,pylibkriging,nlpd,0.343631 +kernels-trends-interpolation_Kriging_python,python,hartmann3/exp/interactive,pylibkriging,fit_s,0.00330567 +kernels-trends-interpolation_Kriging_python,python,hartmann3/exp/interactive,pylibkriging,rmse,0.378349 +kernels-trends-interpolation_Kriging_python,python,hartmann3/exp/interactive,pylibkriging,q2,0.83059 +kernels-trends-interpolation_Kriging_python,python,hartmann3/exp/interactive,pylibkriging,nlpd,0.385842 +kernels-trends-interpolation_Kriging_python,python,hartmann3/exp/quadratic,pylibkriging,fit_s,0.00355411 +kernels-trends-interpolation_Kriging_python,python,hartmann3/exp/quadratic,pylibkriging,rmse,0.355119 +kernels-trends-interpolation_Kriging_python,python,hartmann3/exp/quadratic,pylibkriging,q2,0.850755 +kernels-trends-interpolation_Kriging_python,python,hartmann3/exp/quadratic,pylibkriging,nlpd,0.334371 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern3_2/constant,pylibkriging,fit_s,0.0107734 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern3_2/constant,pylibkriging,rmse,0.227521 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern3_2/constant,pylibkriging,q2,0.938737 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern3_2/constant,pylibkriging,nlpd,-0.181207 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern3_2/linear,pylibkriging,fit_s,0.0118749 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern3_2/linear,pylibkriging,rmse,0.23038 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern3_2/linear,pylibkriging,q2,0.937188 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern3_2/linear,pylibkriging,nlpd,-0.162203 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern3_2/interactive,pylibkriging,fit_s,0.00637007 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern3_2/interactive,pylibkriging,rmse,0.315752 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern3_2/interactive,pylibkriging,q2,0.88201 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern3_2/interactive,pylibkriging,nlpd,0.898253 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern3_2/quadratic,pylibkriging,fit_s,0.00598764 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern3_2/quadratic,pylibkriging,rmse,0.291393 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern3_2/quadratic,pylibkriging,q2,0.899513 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern3_2/quadratic,pylibkriging,nlpd,0.53743 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern5_2/constant,pylibkriging,fit_s,0.0100911 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern5_2/constant,pylibkriging,rmse,0.190287 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern5_2/constant,pylibkriging,q2,0.957148 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern5_2/constant,pylibkriging,nlpd,-0.403054 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern5_2/constant,pylibkriging BFGS10,rmse,0.190286 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern5_2/constant,pylibkriging BFGS10,q2,0.957148 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern5_2/constant,pylibkriging BFGS10,nlpd,-0.40306 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern5_2/linear,pylibkriging,fit_s,0.0106812 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern5_2/linear,pylibkriging,rmse,0.189633 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern5_2/linear,pylibkriging,q2,0.957442 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern5_2/linear,pylibkriging,nlpd,-0.410075 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern5_2/interactive,pylibkriging,fit_s,0.00839925 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern5_2/interactive,pylibkriging,rmse,0.307114 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern5_2/interactive,pylibkriging,q2,0.888377 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern5_2/interactive,pylibkriging,nlpd,1.50063 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern5_2/quadratic,pylibkriging,fit_s,0.00724792 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern5_2/quadratic,pylibkriging,rmse,0.285355 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern5_2/quadratic,pylibkriging,q2,0.903634 +kernels-trends-interpolation_Kriging_python,python,hartmann3/matern5_2/quadratic,pylibkriging,nlpd,0.898128 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,branin/LL,pylibkriging,rmse,71.4801 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,branin/LL,pylibkriging,q2,-1991.62 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,branin/LL,pylibkriging,nlpd,13603.3 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,branin/LOO,pylibkriging,rmse,71.5631 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,branin/LOO,pylibkriging,q2,-1996.25 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,branin/LOO,pylibkriging,nlpd,18836.2 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,branin/LMP,pylibkriging,rmse,71.5295 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,branin/LMP,pylibkriging,q2,-1994.38 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,branin/LMP,pylibkriging,nlpd,14603.3 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,hartmann3/LL,pylibkriging,rmse,1.30952 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,hartmann3/LL,pylibkriging,q2,-2013.29 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,hartmann3/LL,pylibkriging,nlpd,41.7128 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,hartmann3/LOO,pylibkriging,rmse,1.31436 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,hartmann3/LOO,pylibkriging,q2,-2028.22 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,hartmann3/LOO,pylibkriging,nlpd,49.7135 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,hartmann3/LMP,pylibkriging,rmse,1.31148 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,hartmann3/LMP,pylibkriging,q2,-2019.34 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,hartmann3/LMP,pylibkriging,nlpd,43.397 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,ishigami/LL,pylibkriging,rmse,5.11103 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,ishigami/LL,pylibkriging,q2,-1783.42 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,ishigami/LL,pylibkriging,nlpd,10.4546 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,ishigami/LOO,pylibkriging,rmse,5.81789 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,ishigami/LOO,pylibkriging,q2,-2311.12 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,ishigami/LOO,pylibkriging,nlpd,40.8323 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,ishigami/LMP,pylibkriging,rmse,5.18495 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,ishigami/LMP,pylibkriging,q2,-1835.41 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,ishigami/LMP,pylibkriging,nlpd,11.5894 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,friedman/LL,pylibkriging,rmse,6.87533 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,friedman/LL,pylibkriging,q2,-1991.33 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,friedman/LL,pylibkriging,nlpd,651.706 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,friedman/LOO,pylibkriging,rmse,6.87323 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,friedman/LOO,pylibkriging,q2,-1990.12 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,friedman/LOO,pylibkriging,nlpd,484.115 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,friedman/LMP,pylibkriging,rmse,6.87647 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,friedman/LMP,pylibkriging,q2,-1991.99 +objectives-LL-LOO-LMP-gradients_Kriging_python,python,friedman/LMP,pylibkriging,nlpd,658.808 +noisy-data_Kriging-nugget-noise_python,python,forrester,nugget,rmse,0.198091 +noisy-data_Kriging-nugget-noise_python,python,forrester,nugget,q2,0.998087 +noisy-data_Kriging-nugget-noise_python,python,forrester,nugget,nlpd,0.286631 +noisy-data_Kriging-nugget-noise_python,python,forrester,noise-free fit,rmse,0.487916 +noisy-data_Kriging-nugget-noise_python,python,forrester,noise-free fit,q2,0.988392 +noisy-data_Kriging-nugget-noise_python,python,forrester,noise-free fit,nlpd,862.687 +noisy-data_Kriging-nugget-noise_python,python,friedman,nugget,rmse,0.467335 +noisy-data_Kriging-nugget-noise_python,python,friedman,nugget,q2,0.99089 +noisy-data_Kriging-nugget-noise_python,python,friedman,nugget,nlpd,1.07073 +noisy-data_Kriging-nugget-noise_python,python,friedman,noise-free fit,rmse,1.27156 +noisy-data_Kriging-nugget-noise_python,python,friedman,noise-free fit,q2,0.932559 +noisy-data_Kriging-nugget-noise_python,python,friedman,noise-free fit,nlpd,1.64495 +noisy-data_Kriging-nugget-noise_python,python,gramacy_lee_2012/hetero,known variances,rmse,0.168741 +noisy-data_Kriging-nugget-noise_python,python,gramacy_lee_2012/hetero,known variances,q2,0.982515 +noisy-data_Kriging-nugget-noise_python,python,gramacy_lee_2012/hetero,known variances,nlpd,-0.570574 +noisy-data_Kriging-nugget-noise_python,python,gramacy_lee_2012/hetero,nugget,rmse,0.170945 +noisy-data_Kriging-nugget-noise_python,python,gramacy_lee_2012/hetero,nugget,q2,0.982055 +noisy-data_Kriging-nugget-noise_python,python,gramacy_lee_2012/hetero,nugget,nlpd,-0.221568 +warpings-categorical-ordinal_WarpKriging_python,python,gl2012,Kriging,rmse,0.167263 +warpings-categorical-ordinal_WarpKriging_python,python,gl2012,Kriging,q2,0.981176 +warpings-categorical-ordinal_WarpKriging_python,python,gl2012,Kriging,nlpd,-1.08917 +warpings-categorical-ordinal_WarpKriging_python,python,gl2012,affine,rmse,0.167263 +warpings-categorical-ordinal_WarpKriging_python,python,gl2012,affine,q2,0.981176 +warpings-categorical-ordinal_WarpKriging_python,python,gl2012,affine,nlpd,-1.08917 +warpings-categorical-ordinal_WarpKriging_python,python,gl2012,boxcox,rmse,0.16759 +warpings-categorical-ordinal_WarpKriging_python,python,gl2012,boxcox,q2,0.981103 +warpings-categorical-ordinal_WarpKriging_python,python,gl2012,boxcox,nlpd,282.422 +warpings-categorical-ordinal_WarpKriging_python,python,gl2012,kumaraswamy,rmse,0.168286 +warpings-categorical-ordinal_WarpKriging_python,python,gl2012,kumaraswamy,q2,0.980945 +warpings-categorical-ordinal_WarpKriging_python,python,gl2012,kumaraswamy,nlpd,2.04577e+06 +warpings-categorical-ordinal_WarpKriging_python,python,gl2012,knots(5),rmse,0.16791 +warpings-categorical-ordinal_WarpKriging_python,python,gl2012,knots(5),q2,0.98103 +warpings-categorical-ordinal_WarpKriging_python,python,gl2012,knots(5),nlpd,2.04376e+06 +warpings-categorical-ordinal_WarpKriging_python,python,gl2012,neural_mono(8),rmse,0.170824 +warpings-categorical-ordinal_WarpKriging_python,python,gl2012,neural_mono(8),q2,0.980366 +warpings-categorical-ordinal_WarpKriging_python,python,gl2012,neural_mono(8),nlpd,-0.265141 +warpings-categorical-ordinal_WarpKriging_python,python,gl2008,Kriging,rmse,0.0534649 +warpings-categorical-ordinal_WarpKriging_python,python,gl2008,Kriging,q2,0.585636 +warpings-categorical-ordinal_WarpKriging_python,python,gl2008,Kriging,nlpd,-1.74569 +warpings-categorical-ordinal_WarpKriging_python,python,gl2008,kumaraswamy,rmse,0.0535298 +warpings-categorical-ordinal_WarpKriging_python,python,gl2008,kumaraswamy,q2,0.58463 +warpings-categorical-ordinal_WarpKriging_python,python,gl2008,kumaraswamy,nlpd,-1.74552 +warpings-categorical-ordinal_WarpKriging_python,python,mixed,Kriging (level as number),rmse,16.7693 +warpings-categorical-ordinal_WarpKriging_python,python,mixed,Kriging (level as number),q2,0.838453 +warpings-categorical-ordinal_WarpKriging_python,python,mixed,Kriging (level as number),nlpd,3.81063 +warpings-categorical-ordinal_WarpKriging_python,python,mixed,"categorical(5,2)",rmse,16.1438 +warpings-categorical-ordinal_WarpKriging_python,python,mixed,"categorical(5,2)",q2,0.85028 +warpings-categorical-ordinal_WarpKriging_python,python,mixed,"categorical(5,2)",nlpd,3.3206 +warpings-categorical-ordinal_WarpKriging_python,python,ordinal,Kriging (rank as number),rmse,0.213845 +warpings-categorical-ordinal_WarpKriging_python,python,ordinal,Kriging (rank as number),q2,0.953177 +warpings-categorical-ordinal_WarpKriging_python,python,ordinal,Kriging (rank as number),nlpd,0.810969 +warpings-categorical-ordinal_WarpKriging_python,python,ordinal,ordinal(6),rmse,0.213043 +warpings-categorical-ordinal_WarpKriging_python,python,ordinal,ordinal(6),q2,0.953528 +warpings-categorical-ordinal_WarpKriging_python,python,ordinal,ordinal(6),nlpd,0.78888 +high-dimension-normalize-screening_Kriging_python,python,otl_circuit,unit cube,fit_s,0.00796986 +high-dimension-normalize-screening_Kriging_python,python,otl_circuit,unit cube,rmse,0.0150514 +high-dimension-normalize-screening_Kriging_python,python,otl_circuit,unit cube,q2,0.999836 +high-dimension-normalize-screening_Kriging_python,python,otl_circuit,unit cube,nlpd,-2.90452 +high-dimension-normalize-screening_Kriging_python,python,otl_circuit,native + normalize,rmse,0.0150514 +high-dimension-normalize-screening_Kriging_python,python,otl_circuit,native + normalize,q2,0.999836 +high-dimension-normalize-screening_Kriging_python,python,otl_circuit,native + normalize,nlpd,-2.90452 +high-dimension-normalize-screening_Kriging_python,python,piston,unit cube,fit_s,0.023371 +high-dimension-normalize-screening_Kriging_python,python,piston,unit cube,rmse,0.0104258 +high-dimension-normalize-screening_Kriging_python,python,piston,unit cube,q2,0.994462 +high-dimension-normalize-screening_Kriging_python,python,piston,unit cube,nlpd,-3.49038 +high-dimension-normalize-screening_Kriging_python,python,piston,native + normalize,rmse,0.0104258 +high-dimension-normalize-screening_Kriging_python,python,piston,native + normalize,q2,0.994462 +high-dimension-normalize-screening_Kriging_python,python,piston,native + normalize,nlpd,-3.49038 +high-dimension-normalize-screening_Kriging_python,python,borehole,unit cube,fit_s,0.0214174 +high-dimension-normalize-screening_Kriging_python,python,borehole,unit cube,rmse,0.893341 +high-dimension-normalize-screening_Kriging_python,python,borehole,unit cube,q2,0.999599 +high-dimension-normalize-screening_Kriging_python,python,borehole,unit cube,nlpd,1.08433 +high-dimension-normalize-screening_Kriging_python,python,borehole,native + normalize,rmse,0.893341 +high-dimension-normalize-screening_Kriging_python,python,borehole,native + normalize,q2,0.999599 +high-dimension-normalize-screening_Kriging_python,python,borehole,native + normalize,nlpd,1.08433 +high-dimension-normalize-screening_Kriging_python,python,robot_arm,unit cube,fit_s,0.187081 +high-dimension-normalize-screening_Kriging_python,python,robot_arm,unit cube,rmse,0.358507 +high-dimension-normalize-screening_Kriging_python,python,robot_arm,unit cube,q2,0.571154 +high-dimension-normalize-screening_Kriging_python,python,robot_arm,unit cube,nlpd,0.355106 +high-dimension-normalize-screening_Kriging_python,python,robot_arm,native + normalize,rmse,0.358507 +high-dimension-normalize-screening_Kriging_python,python,robot_arm,native + normalize,q2,0.571154 +high-dimension-normalize-screening_Kriging_python,python,robot_arm,native + normalize,nlpd,0.355106 +high-dimension-normalize-screening_Kriging_python,python,dette_pepelyshev,unit cube,fit_s,0.0181198 +high-dimension-normalize-screening_Kriging_python,python,dette_pepelyshev,unit cube,rmse,0.424908 +high-dimension-normalize-screening_Kriging_python,python,dette_pepelyshev,unit cube,q2,0.997757 +high-dimension-normalize-screening_Kriging_python,python,dette_pepelyshev,unit cube,nlpd,0.510276 +high-dimension-normalize-screening_Kriging_python,python,dette_pepelyshev,native + normalize,rmse,0.424908 +high-dimension-normalize-screening_Kriging_python,python,dette_pepelyshev,native + normalize,q2,0.997757 +high-dimension-normalize-screening_Kriging_python,python,dette_pepelyshev,native + normalize,nlpd,0.510276 +high-dimension-normalize-screening_Kriging_python,python,wing_weight,unit cube,fit_s,0.0303686 +high-dimension-normalize-screening_Kriging_python,python,wing_weight,unit cube,rmse,0.809743 +high-dimension-normalize-screening_Kriging_python,python,wing_weight,unit cube,q2,0.999716 +high-dimension-normalize-screening_Kriging_python,python,wing_weight,unit cube,nlpd,1.07161 +high-dimension-normalize-screening_Kriging_python,python,wing_weight,native + normalize,rmse,0.809743 +high-dimension-normalize-screening_Kriging_python,python,wing_weight,native + normalize,q2,0.999716 +high-dimension-normalize-screening_Kriging_python,python,wing_weight,native + normalize,nlpd,1.07161 +high-dimension-normalize-screening_Kriging_python,python,welch,unit cube,fit_s,0.500499 +high-dimension-normalize-screening_Kriging_python,python,welch,unit cube,rmse,0.167729 +high-dimension-normalize-screening_Kriging_python,python,welch,unit cube,q2,0.993452 +high-dimension-normalize-screening_Kriging_python,python,welch,unit cube,nlpd,-0.429138 +high-dimension-normalize-screening_Kriging_python,python,welch,native + normalize,rmse,0.167692 +high-dimension-normalize-screening_Kriging_python,python,welch,native + normalize,q2,0.993454 +high-dimension-normalize-screening_Kriging_python,python,welch,native + normalize,nlpd,-0.429291 +deep-kernel-projected-branin_MLPKriging_python,python,projected_branin_10d,Kriging,rmse,4.42284 +deep-kernel-projected-branin_MLPKriging_python,python,projected_branin_10d,Kriging,q2,0.959122 +deep-kernel-projected-branin_MLPKriging_python,python,projected_branin_10d,Kriging,nlpd,2.89997 +deep-kernel-projected-branin_MLPKriging_python,python,projected_branin_10d,MLPKriging tanh BFGS,fit_s,0.171914 +deep-kernel-projected-branin_MLPKriging_python,python,projected_branin_10d,MLPKriging tanh BFGS,rmse,59.5068 +deep-kernel-projected-branin_MLPKriging_python,python,projected_branin_10d,MLPKriging tanh BFGS,q2,-6.39973 +deep-kernel-projected-branin_MLPKriging_python,python,projected_branin_10d,MLPKriging tanh BFGS,nlpd,100.745 +deep-kernel-projected-branin_MLPKriging_python,python,projected_branin_10d,MLPKriging tanh BFGS10,fit_s,0.53444 +deep-kernel-projected-branin_MLPKriging_python,python,projected_branin_10d,MLPKriging tanh BFGS10,rmse,4.41161 +deep-kernel-projected-branin_MLPKriging_python,python,projected_branin_10d,MLPKriging tanh BFGS10,q2,0.95933 +deep-kernel-projected-branin_MLPKriging_python,python,projected_branin_10d,MLPKriging tanh BFGS10,nlpd,28.4236 +deep-kernel-projected-branin_MLPKriging_python,python,projected_branin_10d,MLPKriging selu BFGS,fit_s,0.145419 +deep-kernel-projected-branin_MLPKriging_python,python,projected_branin_10d,MLPKriging selu BFGS,rmse,58.9844 +deep-kernel-projected-branin_MLPKriging_python,python,projected_branin_10d,MLPKriging selu BFGS,q2,-6.27038 +deep-kernel-projected-branin_MLPKriging_python,python,projected_branin_10d,MLPKriging selu BFGS,nlpd,46.9347 +deep-kernel-projected-branin_MLPKriging_python,python,projected_branin_10d,MLPKriging selu BFGS10,fit_s,0.518803 +deep-kernel-projected-branin_MLPKriging_python,python,projected_branin_10d,MLPKriging selu BFGS10,rmse,3.79093 +deep-kernel-projected-branin_MLPKriging_python,python,projected_branin_10d,MLPKriging selu BFGS10,q2,0.969969 +deep-kernel-projected-branin_MLPKriging_python,python,projected_branin_10d,MLPKriging selu BFGS10,nlpd,12.6814 +large-n_Kriging-vecchia-nystrom_python,python,hartmann3/n=1500,exact,fit_s,11.2644 +aggregations-large-n_NestedKriging_python,python,hartmann3/n=1500,exact,fit_s,11.2644 +large-n_Kriging-vecchia-nystrom_python,python,hartmann3/n=1500,exact,rmse,0.000259233 +aggregations-large-n_NestedKriging_python,python,hartmann3/n=1500,exact,rmse,0.000259233 +large-n_Kriging-vecchia-nystrom_python,python,hartmann3/n=1500,exact,q2,1 +aggregations-large-n_NestedKriging_python,python,hartmann3/n=1500,exact,q2,1 +large-n_Kriging-vecchia-nystrom_python,python,hartmann3/n=1500,exact,nlpd,-6.92184 +aggregations-large-n_NestedKriging_python,python,hartmann3/n=1500,exact,nlpd,-6.92184 +aggregations-large-n_NestedKriging_python,python,hartmann3/n=1500,nested NK (20 groups),fit_s,0.324342 +aggregations-large-n_NestedKriging_python,python,hartmann3/n=1500,nested NK (20 groups),rmse,0.0029908 +aggregations-large-n_NestedKriging_python,python,hartmann3/n=1500,nested NK (20 groups),q2,0.999991 +aggregations-large-n_NestedKriging_python,python,hartmann3/n=1500,nested NK (20 groups),nlpd,-4.85946 +aggregations-large-n_NestedKriging_python,python,hartmann3/n=5000,nested NK (20 groups),fit_s,3.25297 +aggregations-large-n_NestedKriging_python,python,hartmann3/n=5000,nested NK (20 groups),rmse,0.000222655 +aggregations-large-n_NestedKriging_python,python,hartmann3/n=5000,nested NK (20 groups),q2,1 +aggregations-large-n_NestedKriging_python,python,hartmann3/n=5000,nested NK (20 groups),nlpd,-7.44453 +aggregations-large-n_NestedKriging_python,python,hartmann3/n=1500,nested rBCM (20 groups),fit_s,0.273419 +aggregations-large-n_NestedKriging_python,python,hartmann3/n=1500,nested rBCM (20 groups),rmse,0.00304431 +aggregations-large-n_NestedKriging_python,python,hartmann3/n=1500,nested rBCM (20 groups),q2,0.99999 +aggregations-large-n_NestedKriging_python,python,hartmann3/n=1500,nested rBCM (20 groups),nlpd,-4.50201 +aggregations-large-n_NestedKriging_python,python,hartmann3/n=5000,nested rBCM (20 groups),fit_s,2.67811 +aggregations-large-n_NestedKriging_python,python,hartmann3/n=5000,nested rBCM (20 groups),rmse,0.000224517 +aggregations-large-n_NestedKriging_python,python,hartmann3/n=5000,nested rBCM (20 groups),q2,1 +aggregations-large-n_NestedKriging_python,python,hartmann3/n=5000,nested rBCM (20 groups),nlpd,-7.18182 +large-n_Kriging-vecchia-nystrom_python,python,hartmann3/n=1500,subset of data (500),fit_s,0.685586 +large-n_Kriging-vecchia-nystrom_python,python,hartmann3/n=1500,subset of data (500),rmse,0.00269153 +large-n_Kriging-vecchia-nystrom_python,python,hartmann3/n=1500,subset of data (500),q2,0.999992 +large-n_Kriging-vecchia-nystrom_python,python,hartmann3/n=1500,subset of data (500),nlpd,-4.68641 +large-n_Kriging-vecchia-nystrom_python,python,hartmann3/n=5000,subset of data (500),fit_s,0.906366 +large-n_Kriging-vecchia-nystrom_python,python,hartmann3/n=5000,subset of data (500),rmse,0.00247552 +large-n_Kriging-vecchia-nystrom_python,python,hartmann3/n=5000,subset of data (500),q2,0.999994 +large-n_Kriging-vecchia-nystrom_python,python,hartmann3/n=5000,subset of data (500),nlpd,-4.71359 +large-n_Kriging-vecchia-nystrom_python,python,otl_circuit/n=1500,exact,fit_s,10.9405 +aggregations-large-n_NestedKriging_python,python,otl_circuit/n=1500,exact,fit_s,10.9405 +large-n_Kriging-vecchia-nystrom_python,python,otl_circuit/n=1500,exact,rmse,9.81598e-05 +aggregations-large-n_NestedKriging_python,python,otl_circuit/n=1500,exact,rmse,9.81598e-05 +large-n_Kriging-vecchia-nystrom_python,python,otl_circuit/n=1500,exact,q2,1 +aggregations-large-n_NestedKriging_python,python,otl_circuit/n=1500,exact,q2,1 +large-n_Kriging-vecchia-nystrom_python,python,otl_circuit/n=1500,exact,nlpd,-8.21962 +aggregations-large-n_NestedKriging_python,python,otl_circuit/n=1500,exact,nlpd,-8.21962 +aggregations-large-n_NestedKriging_python,python,otl_circuit/n=1500,nested NK (20 groups),fit_s,0.279359 +aggregations-large-n_NestedKriging_python,python,otl_circuit/n=1500,nested NK (20 groups),rmse,0.00226873 +aggregations-large-n_NestedKriging_python,python,otl_circuit/n=1500,nested NK (20 groups),q2,0.999996 +aggregations-large-n_NestedKriging_python,python,otl_circuit/n=1500,nested NK (20 groups),nlpd,-4.90542 +aggregations-large-n_NestedKriging_python,python,otl_circuit/n=5000,nested NK (20 groups),fit_s,3.08712 +aggregations-large-n_NestedKriging_python,python,otl_circuit/n=5000,nested NK (20 groups),rmse,0.000261686 +aggregations-large-n_NestedKriging_python,python,otl_circuit/n=5000,nested NK (20 groups),q2,1 +aggregations-large-n_NestedKriging_python,python,otl_circuit/n=5000,nested NK (20 groups),nlpd,-7.20623 +aggregations-large-n_NestedKriging_python,python,otl_circuit/n=1500,nested rBCM (20 groups),fit_s,0.251989 +aggregations-large-n_NestedKriging_python,python,otl_circuit/n=1500,nested rBCM (20 groups),rmse,0.00291869 +aggregations-large-n_NestedKriging_python,python,otl_circuit/n=1500,nested rBCM (20 groups),q2,0.999993 +aggregations-large-n_NestedKriging_python,python,otl_circuit/n=1500,nested rBCM (20 groups),nlpd,-2.55512 +aggregations-large-n_NestedKriging_python,python,otl_circuit/n=5000,nested rBCM (20 groups),fit_s,3.67768 +aggregations-large-n_NestedKriging_python,python,otl_circuit/n=5000,nested rBCM (20 groups),rmse,0.000278149 +aggregations-large-n_NestedKriging_python,python,otl_circuit/n=5000,nested rBCM (20 groups),q2,1 +aggregations-large-n_NestedKriging_python,python,otl_circuit/n=5000,nested rBCM (20 groups),nlpd,-4.34927 +large-n_Kriging-vecchia-nystrom_python,python,otl_circuit/n=1500,subset of data (500),fit_s,0.709236 +large-n_Kriging-vecchia-nystrom_python,python,otl_circuit/n=1500,subset of data (500),rmse,0.0004286 +large-n_Kriging-vecchia-nystrom_python,python,otl_circuit/n=1500,subset of data (500),q2,1 +large-n_Kriging-vecchia-nystrom_python,python,otl_circuit/n=1500,subset of data (500),nlpd,-6.72678 +large-n_Kriging-vecchia-nystrom_python,python,otl_circuit/n=5000,subset of data (500),fit_s,0.905534 +large-n_Kriging-vecchia-nystrom_python,python,otl_circuit/n=5000,subset of data (500),rmse,0.000446879 +large-n_Kriging-vecchia-nystrom_python,python,otl_circuit/n=5000,subset of data (500),q2,1 +large-n_Kriging-vecchia-nystrom_python,python,otl_circuit/n=5000,subset of data (500),nlpd,-6.78844 +aggregations-large-n_NestedKriging_python,python,branin/nested,NK,rmse,0.0343984 +aggregations-large-n_NestedKriging_python,python,branin/nested,NK,q2,1 +aggregations-large-n_NestedKriging_python,python,branin/nested,NK,nlpd,2.00083e+14 +aggregations-large-n_NestedKriging_python,python,branin/nested,PoE,rmse,0.0354265 +aggregations-large-n_NestedKriging_python,python,branin/nested,PoE,q2,1 +aggregations-large-n_NestedKriging_python,python,branin/nested,PoE,nlpd,-1.64533 +aggregations-large-n_NestedKriging_python,python,branin/nested,gPoE,rmse,0.0354265 +aggregations-large-n_NestedKriging_python,python,branin/nested,gPoE,q2,1 +aggregations-large-n_NestedKriging_python,python,branin/nested,gPoE,nlpd,-0.621836 +aggregations-large-n_NestedKriging_python,python,branin/nested,BCM,rmse,0.0354196 +aggregations-large-n_NestedKriging_python,python,branin/nested,BCM,q2,1 +aggregations-large-n_NestedKriging_python,python,branin/nested,BCM,nlpd,-1.64533 +aggregations-large-n_NestedKriging_python,python,branin/nested,rBCM,rmse,0.035287 +aggregations-large-n_NestedKriging_python,python,branin/nested,rBCM,q2,1 +aggregations-large-n_NestedKriging_python,python,branin/nested,rBCM,nlpd,-2.61369 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,branin,pylibkriging,q2,0.999944 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,branin,pylibkriging,nlpd,0.137225 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,branin,pylibkriging,fit_s,0.00151396 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,branin,pylibkriging BFGS10,q2,0.999945 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,branin,pylibkriging BFGS10,nlpd,0.137254 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,branin,pylibkriging BFGS10,fit_s,0.0161211 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,branin,scikit-learn,q2,0.999597 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,branin,scikit-learn,nlpd,0.928508 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,branin,scikit-learn,fit_s,0.147773 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,branin,SMT,q2,0.999944 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,branin,SMT,nlpd,0.137259 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,branin,SMT,fit_s,0.792991 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,branin,OpenTURNS,q2,0.998221 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,branin,OpenTURNS,nlpd,1.55599 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,branin,OpenTURNS,fit_s,0.00512815 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,six_hump_camel,pylibkriging,q2,0.999418 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,six_hump_camel,pylibkriging,nlpd,0.0429346 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,six_hump_camel,pylibkriging,fit_s,0.00358009 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,six_hump_camel,pylibkriging BFGS10,q2,0.999418 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,six_hump_camel,pylibkriging BFGS10,nlpd,0.0429347 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,six_hump_camel,pylibkriging BFGS10,fit_s,0.0500369 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,six_hump_camel,scikit-learn,q2,0.0163311 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,six_hump_camel,scikit-learn,nlpd,4.63763 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,six_hump_camel,scikit-learn,fit_s,0.156482 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,six_hump_camel,SMT,q2,0.999594 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,six_hump_camel,SMT,nlpd,-0.121007 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,six_hump_camel,SMT,fit_s,0.941661 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,six_hump_camel,OpenTURNS,q2,0.956713 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,six_hump_camel,OpenTURNS,nlpd,2.55446 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,six_hump_camel,OpenTURNS,fit_s,0.00930476 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann3,pylibkriging,q2,0.986148 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann3,pylibkriging,nlpd,-0.847247 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann3,pylibkriging,fit_s,0.023098 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann3,pylibkriging BFGS10,q2,0.986148 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann3,pylibkriging BFGS10,nlpd,-0.847249 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann3,pylibkriging BFGS10,fit_s,0.113887 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann3,scikit-learn,q2,0.983889 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann3,scikit-learn,nlpd,-0.782697 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann3,scikit-learn,fit_s,0.208279 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann3,SMT,q2,0.986148 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann3,SMT,nlpd,-0.847249 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann3,SMT,fit_s,1.35887 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann3,OpenTURNS,q2,0.984712 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann3,OpenTURNS,nlpd,-0.809851 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann3,OpenTURNS,fit_s,0.0169773 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,ishigami,pylibkriging,q2,0.259556 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,ishigami,pylibkriging,nlpd,4.38991 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,ishigami,pylibkriging,fit_s,0.0264924 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,ishigami,pylibkriging BFGS10,q2,0.935722 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,ishigami,pylibkriging BFGS10,nlpd,0.857443 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,ishigami,pylibkriging BFGS10,fit_s,0.322963 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,ishigami,scikit-learn,q2,0.831142 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,ishigami,scikit-learn,nlpd,1.5455 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,ishigami,scikit-learn,fit_s,0.512728 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,ishigami,SMT,q2,0.51196 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,ishigami,SMT,nlpd,3.31349 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,ishigami,SMT,fit_s,2.70852 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,ishigami,OpenTURNS,q2,0.887251 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,ishigami,OpenTURNS,nlpd,1.39348 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,ishigami,OpenTURNS,fit_s,0.0510182 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,friedman,pylibkriging,q2,0.999771 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,friedman,pylibkriging,nlpd,-1.29726 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,friedman,pylibkriging,fit_s,0.0165708 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,friedman,pylibkriging BFGS10,q2,0.999771 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,friedman,pylibkriging BFGS10,nlpd,-1.29727 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,friedman,pylibkriging BFGS10,fit_s,0.279774 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,friedman,scikit-learn,q2,0.999754 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,friedman,scikit-learn,nlpd,-1.37702 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,friedman,scikit-learn,fit_s,0.308127 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,friedman,SMT,q2,0.999899 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,friedman,SMT,nlpd,-1.871 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,friedman,SMT,fit_s,4.71948 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,friedman,OpenTURNS,q2,0.990862 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,friedman,OpenTURNS,nlpd,0.469828 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,friedman,OpenTURNS,fit_s,0.0511642 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,otl_circuit,pylibkriging,q2,0.999922 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,otl_circuit,pylibkriging,nlpd,-3.46228 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,otl_circuit,pylibkriging,fit_s,0.0136361 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,otl_circuit,pylibkriging BFGS10,q2,0.999922 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,otl_circuit,pylibkriging BFGS10,nlpd,-3.46227 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,otl_circuit,pylibkriging BFGS10,fit_s,0.168652 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,otl_circuit,scikit-learn,q2,0.999906 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,otl_circuit,scikit-learn,nlpd,-3.43062 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,otl_circuit,scikit-learn,fit_s,0.64982 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,otl_circuit,SMT,q2,0.999954 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,otl_circuit,SMT,nlpd,-3.35016 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,otl_circuit,SMT,fit_s,6.837 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,otl_circuit,OpenTURNS,q2,0.99777 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,otl_circuit,OpenTURNS,nlpd,-1.61723 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,otl_circuit,OpenTURNS,fit_s,0.0662544 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,piston,pylibkriging,q2,0.997498 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,piston,pylibkriging,nlpd,-3.80667 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,piston,pylibkriging,fit_s,0.03055 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,piston,pylibkriging BFGS10,q2,0.997498 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,piston,pylibkriging BFGS10,nlpd,-3.8067 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,piston,pylibkriging BFGS10,fit_s,0.326413 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,piston,scikit-learn,q2,0.998684 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,piston,scikit-learn,nlpd,-4.10497 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,piston,scikit-learn,fit_s,0.856336 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,piston,SMT,q2,0.997511 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,piston,SMT,nlpd,-3.86618 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,piston,SMT,fit_s,12.0067 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,piston,OpenTURNS,q2,0.990631 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,piston,OpenTURNS,nlpd,-2.9906 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,piston,OpenTURNS,fit_s,0.137117 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,borehole,pylibkriging,q2,0.999694 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,borehole,pylibkriging,nlpd,1.00118 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,borehole,pylibkriging,fit_s,0.0166521 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,borehole,pylibkriging BFGS10,q2,0.999694 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,borehole,pylibkriging BFGS10,nlpd,1.00121 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,borehole,pylibkriging BFGS10,fit_s,0.200685 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,borehole,scikit-learn,q2,0.999929 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,borehole,scikit-learn,nlpd,0.194455 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,borehole,scikit-learn,fit_s,1.18352 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,borehole,SMT,q2,0.999925 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,borehole,SMT,nlpd,0.279122 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,borehole,SMT,fit_s,12.3314 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,borehole,OpenTURNS,q2,0.996206 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,borehole,OpenTURNS,nlpd,2.55431 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,borehole,OpenTURNS,fit_s,0.121907 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,wing_weight,pylibkriging,q2,0.99972 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,wing_weight,pylibkriging,nlpd,1.11517 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,wing_weight,pylibkriging,fit_s,0.0297599 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,wing_weight,pylibkriging BFGS10,q2,0.99972 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,wing_weight,pylibkriging BFGS10,nlpd,1.11512 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,wing_weight,pylibkriging BFGS10,fit_s,0.335854 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,wing_weight,scikit-learn,q2,0.998048 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,wing_weight,scikit-learn,nlpd,2.57304 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,wing_weight,scikit-learn,fit_s,0.833252 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,wing_weight,SMT,q2,0.999898 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,wing_weight,SMT,nlpd,0.604023 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,wing_weight,SMT,fit_s,26.7889 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,wing_weight,OpenTURNS,q2,0.993145 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,wing_weight,OpenTURNS,nlpd,2.83627 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,wing_weight,OpenTURNS,fit_s,0.253929 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann6,pylibkriging,q2,0.422939 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann6,pylibkriging,nlpd,0.0965158 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann6,pylibkriging,fit_s,0.068651 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann6,pylibkriging BFGS10,q2,0.72352 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann6,pylibkriging BFGS10,nlpd,-0.156898 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann6,pylibkriging BFGS10,fit_s,0.877097 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann6,scikit-learn,q2,0.69049 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann6,scikit-learn,nlpd,-0.137845 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann6,scikit-learn,fit_s,1.43153 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann6,SMT,q2,0.708463 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann6,SMT,nlpd,-0.0177081 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann6,SMT,fit_s,11.2226 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann6,OpenTURNS,q2,0.696177 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann6,OpenTURNS,nlpd,-0.132838 +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,python,hartmann6,OpenTURNS,fit_s,0.141383 diff --git a/tools/install/julia.sh b/tools/install/julia.sh new file mode 100755 index 0000000..01bd170 --- /dev/null +++ b/tools/install/julia.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# Install JLibKriging.jl (compiled against a libKriging checkout) and the Julia Jupyter +# kernel into the active Julia environment. +# tools/install/julia.sh LIBKRIGING_SRC [KERNEL_NAME] (default kernel name: julia-1.11, +# the one of the notebooks) +# JLibKriging.jl's build (deps/build.jl) compiles the checkout given by +# LIBKRIGING_SRC_DIR and generates the module from its bindings/Julia/jlibkriging. +# Already installed (e.g. a restored ~/.julia depot): only the kernel is (re)installed, +# and LIBKRIGING_SRC need not exist. +set -euo pipefail +kernel=${2:-julia-1.11} +python -m pip install nbclient nbformat ipykernel jupyter_client +if ! julia -e 'using JLibKriging, IJulia, DelimitedFiles' 2> /dev/null; then + LIBKRIGING_SRC_DIR=$(cd "$1" && pwd); export LIBKRIGING_SRC_DIR + julia -e 'using Pkg + Pkg.add(url="https://github.com/libKriging/JLibKriging.jl") + Pkg.add(["IJulia", "DelimitedFiles"])' +fi +julia -e "using IJulia; installkernel(\"Julia\"; specname=\"$kernel\") + using JLibKriging; println(\"JLibKriging OK\")" diff --git a/tools/install/octave.sh b/tools/install/octave.sh new file mode 100755 index 0000000..9155b4a --- /dev/null +++ b/tools/install/octave.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +# Build mLibKriging (the Octave binding) from a libKriging checkout into OUT_DIR, which +# then goes on OCTAVE_PATH; install the Octave Jupyter kernel. +# tools/install/octave.sh LIBKRIGING_SRC OUT_DIR +# Built from source for both channels (master: libKriging master; release: the last +# release tag): the published binaries target another Octave version than Ubuntu's. +# Needs octave, octave-dev, liblapack-dev, libopenblas-dev. +set -euo pipefail +src=$(cd "$1" && pwd); out=$(mkdir -p "$2" && cd "$2" && pwd) +if [ ! -f "$out/mLibKriging.mex" ]; then + cmake -S "$src" -B "$src/build-octave" -DCMAKE_BUILD_TYPE=Release -DENABLE_OCTAVE_BINDING=on \ + -DENABLE_PYTHON_BINDING=off -DENABLE_MATLAB_BINDING=off -DBUILD_SHARED_LIBS=off -DSTATIC_LIB=on + cmake --build "$src/build-octave" --target mLibKriging -j "$(nproc)" + cp "$src/build-octave/bindings/Octave/mlibkriging/mLibKriging.mex" "$src"/bindings/Octave/mlibkriging/*.m "$out/" +fi +python -m pip install nbclient nbformat ipykernel octave_kernel +OCTAVE_PATH="$out" octave --no-gui --eval 'k = Kriging([0; 1; 0.5], [0; 1; 0.5], "gauss"); printf("mLibKriging OK (theta %g)\n", k.theta())' diff --git a/tools/install/python.sh b/tools/install/python.sh new file mode 100755 index 0000000..8dc3cac --- /dev/null +++ b/tools/install/python.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# Install pylibkriging, then what the notebooks need besides it (tools/python/requirements.txt; +# their helpers, tools/python/gallery.py, are loaded from the repository, not installed). +# tools/install/python.sh wheel LIBKRIGING_SRC OUT_DIR only build a wheel from a libKriging checkout +# tools/install/python.sh master WHEEL_DIR|LIBKRIGING_SRC install the wheel of WHEEL_DIR, or build it first +# tools/install/python.sh release the last release, from PyPI +# The wheel needs BLAS/LAPACK headers (Ubuntu: liblapack-dev libopenblas-dev). +set -euo pipefail +tools=$(cd "$(dirname "$0")/.." && pwd) +build_wheel() { # SRC OUT (absolute paths: pip reads a bare "libKriging" as a package name) + set -- "$(cd "$1" && pwd)" "$(mkdir -p "$2" && cd "$2" && pwd)" + python -m pip install --upgrade pip + python -m pip install -r "$1/bindings/Python/pylibkriging/requirements.txt" \ + -r "$1/bindings/Python/pylibkriging/dev-requirements.txt" cmake + CMAKE_BUILD_PARALLEL_LEVEL=${CMAKE_BUILD_PARALLEL_LEVEL:-$(nproc)} \ + python -m pip wheel --no-build-isolation --no-deps -w "$2" "$1" +} +case "$1" in + wheel) build_wheel "$2" "$3"; exit 0 ;; + master) + if ! ls "$2"/pylibkriging-*.whl > /dev/null 2>&1; then + build_wheel "$2" "$2/dist"; set -- "$1" "$2/dist" + fi + python -m pip install "$2"/pylibkriging-*.whl ;; + release) python -m pip install pylibkriging ;; + *) echo "usage: $0 wheel SRC OUT | master WHEEL_DIR|SRC | release" >&2; exit 2 ;; +esac +python -m pip install -r "$tools/python/requirements.txt" +python -c "import pylibkriging as lk; print('pylibkriging', lk.__version__, 'from', lk.__file__)" diff --git a/tools/install/r.sh b/tools/install/r.sh new file mode 100755 index 0000000..0416034 --- /dev/null +++ b/tools/install/r.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +# Install rlibkriging into the library directory LIB (added to R_LIBS_USER by the caller), +# and the R Jupyter kernel (IRkernel). +# tools/install/r.sh master LIB LIBKRIGING_SRC build from a libKriging checkout (C++ core + bindings/R) +# tools/install/r.sh release LIB the last release, from CRAN +# tools/install/r.sh kernel LIB only the R Jupyter kernel (rlibkriging already in LIB) +# The master build follows libKriging's tools/r-linux-macos/build.sh: the core is compiled +# with R's own compilers as a static library, then bindings/R is built against it. +set -euo pipefail +channel=$1 lib=$(mkdir -p "$2" && cd "$2" && pwd) +export R_LIBS_USER="$lib" +if [ "$channel" = kernel ]; then + : +elif [ "$channel" = release ]; then + Rscript -e 'install.packages("rlibkriging", lib = Sys.getenv("R_LIBS_USER"))' +else + src=$(cd "$3" && pwd) + # Rcpp & co. (bindings/R/rlibkriging/install/install.R would also install test-only packages) + Rscript -e 'for (p in c("Rcpp", "RcppArmadillo", "RhpcBLASctl")) + if (!requireNamespace(p, quietly = TRUE)) install.packages(p, lib = Sys.getenv("R_LIBS_USER"))' + CC="$(R CMD config CC)" CXX="$(R CMD config CXX)" cmake -S "$src" -B "$src/build-r" -G "Unix Makefiles" \ + -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_LIBDIR=lib -DBUILD_SHARED_LIBS=off -DSTATIC_LIB=on \ + -DENABLE_OCTAVE_BINDING=off -DENABLE_MATLAB_BINDING=off -DENABLE_PYTHON_BINDING=off + cmake --build "$src/build-r" --target install.lib -j "$(nproc)" + export LIBKRIGING_PATH="$src/build-r/installed" + pushd "$src/bindings/R" > /dev/null + mkdir -p Rlibs && touch Rlibs/.prepared # dependencies already installed above + R_LIBS="$lib" Rscript -e "Rcpp::compileAttributes(pkgdir = 'rlibkriging')" + R_LIBS="$lib" MAKE_SHARED_LIBS=off STATIC_LIB=on MAKEFLAGS="-j$(nproc)" R CMD INSTALL --no-multiarch -l "$lib" rlibkriging + popd > /dev/null +fi +python -m pip install nbclient nbformat ipykernel jupyter_client +# The kernelspec is written directly, as IRkernel::installspec() would, without its check of +# the `jupyter kernelspec` command (which fails on CI runners). +# shellcheck disable=SC2016 # R code, not shell expansions +Rscript -e 'if (!requireNamespace("IRkernel", quietly = TRUE)) install.packages("IRkernel", lib = Sys.getenv("R_LIBS_USER")) + dest <- file.path(Sys.getenv("JUPYTER_DATA_DIR", file.path(Sys.getenv("HOME"), ".local/share/jupyter")), "kernels", "ir") + dir.create(dest, recursive = TRUE, showWarnings = FALSE) + spec <- system.file("kernelspec", package = "IRkernel") + file.copy(list.files(spec, full.names = TRUE), dest, overwrite = TRUE) + k <- jsonlite::fromJSON(file.path(dest, "kernel.json")) + k$argv[1] <- file.path(R.home("bin"), "R") + jsonlite::write_json(k, file.path(dest, "kernel.json"), auto_unbox = TRUE, pretty = TRUE) + cat("R kernel in", dest, "\n") + cat("rlibkriging", as.character(packageVersion("rlibkriging")), "in", find.package("rlibkriging"), "\n")' diff --git a/tools/julia/gallery.jl b/tools/julia/gallery.jl new file mode 100644 index 0000000..7412e26 --- /dev/null +++ b/tools/julia/gallery.jl @@ -0,0 +1,136 @@ +# Helpers of the libKriging gallery notebooks (Julia), in one file: include("tools/julia/gallery.jl"). +# +# The same functions, under the same names, as tools/python/gallery.py (where they are +# methods of Suite: suite.check_close(...) is check_close!(s, ...) here), and the same CSV +# outputs (results/.checks.csv, .metrics.csv), aggregated with the other +# languages by tools/report.py. +# +# s = Suite(name) a suite of checks, named after the notebook +# check!(s, check, ok; value, expected, detail, severity) +# check_true!, check_less!, check_greater!, check_between!, check_close!, check_finite!, +# check_runs!(f, s, check) (do-block), check_skip! the usual checks +# metric!(s, case, package; name=value, ...) record metrics +# suite_summary(s) end-of-notebook summary +# suite_finalize!(s) write the CSV files, error if an error-level check failed +# read_data(name, stem), rmse, q2 +# +# Paths are relative to the repository root ($LKGALLERY_ROOT, else the working +# directory, where the notebooks are executed). +using DelimitedFiles, Printf + +const GALLERY_ROOT = get(ENV, "LKGALLERY_ROOT", ".") +const RESULTS_DIR = get(ENV, "LKGALLERY_RESULTS", joinpath(GALLERY_ROOT, "results")) +mkpath(RESULTS_DIR) + +# ---- suite and checks ------------------------------------------------------------------ + +function read_known() + f = joinpath(GALLERY_ROOT, "tools", "known_issues.csv") + isfile(f) || error("run the notebooks from the repository root ($f not found)") + rows = readdlm(f, ',', String; header=true, quotes=true)[1] # quoted fields may hold commas + Dict{Tuple{String,String},String}((rows[i, 1], rows[i, 2]) => rows[i, 3] for i in 1:size(rows, 1)) +end + +mutable struct Suite + name::String + checks::Vector{Vector{String}} + metrics::Vector{Vector{String}} + t0::Float64 + known::Dict{Tuple{String,String},String} +end +Suite(name) = Suite(name, [], [], time(), read_known()) + +fmt(v::Real) = @sprintf("%.6g", v) +fmt(v) = string(v) + +function check!(s::Suite, check, ok; value="", expected="", detail="", severity="error") + status = ok ? "PASS" : (severity == "error" ? "FAIL" : "WARN") + if !ok && haskey(s.known, (s.name, check)) + status = "KNOWN"; detail = s.known[(s.name, check)] * " | " * detail + end + ok || println("[$status] $check: value=$(fmt(value)) expected $expected $detail") + push!(s.checks, [s.name, "julia", check, severity, status, fmt(value), expected, detail]) + ok +end + +check_true!(s::Suite, check, cond; detail="", severity="error") = + check!(s, check, cond === true; value=string(cond), expected="true", detail, severity) +check_less!(s::Suite, check, value, bound; detail="", severity="error") = + check!(s, check, isfinite(value) && value < bound; value, expected="< $bound", detail, severity) +check_greater!(s::Suite, check, value, bound; detail="", severity="error") = + check!(s, check, isfinite(value) && value > bound; value, expected="> $bound", detail, severity) +check_between!(s::Suite, check, value, lo, hi; detail="", severity="error") = + check!(s, check, isfinite(value) && lo <= value <= hi; value, expected="in [$lo, $hi]", detail, severity) + +function check_close!(s::Suite, check, a, b; rtol=1e-6, atol=0.0, detail="", severity="error") + a = Float64.(vec(collect(a))); b = Float64.(vec(collect(b))) # vec first: Float64.() of a 0-dim array is a scalar + length(a) == length(b) || return check!(s, check, false; value="length $(length(a))", + expected="length $(length(b))", detail, severity) + gap = abs.(a .- b) + ok = all(isfinite, a) && all(gap .<= atol .+ rtol .* abs.(b)) + rel = maximum(gap ./ max.(abs.(b), 1e-300)) + check!(s, check, ok; value=rel, expected="rtol $rtol, atol $atol", detail, severity) +end + +check_finite!(s::Suite, check, a; detail="", severity="error") = + check!(s, check, !isempty(a) && all(isfinite, a); value="$(count(!isfinite, a)) non-finite", + expected="all finite", detail, severity) + +function check_runs!(f, s::Suite, check; severity="error") + # check_runs!(s, "fit") do ... end: an error becomes a failed check instead of stopping the notebook + try + f() + check!(s, check, true; value="ok", expected="no error", severity) + catch e + check!(s, check, false; value=string(typeof(e)), expected="no error", + detail=first(sprint(showerror, e), 200), severity) + end +end + +function check_skip!(s::Suite, check, reason) + push!(s.checks, [s.name, "julia", check, "info", "SKIP", "", "", reason]) + println("[SKIP] $check: $reason") +end + +# ---- metrics and results --------------------------------------------------------------- + +metric!(s::Suite, case, package; kw...) = + for (k, v) in kw; push!(s.metrics, [s.name, "julia", case, package, string(k), fmt(v)]); end + +function suite_summary(s::Suite) + for r in s.metrics + println(rpad(r[3], 28), rpad(r[4], 28), rpad(r[5], 12), r[6]) + end + st = [c[5] for c in s.checks] + println(length(st), " checks: ", join(["$k $(count(==(k), st))" for k in ("PASS", "FAIL", "WARN", "KNOWN", "SKIP")], ", ")) + for c in s.checks + c[5] == "PASS" || println(rpad(c[5], 7), rpad(c[4], 7), c[3], ": value ", c[6], ", expected ", c[7], " ", c[8]) + end +end + +csvq(x) = "\"" * replace(string(x), "\"" => "\"\"") * "\"" + +function write_rows(f, header, rows) + open(f, "w") do io + println(io, join(header, ",")) + for r in rows; println(io, join(csvq.(r), ",")); end + end +end + +function suite_finalize!(s::Suite) + metric!(s, "_suite", "_"; elapsed_s=time() - s.t0) + write_rows(joinpath(RESULTS_DIR, s.name * ".checks.csv"), + ["suite", "lang", "check", "severity", "status", "value", "expected", "detail"], s.checks) + write_rows(joinpath(RESULTS_DIR, s.name * ".metrics.csv"), + ["suite", "lang", "case", "package", "metric", "value"], s.metrics) + st = [c[5] for c in s.checks] + println(s.name, ": ", join(["$k $(count(==(k), st))" for k in ("PASS", "FAIL", "WARN", "KNOWN", "SKIP")], ", ")) + failed = [c[3] for c in s.checks if c[5] == "FAIL"] + isempty(failed) || error("$(length(failed)) check(s) failed in $(s.name): " * join(first(failed, 20), "; ")) +end + +# ---- data and metrics ------------------------------------------------------------------ + +read_data(name, stem) = readdlm(joinpath(GALLERY_ROOT, "data", "$(name)_$(stem).csv"), ',', Float64) +rmse(y, m) = sqrt(sum((y .- m) .^ 2) / length(y)) +q2(y, m) = 1 - sum((y .- m) .^ 2) / sum((y .- sum(y) / length(y)) .^ 2) diff --git a/tools/known_issues.csv b/tools/known_issues.csv new file mode 100644 index 0000000..6829036 --- /dev/null +++ b/tools/known_issues.csv @@ -0,0 +1,11 @@ +suite,check,issue +large-n_Kriging-vecchia-nystrom_python,vecchia(n-1) LL == exact LL (same theta),With m >= n-1 neighbours the Vecchia likelihood should be the exact one; it matches for n=20 but not for n=40 (-111.349 vs -110.834) +large-n_Kriging-vecchia-nystrom_python,hartmann3/n=1500/nystrom(100): Q2,Nystrom predictor degrades as n grows at fixed k (theta estimates are fine: the exact predictor at the Nystrom theta has Q2 ~ 1) +large-n_Kriging-vecchia-nystrom_python,hartmann3/n=5000/nystrom(100): Q2,Nystrom predictor degrades as n grows at fixed k (theta estimates are fine: the exact predictor at the Nystrom theta has Q2 ~ 1) +large-n_Kriging-vecchia-nystrom_python,otl_circuit/n=1500/nystrom(100): Q2,Nystrom predictor degrades as n grows at fixed k (theta estimates are fine: the exact predictor at the Nystrom theta has Q2 ~ 1) +large-n_Kriging-vecchia-nystrom_python,otl_circuit/n=5000/nystrom(100): Q2,Nystrom predictor degrades as n grows at fixed k (theta estimates are fine: the exact predictor at the Nystrom theta has Q2 ~ 1) +robustness-invalid-inputs_Kriging_python,NaN in y is rejected,Kriging accepts NaN in y without raising +robustness-invalid-inputs_Kriging_python,Inf in y is rejected,Kriging accepts Inf in y without raising +robustness-invalid-inputs_Kriging_python,WarpKriging with 1 spec for 2 columns is rejected,WarpKriging accepts fewer warping specs than columns of X +robustness-invalid-inputs_Kriging_python,predict(stdev) time growth exponent in q (8000 vs 4000),"predict(return_stdev=True, return_cov=False) is O(q^2) in time and memory (10 000 points: 1.6 GB; 30 000: out of memory)" +vs-sklearn-SMT-OpenTURNS-GPy_Kriging_python,sklearn RBF == libKriging gauss: simple kriging stdev (beta fixed),With beta fixed (is_beta_estim=False) predict still adds the trend-estimation variance (universal kriging) instead of the simple kriging variance diff --git a/tools/make_data.py b/tools/make_data.py new file mode 100644 index 0000000..a346669 --- /dev/null +++ b/tools/make_data.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python3 +"""Write the shared datasets of data/ (committed): the cross-binding notebooks of every +language read these exact files, so that all bindings fit the very same points. + +Plain CSV, no header, full precision; X is n x d (rows = observations), y is a column. +Regenerating them is only needed when a dataset is added: the files are the reference, +not the script (a new scipy could draw other LHS points).""" +import sys +from pathlib import Path + +import numpy as np + +sys.path.insert(0, str(Path(__file__).resolve().parent / "python")) +from gallery import TESTS, design # noqa: E402 (tools/python/gallery.py) + +DATA = Path(__file__).resolve().parents[1] / "data" +DATASETS = [("branin", 25), ("hartmann3", 40), ("borehole", 80)] + + +def main(): + DATA.mkdir(exist_ok=True) + for name, n in DATASETS: + X, y, Xt, yt = design.dataset(TESTS[name], n, n_test=200, seed=2024) + for stem, arr in [("X", X), ("y", y), ("Xt", Xt), ("yt", yt)]: + np.savetxt(DATA / f"{name}_{stem}.csv", arr.reshape(len(arr), -1), delimiter=",", fmt="%.17g") + print(f"data/{name}_*.csv: n={n}, d={X.shape[1]}, n_test={len(Xt)}") + + +if __name__ == "__main__": + main() diff --git a/tools/notebook_setup.py b/tools/notebook_setup.py new file mode 100644 index 0000000..a81e4f1 --- /dev/null +++ b/tools/notebook_setup.py @@ -0,0 +1,26 @@ +# Setup for running this notebook outside the repository (Colab, Binder, a lone copy): +# nothing to do in the repository itself (CI, local runs). Otherwise the repository +# (datasets of data/, helpers of tools/python/gallery.py) is cloned and the released pylibkriging installed. +import importlib +import os +import subprocess +import sys +from pathlib import Path + +if not Path("tools/known_issues.csv").is_file(): + if not Path("gallery/tools/known_issues.csv").is_file(): + subprocess.run(["git", "clone", "--depth", "1", "https://github.com/libKriging/gallery"], check=True) + os.chdir("gallery") + try: + import pylibkriging # noqa: F401 + except ImportError: + r = subprocess.run([sys.executable, "-m", "pip", "install", "--quiet", "pylibkriging"], + capture_output=True, text=True) + if r.returncode != 0: + raise RuntimeError( + f"pylibkriging could not be installed for Python {sys.version_info.major}.{sys.version_info.minor} " + "(PyPI has wheels for CPython 3.10 to 3.12 only). On Colab: Runtime > Change runtime version > " + "2026.07 (Python 3.12), then run again.\n" + r.stderr[-500:]) + importlib.invalidate_caches() +sys.path.insert(0, str(Path("tools/python").resolve())) # for "import gallery" (tools/python/gallery.py) +print("working directory:", os.getcwd()) diff --git a/tools/octave/gallery.m b/tools/octave/gallery.m new file mode 100644 index 0000000..337ab63 --- /dev/null +++ b/tools/octave/gallery.m @@ -0,0 +1,190 @@ +% Helpers of the libKriging gallery notebooks (Octave), in one script of functions: +% source("tools/octave/gallery.m") (the leading "1;" makes it a script, so that it can +% define several functions). +% +% The same functions, under the same names, as tools/python/gallery.py (where they are +% methods of Suite: suite.check_close(...) is s = check_close(s, ...) here; a suite is a +% struct passed by value), and the same CSV outputs (results/.checks.csv, +% .metrics.csv), aggregated with the other languages by tools/report.py. +% +% s = suite(name) a suite of checks, named after the notebook +% s = check(s, check, ok, value, expected, detail, severity) +% s = check_true / check_less / check_greater / check_between / check_close / +% check_finite / check_skip (s, ...) the usual checks +% s = metric(s, case, package, name, value) record a metric +% suite_summary(s) end-of-notebook summary +% suite_finalize(s) write the CSV files, error if an error-level check failed +% read_data(name, stem), rmse, q2 +% +% Paths are relative to the repository root ($LKGALLERY_ROOT, else the working +% directory, where the notebooks are executed). +1; + +function r = gallery_root() + r = getenv("LKGALLERY_ROOT"); if isempty(r), r = "."; end +end + +% ---- suite and checks ------------------------------------------------------------------ + +function s = suite(name) + s.name = name; + s.checks = cell(0, 8); + s.metrics = cell(0, 6); + s.t0 = tic(); + s.known = cell(0, 3); + f = fullfile(gallery_root(), "tools", "known_issues.csv"); + if ~exist(f, "file"), error("run the notebooks from the repository root (%s not found)", f); end + fid = fopen(f, "r"); fgetl(fid); + while true + l = fgetl(fid); + if ~ischar(l), break; end + p = gallery_csv_fields(l); + if numel(p) >= 3, s.known(end + 1, :) = p(1:3); end + end + fclose(fid); +end + +function s = check(s, name, ok, value, expected, detail, severity) + if nargin < 4, value = ""; end + if nargin < 5, expected = ""; end + if nargin < 6, detail = ""; end + if nargin < 7, severity = "error"; end + if ok, status = "PASS"; elseif strcmp(severity, "error"), status = "FAIL"; else, status = "WARN"; end + if ~ok && ~isempty(s.known) + hit = strcmp(s.known(:, 1), s.name) & strcmp(s.known(:, 2), name); + if any(hit), status = "KNOWN"; detail = [s.known{find(hit, 1), 3} " | " detail]; end + end + if isnumeric(value) || islogical(value), value = sprintf("%.6g", double(value)); end + if ~ok, printf("[%s] %s: value=%s expected %s %s\n", status, name, value, expected, detail); end + s.checks(end + 1, :) = {s.name, "octave", name, severity, status, value, expected, detail}; +end + +function s = check_true(s, name, cond, detail, severity) + if nargin < 4, detail = ""; end + if nargin < 5, severity = "error"; end + s = check(s, name, logical(cond), logical(cond), "1", detail, severity); +end + +function s = check_less(s, name, value, bound, detail, severity) + if nargin < 5, detail = ""; end + if nargin < 6, severity = "error"; end + s = check(s, name, isfinite(value) && value < bound, value, sprintf("< %g", bound), detail, severity); +end + +function s = check_greater(s, name, value, bound, detail, severity) + if nargin < 5, detail = ""; end + if nargin < 6, severity = "error"; end + s = check(s, name, isfinite(value) && value > bound, value, sprintf("> %g", bound), detail, severity); +end + +function s = check_between(s, name, value, lo, hi, detail, severity) + if nargin < 6, detail = ""; end + if nargin < 7, severity = "error"; end + s = check(s, name, isfinite(value) && value >= lo && value <= hi, value, sprintf("in [%g, %g]", lo, hi), detail, severity); +end + +function s = check_close(s, name, a, b, rtol, atol, detail, severity) + if nargin < 5, rtol = 1e-6; end + if nargin < 6, atol = 0; end + if nargin < 7, detail = ""; end + if nargin < 8, severity = "error"; end + a = double(a(:)); b = double(b(:)); + if numel(a) ~= numel(b) + s = check(s, name, false, sprintf("length %d", numel(a)), sprintf("length %d", numel(b)), detail, severity); + return; + end + gap = abs(a - b); + ok = all(isfinite(a)) && all(gap <= atol + rtol * abs(b)); + rel = max(gap ./ max(abs(b), 1e-300)); + s = check(s, name, ok, rel, sprintf("rtol %g, atol %g", rtol, atol), detail, severity); +end + +function s = check_finite(s, name, a, detail, severity) + if nargin < 4, detail = ""; end + if nargin < 5, severity = "error"; end + s = check(s, name, ~isempty(a) && all(isfinite(a(:))), sprintf("%d non-finite", sum(~isfinite(a(:)))), "all finite", detail, severity); +end + +function s = check_skip(s, name, reason) + s.checks(end + 1, :) = {s.name, "octave", name, "info", "SKIP", "", "", reason}; + printf("[SKIP] %s: %s\n", name, reason); +end + +% ---- metrics and results --------------------------------------------------------------- + +function s = metric(s, case_name, package, name, value) + s.metrics(end + 1, :) = {s.name, "octave", case_name, package, name, sprintf("%.6g", double(value))}; +end + +function suite_summary(s) + for i = 1:size(s.metrics, 1) + printf("%-28s %-28s %-12s %s\n", s.metrics{i, 3}, s.metrics{i, 4}, s.metrics{i, 5}, s.metrics{i, 6}); + end + st = s.checks(:, 5); + printf("%d checks: PASS %d, FAIL %d, WARN %d, KNOWN %d, SKIP %d\n", numel(st), sum(strcmp(st, "PASS")), ... + sum(strcmp(st, "FAIL")), sum(strcmp(st, "WARN")), sum(strcmp(st, "KNOWN")), sum(strcmp(st, "SKIP"))); + for i = find(~strcmp(st, "PASS"))' + printf("%-6s %-6s %s: value %s, expected %s %s\n", s.checks{i, 5}, s.checks{i, 4}, s.checks{i, 3}, ... + s.checks{i, 6}, s.checks{i, 7}, s.checks{i, 8}); + end +end + +function suite_finalize(s) + s = metric(s, "_suite", "_", "elapsed_s", toc(s.t0)); + d = getenv("LKGALLERY_RESULTS"); if isempty(d), d = fullfile(gallery_root(), "results"); end + if ~exist(d, "dir"), mkdir(d); end + gallery_write_csv(fullfile(d, [s.name ".checks.csv"]), {"suite", "lang", "check", "severity", "status", "value", "expected", "detail"}, s.checks); + gallery_write_csv(fullfile(d, [s.name ".metrics.csv"]), {"suite", "lang", "case", "package", "metric", "value"}, s.metrics); + st = s.checks(:, 5); + printf("%s: PASS %d, FAIL %d, WARN %d, KNOWN %d, SKIP %d\n", s.name, sum(strcmp(st, "PASS")), sum(strcmp(st, "FAIL")), ... + sum(strcmp(st, "WARN")), sum(strcmp(st, "KNOWN")), sum(strcmp(st, "SKIP"))); + failed = s.checks(strcmp(st, "FAIL"), 3); + if ~isempty(failed) + error("%d check(s) failed in %s: %s", numel(failed), s.name, strjoin(failed', "; ")); + end +end + +% ---- data and metrics ------------------------------------------------------------------ + +function A = read_data(name, stem) % data/_.csv (no header) + A = dlmread(fullfile(gallery_root(), "data", [name "_" stem ".csv"]), ","); +end + +function r = rmse(y, m) + r = sqrt(mean((y(:) - m(:)).^2)); +end + +function r = q2(y, m) + r = 1 - sum((y(:) - m(:)).^2) / sum((y(:) - mean(y(:))).^2); +end + +% ---- internals ------------------------------------------------------------------------- + +function gallery_write_csv(f, header, rows) + fid = fopen(f, "w"); + fprintf(fid, "%s\n", strjoin(header, ",")); + for i = 1:size(rows, 1) + q = cellfun(@(c) ["\"" strrep(char(c), "\"", "\"\"") "\""], rows(i, :), "UniformOutput", false); + fprintf(fid, "%s\n", strjoin(q, ",")); + end + fclose(fid); +end + +function f = gallery_csv_fields(l) + % one CSV line into fields (RFC 4180 quoting: "a,b" and "" inside quotes) + f = {}; cur = ""; q = false; i = 1; + while i <= numel(l) + c = l(i); + if q && c == '"' && i < numel(l) && l(i + 1) == '"' + cur(end + 1) = '"'; i = i + 1; + elseif c == '"' + q = ~q; + elseif c == "," && ~q + f{end + 1} = cur; cur = ""; + else + cur(end + 1) = c; + end + i = i + 1; + end + f{end + 1} = cur; +end diff --git a/tools/py2nb.py b/tools/py2nb.py new file mode 100644 index 0000000..b414ade --- /dev/null +++ b/tools/py2nb.py @@ -0,0 +1,172 @@ +#!/usr/bin/env python3 +"""Convert a "percent" script (cells separated by ``# %%`` / ``# %% [markdown]``, or +``% %%`` in Octave) into a notebook without outputs. The notebooks of this gallery are +written this way, in ``src/``, so that they review as plain diffs; the ``.ipynb`` files +are not committed: CI generates and executes them, and publishes the executed ones (see +``tools/site.py``). Every Python notebook gets the cell of ``tools/notebook_setup.py`` +after its title, to run on Colab or Binder too. + +Every source is named ``__.`` and starts with a +markdown cell holding its title (``# · — ``), an ``**Idea.**`` +paragraph, a ``**Reference.**`` paragraph and a ``**Tags.**`` paragraph +(``key: value, value · key: value``, keys from ``TAG_KEYS``); the suite it writes +(``Suite("")``) is named after the file. ``--check`` validates all that; the title, +idea and tags also go to the notebook metadata (``metadata.gallery``), for the site. + +Usage: python tools/py2nb.py [--check] [--out DIR] [SRC...] (default: every src/* file) + each ``src/.`` gives ``/.ipynb`` (default DIR: the repository root). +""" +import argparse +import re +import sys +from pathlib import Path + +import nbformat + +ROOT = Path(__file__).resolve().parents[1] +SRC = ROOT / "src" +# code cell inserted after the title cell of every Python notebook, so that it also runs +# outside the repository (Colab, Binder): a no-op in the repository itself +SETUP = {".py": ROOT / "tools" / "notebook_setup.py"} + +KERNELS = { + ".py": ({"name": "python3", "display_name": "Python 3", "language": "python"}, + {"name": "python"}, "#"), + ".R": ({"name": "ir", "display_name": "R", "language": "R"}, + {"name": "R"}, "#"), + ".jl": ({"name": "julia-1.11", "display_name": "Julia 1.11", "language": "julia"}, + {"name": "julia"}, "#"), + ".m": ({"name": "octave", "display_name": "Octave", "language": "octave"}, + {"name": "octave"}, "%"), +} +BINDINGS = {".py": "python", ".R": "R", ".jl": "julia", ".m": "octave"} +LANGUAGES = {".py": "Python", ".R": "R", ".jl": "Julia", ".m": "Octave"} +# keys of the **Tags.** paragraph (all optional but class), in display order +TAG_KEYS = ("class", "kernel", "trend", "objective", "method", "noise", "warping", + "aggregation", "compared to", "topic") +SUITE = re.compile(r'\b(?:Suite|suite)\("([^"]+)"\)') + + +def sources(): + return sorted(f for f in SRC.iterdir() if f.suffix in KERNELS) + + +def paragraph(text, label): + """The paragraph starting with ``**