Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 0 additions & 12 deletions .vscode/c_cpp_properties.json

This file was deleted.

67 changes: 0 additions & 67 deletions .vscode/settings.json

This file was deleted.

43 changes: 41 additions & 2 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,9 @@ endif()
# before project()/CXX language enablement so the toolchain is detected with
# import-std support. Requires the Ninja generator plus a toolchain that ships a
# libc++ `std` module. Configure with: cmake -G Ninja -B build
if(CMAKE_VERSION VERSION_GREATER_EQUAL "4.3" AND CMAKE_VERSION VERSION_LESS "4.4")
if(CMAKE_VERSION VERSION_GREATER_EQUAL "4.4" AND CMAKE_VERSION VERSION_LESS "4.5")
set(CMAKE_EXPERIMENTAL_CXX_IMPORT_STD "f35a9ac6-8463-4d38-8eec-5d6008153e7d")
elseif(CMAKE_VERSION VERSION_GREATER_EQUAL "4.3" AND CMAKE_VERSION VERSION_LESS "4.4")
set(CMAKE_EXPERIMENTAL_CXX_IMPORT_STD "451f2fe2-a8a2-47c3-bc32-94786d8fc91b")
elseif(CMAKE_VERSION VERSION_GREATER_EQUAL "4.2" AND CMAKE_VERSION VERSION_LESS "4.3")
set(CMAKE_EXPERIMENTAL_CXX_IMPORT_STD "d0edc3af-4c50-42ea-a356-e2862fe7a444")
Expand Down Expand Up @@ -93,6 +95,18 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON)
# module flags for loadCommand.cppm — are always present for the editor.
set(CMAKE_EXPORT_COMPILE_COMMANDS ON)

# The macOS 27 SDK's <math.h> includes <float.h> asking only for INFINITY/NAN
# (__need_infinity_nan), but libc++'s float.h wrapper sets its include guard
# anyway, so any later <cfloat> is skipped and the DBL_*/LDBL_* macros never
# appear. Any TU that includes <cmath> before <cfloat> trips on it: Catch2's
# reporter helpers lose DBL_MAX_10_EXP, and boost-decimal's bit_layouts.hpp
# sees no LDBL_MANT_DIG, so it takes long double as unsupported and fails to
# compile. Pre-including the full <float.h> sets the guard with every macro
# defined. Set before the add_subdirectory() calls so vendored targets get it.
if(APPLE)
add_compile_options("$<$<COMPILE_LANGUAGE:CXX>:SHELL:-include float.h>")
endif()

# Clang 22 made "reduced BMI" the default for C++20 modules, and reduced BMIs
# drop the predeclared global operator new/delete from the global module
# (llvm/llvm-project#166068). A TU that imports std alongside enough of this
Expand All @@ -112,7 +126,7 @@ set(PQXX_LIBRARIES_INSTALL ON)
set(SKIP_BUILD_TEST ON)
set(SKIP_CONFIGURE_LIBPQXX OFF)

# Disable warningsfor external libraries
# Disable warnings for external libraries
set(PREV_CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS})
if(CMAKE_CXX_COMPILER_ID MATCHES "Clang|GNU")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -w")
Expand Down Expand Up @@ -142,6 +156,9 @@ find_package(PostgreSQL REQUIRED)
# Build libpqxx from source
add_subdirectory(external/libpqxx EXCLUDE_FROM_ALL)

# Restore compiler flags for project targets
set(CMAKE_CXX_FLAGS ${PREV_CMAKE_CXX_FLAGS})

# Collect all .cpp files in the src directory. CONFIGURE_DEPENDS re-runs the
# glob when .cpp files are added/removed (matching the MODULE_INTERFACES glob
# below), so a new translation unit is picked up without a manual reconfigure.
Expand Down Expand Up @@ -169,6 +186,18 @@ target_sources(BacktestingEngineLib
set_target_properties(BacktestingEngineLib PROPERTIES CXX_MODULE_STD ON)
target_compile_features(BacktestingEngineLib PUBLIC cxx_std_23)

if(CMAKE_CXX_COMPILER_ID MATCHES "Clang|GNU")
# -Wmissing-designated-field-initializers (part of newer Clangs' -Wextra)
# fires on partial designated initializers, but every struct in this
# project default-initializes its fields, so omitted fields are always
# well-defined. Partial initializers are the intended style — e.g. the
# test mocks set only the field under test.
target_compile_options(BacktestingEngineLib PUBLIC
-Wall -Wextra -Wno-missing-designated-field-initializers)
elseif(MSVC)
target_compile_options(BacktestingEngineLib PUBLIC /W4)
endif()

# Include directories — PUBLIC so they propagate to the executable (and any
# other consumer) through target_link_libraries. Every project #include is
# source-relative and path-qualified (e.g. "shared/utilities/decimalJson.hpp",
Expand Down Expand Up @@ -227,7 +256,12 @@ target_link_libraries(BacktestingEngineLib PUBLIC
CURL::libcurl
$<$<PLATFORM_ID:Linux>:rt>)

set(SAVED_FLAGS ${CMAKE_CXX_FLAGS})
if(CMAKE_CXX_COMPILER_ID MATCHES "Clang|GNU")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -w")
endif()
add_subdirectory(external/boost-decimal)
set(CMAKE_CXX_FLAGS ${SAVED_FLAGS})
target_link_libraries(BacktestingEngineLib PUBLIC Boost::decimal)

target_link_libraries(BacktestingEngineLib PUBLIC pqxx OpenMP::OpenMP_CXX)
Expand Down Expand Up @@ -265,6 +299,11 @@ endif()
# of the default `all` build until the test target pulls it in.
include(CTest)
if(BUILD_TESTING)
set(SAVED_FLAGS ${CMAKE_CXX_FLAGS})
if(CMAKE_CXX_COMPILER_ID MATCHES "Clang|GNU")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -w")
endif()
add_subdirectory(external/Catch2 EXCLUDE_FROM_ALL)
set(CMAKE_CXX_FLAGS ${SAVED_FLAGS})
add_subdirectory(tests)
endif()
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ The engine is C++23 (modules, `import std;`) and one binary with eight subcomman

I'm extracting results and creating various graphs for trend analyses using SciPy for calculations and Plotly for visualization.

![alt text](documents/images/random-indices-sp500-variable.svg)
![Equity curves for 25 random S&P 500 backtests across a 5×5 stop-loss/take-profit parameter sweep (20–100), growing portfolio equity from roughly 5k to 12k over the test window.](documents/images/random-indices-sp500-variable.svg)

*Read more results on https://mccaffers.com/quantitative_analysis/randomly_trading/*

Expand Down
9 changes: 8 additions & 1 deletion documents/questdb.md
Original file line number Diff line number Diff line change
Expand Up @@ -25,11 +25,18 @@ One table per symbol, table name == symbol (`EURUSD`, ...):

- `ask`, `bid` — scaled fixed-point INT32 "points", **not** floats: the real price times the symbol's multiplier (FX majors ×100000, JPY pairs & metals ×1000, indices/commodities ×100 — `source/shared/models/priceData.cppm`, `source/shared/utilities/symbolScale.cppm`). e.g. EURUSD 1.10001 is stored as 110001.
- designated `timestamp` — written in nanoseconds, the QuestDB HTTP default precision.
- `source` — `SYMBOL`, the feed the tick came from: `IG` (IG Markets) or `MX` (Massive, formerly Polygon). Rows from before source tracking, and any tick with an empty or unrecognised source, are `NULL`.

The write side emits ILP lines of the form `{symbol} ask={..}i,bid={..}i {tsNanos}` (`ingestCommand.cppm`). The read side names columns explicitly and parses positionally as (symbol, ask, bid, timestamp) (`source/shared/questdb/sqlManager.cppm`), so tables must serve exactly those columns.
The write side emits ILP lines of the form `{symbol},source={IG|MX} ask={..}i,bid={..}i {tsNanos}` (`source/ingest/tickLine.cppm`), omitting the `source` tag when the feed id isn't one of the two known values. The read side names columns explicitly and parses positionally as (symbol, ask, bid, timestamp) (`source/shared/questdb/sqlManager.cppm`), so tables must serve those columns; `source` is not read yet, so backtests see both feeds' ticks together.

Symbols are whitelisted: `SqlManager` throws `std::invalid_argument` for any symbol not in the `symbolScale` table, so a table named outside the whitelist is unreadable by the engine.

### How the `source` column appears

- `source` must stay an ILP **tag**, which QuestDB stores as `SYMBOL`. If it were ever sent as a field (`source="IG"`), a new table would get a string column and every later tagged write to it would fail (counted as `post-failed`).
- QuestDB adds the column to an existing table on that table's first tagged line; no migration is needed. Existing rows are not rewritten and read as `NULL`. Backfilling them with `UPDATE` would rewrite every affected partition, so it is slow on large tables.
- A table created from scratch by `ingest` gets `source` from its first line.

## Reachability

`scripts/ingest.sh` probes `http://$QUESTDB_HOST:$QUESTDB_ILP_PORT/exec?query=SELECT%201` before starting — non-fatal, since the ingest writer buffers (and eventually sheds) while QuestDB is down.
2 changes: 1 addition & 1 deletion external/boost-decimal
Submodule boost-decimal updated 294 files
2 changes: 1 addition & 1 deletion external/libpqxx
Submodule libpqxx updated 400 files
51 changes: 51 additions & 0 deletions scripts/analysis.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
#!/bin/bash
# This executes the analysis script — run.sh's twin for the experiment queue:
# drains BACKTESTING_QUEUE_EXPERIMENT_RUN and evaluates each experiment chain
# against the QuestDB tick stream, reporting one aggregate document per
# experiment x symbol group to the weekly backtesting-experiments index.

required_vars=(ELASTIC_HOST ELASTIC_USER ELASTIC_USER_PASSWORD REDIS_HOST)
missing=()
for var in "${required_vars[@]}"; do
[[ -z "${!var}" ]] && missing+=("$var")
done
if [[ ${#missing[@]} -gt 0 ]]; then
echo "Error: missing required environment variables: ${missing[*]}"
echo "eg. infisical run -- sh ./scripts/analysis.sh (if using infisical)"
echo "or ELASTIC_HOST=localhost ELASTIC_USER=elastic ELASTIC_USER_PASSWORD=password REDIS_HOST=localhost sh scripts/analysis.sh"
exit 1
fi

current_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"

# Build the source code
source "$current_dir/clean.sh"
if ! source "$current_dir/build.sh"; then
echo "Error: Build failed. Aborting."
exit 1
fi

# Debug: Check if the executable exists
if [ -f "$BUILD_DIR/$EXECUTABLE_NAME" ]; then
echo "Executable $EXECUTABLE_NAME found in $BUILD_DIR."
else
echo "Error: Executable $EXECUTABLE_NAME not found in $BUILD_DIR."
ls -la "$BUILD_DIR" # List the contents of the build directory for debugging
exit 1
fi

if ! redis-cli -h "$REDIS_HOST" ping >/dev/null 2>&1; then
echo "redis-server not reachable on $REDIS_HOST:6379 — skipping"
exit 0
fi

start_time=$(date +%s%N)
# Invoke the `analysis` subcommand: BacktestingEngine peeks a run descriptor
# off the experiment Redis queue and evaluates its experiment chains against
# the QuestDB host passed as the second argument (here, localhost). Daemon —
# Ctrl+C to stop (analysis has no shared-memory control channel; that is
# run-only).
./"$BUILD_DIR/$EXECUTABLE_NAME" analysis localhost
end_time=$(date +%s%N)
elapsed=$(( (end_time - start_time) / 1000000 ))
echo "Execution time: ${elapsed}ms"
7 changes: 7 additions & 0 deletions scripts/build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -48,9 +48,16 @@ if command -v brew &>/dev/null; then

LLVM_PREFIX="$(brew --prefix llvm)"
export COMPILER_PATH="$LLVM_PREFIX/lib/c++"
# Pin OpenSSL 3: Homebrew's default `openssl` is now 4.x, which dropped
# ASN1_STRING_data, and the AWS SDK's static s2n still calls it. The
# engine's FindOpenSSL and s2n's Findcrypto (via CMAKE_PREFIX_PATH) must
# both resolve here, or the link pulls in OpenSSL 4 and fails.
OPENSSL_PREFIX="$(brew --prefix openssl@3)"
TOOLCHAIN_ARGS+=(
-DCMAKE_C_COMPILER="$LLVM_PREFIX/bin/clang"
-DCMAKE_CXX_COMPILER="$LLVM_PREFIX/bin/clang++"
-DOPENSSL_ROOT_DIR="$OPENSSL_PREFIX"
-DCMAKE_PREFIX_PATH="$OPENSSL_PREFIX"
)
else
# Off Homebrew (Linux/CI): use Clang + libc++ so `import std` works, matching
Expand Down
8 changes: 8 additions & 0 deletions scripts/build_dep.sh
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,14 @@ if ! command -v brew &>/dev/null; then
-DCMAKE_CXX_COMPILER="${CXX:-clang++}"
)
AWS_CXX_FLAGS="-w -stdlib=libc++"
else
# Build s2n/aws-c-cal against OpenSSL 3, matching build.sh: Homebrew's
# default `openssl` is now 4.x, which dropped APIs s2n still uses.
OPENSSL_PREFIX="$(brew --prefix openssl@3)"
AWS_CMAKE_ARGS+=(
-DOPENSSL_ROOT_DIR="$OPENSSL_PREFIX"
-DCMAKE_PREFIX_PATH="$OPENSSL_PREFIX"
)
fi

AWS_SRC="$EXTERNAL_DIR/aws"
Expand Down
57 changes: 57 additions & 0 deletions scripts/experiments.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
#!/bin/bash
# Builds the engine and runs the `experiments` subcommand: expands the given
# experiment sweep's parameter grid into keyed Redis payloads plus a run
# descriptor on BACKTESTING_QUEUE_EXPERIMENT_RUN (see
# shared/utilities/queueKeys.hpp), and prepares the batch's weekly
# backtesting-experiments Elasticsearch index + -current alias.
#
# Usage mirrors load.sh:
# sh scripts/experiments.sh dipRecovery # queue one named sweep
# sh scripts/experiments.sh # queue the default sweep set
#
# Interactive: the engine prints the total sweep size and waits for Enter on
# stdin before touching Redis — run it from a TTY, or pipe the newline for
# non-interactive use (closed stdin counts as a decline and aborts).

current_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
experiments_args=("$@")

# One batch label for every invocation below: the label names the weekly
# backtesting-experiments index (backtesting-experiments-2026-28, ...), and
# pinning it here keeps a load that straddles the Sunday-midnight-UTC ISO-week
# boundary from splitting one batch across two labels. Pre-set BACKTEST_BATCH
# wins — that is the re-run/test override.
export BACKTEST_BATCH="${BACKTEST_BATCH:-$(date -u +%G-%V)}"

# Sweeps queued by a no-arg run.
all_sweeps=(dipRecovery rangeRelativeDipRecovery)

if ! source "$current_dir/build.sh"; then
echo "Error: Build failed. Aborting."
exit 1
fi

if [ ! -f "$BUILD_DIR/$EXECUTABLE_NAME" ]; then
echo "Error: Executable $EXECUTABLE_NAME not found in $BUILD_DIR."
ls -la "$BUILD_DIR"
exit 1
fi

if ! redis-cli -h "${REDIS_HOST:-127.0.0.1}" ping >/dev/null 2>&1; then
echo "redis-server not reachable on ${REDIS_HOST:-127.0.0.1}:6379 — skipping"
exit 0
fi

if [ ${#experiments_args[@]} -gt 0 ]; then
exec ./"$BUILD_DIR/$EXECUTABLE_NAME" experiments "${experiments_args[@]}"
fi

i=0
for sweep in "${all_sweeps[@]}"; do
echo ""
echo "=== [$((++i))/${#all_sweeps[@]}] Loading experiment sweep: $sweep ==="
if ! ./"$BUILD_DIR/$EXECUTABLE_NAME" experiments "$sweep"; then
echo "Error: experiments failed for '$sweep'. Aborting remaining sweeps."
exit 1
fi
done
1 change: 1 addition & 0 deletions source/analysis/analysisCommand.cppm
Original file line number Diff line number Diff line change
Expand Up @@ -29,4 +29,5 @@ int AnalysisCommand::run(const int argc, const char* argv[]) {

// Redis host from the environment, mirroring `run` (runCommand.cppm).
return AnalysisRunner::run(argv[2], env::getOr("REDIS_HOST", "127.0.0.1"));

}
4 changes: 4 additions & 0 deletions source/analysis/queue/drainExperiments.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,10 @@

#pragma once

// Boost.Redis's libc++ double converter uses HUGE_VAL and std::strtod without
// including <cmath>/<cstdlib>; include them first so this header stands alone.
#include <cmath>
#include <cstdlib>
#include <memory>
#include <string>

Expand Down
Loading
Loading