diff --git a/CMakeLists.txt b/CMakeLists.txt index 563042735..e7c2b51c6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -337,6 +337,8 @@ configure_file(${PROJECT_SOURCE_DIR}/${PROJECT_NAME}.pc.in ${CMAKE_CURRENT_BINAR configure_file(${PROJECT_SOURCE_DIR}/${PROJECT_NAME}.cmake.in ${CMAKE_CURRENT_BINARY_DIR}/Find${PROJECT_NAME}.cmake @ONLY) # Install configuration +install(FILES LICENSE NOTICE + DESTINATION ${CMAKE_INSTALL_DATADIR}/licenses/${PROJECT_NAME}) install(DIRECTORY ${PROJECT_SOURCE_DIR}/libCacheSim/include/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) install(FILES ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}.pc diff --git a/NOTICE b/NOTICE new file mode 100644 index 000000000..3fda55c7b --- /dev/null +++ b/NOTICE @@ -0,0 +1,39 @@ +Portions of libCacheSim's native synthetic reader are adapted from: + +Apache Commons RNG +Copyright 2016-2025 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +The bounded Zipf sampler in synthetic.c is adapted to C and uses indexed +random draws. It is distributed under the Apache License 2.0 (see LICENSE). + +Random123 (Philox2x64-10) + +Copyright 2010-2011, D. E. Shaw Research. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or other materials provided with the distribution. +* Neither the name of D. E. Shaw Research nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. diff --git a/README.md b/README.md index afa9e3603..93d308c90 100644 --- a/README.md +++ b/README.md @@ -307,6 +307,13 @@ If you need to add a new trace type or a new algorithm, please see [here](/doc/a We encourage users to check [deepWiki](https://deepwiki.com/1a1a11a/libCacheSim) for more detailed documentation. --- + +### Synthetic traces + +Generate native uniform or Zipf workloads without a trace file using the `syn` +trace type. See [synthetic traces](doc/synthetic_traces.md) for syntax, seeds, +reader behavior and limitations. + ## Python package If you are not extremely sensitive to performance, our Python binding offers an easier way to access the core features of libCacheSim. diff --git a/doc/advanced_lib_extend.md b/doc/advanced_lib_extend.md index e69f371ba..70d7f0168 100644 --- a/doc/advanced_lib_extend.md +++ b/doc/advanced_lib_extend.md @@ -81,7 +81,7 @@ int myReader_read_one_req(reader_t *reader, request_t *req); Here are the steps to add a new trace reader: 1. add a new new trace type, e.g., `MYREADER_TRACE` in `trace_type_e` in [include/libCacheSim/enum.h](/libCacheSim/include/libCacheSim/enum.h). 2. add a new reader file, e.g., `myReader.h` in [traceReader/customizedReader/](/libCacheSim/traceReader/customizedReader/) and implement the two functions. -3. add `myReader_setup()` to `setup_reader()`and `myReader_read_one_req()` to `read_one_req()` in [traceReader/reader.c](/libCacheSim/traceReader/reader.c). +3. add `myReader_setup()` to `setup_reader()` in [traceReader/reader.c](/libCacheSim/traceReader/reader.c), and `myReader_read_one_req()` to `read_one_req()` in [traceReader/readerRequest.c](/libCacheSim/traceReader/readerRequest.c). 4. add `MYREADER_TRACE` to `trace_type_str_to_enum()` in [bin/cli_reader_utils.c](/libCacheSim/bin/cli_reader_utils.c) diff --git a/doc/synthetic_traces.md b/doc/synthetic_traces.md new file mode 100644 index 000000000..f48856ea6 --- /dev/null +++ b/doc/synthetic_traces.md @@ -0,0 +1,93 @@ +# Native synthetic traces + +Use the `syn` trace type to generate requests directly in the native reader: + +```sh +./_build/bin/cachesim Uniform_1000000_10000 syn lru 1000 +./_build/bin/cachesim Zipf_1000000_10000_0.9_42 syn lru,s3fifo 100,1000 +``` + +The specifications are `Uniform_n_m[_seed]` and `Zipf_n_m_alpha[_seed]`: + +| Parameter | Meaning | Supported values | +| --- | --- | --- | +| `n` | Number of source requests, before sampling or a request limit | Integer from 1 through `INT64_MAX` | +| `m` | Number of possible objects | Integer from 1 through `INT32_MAX` | +| `alpha` | Zipf exponent; zero selects uniform generation | Finite nonnegative decimal number, including scientific notation | +| `seed` | Independent workload seed | Integer from 0 through `UINT64_MAX`; defaults to 1 | + +Distribution names are case-insensitive. Numeric fields must be complete tokens; +negative values, overflow, exponents that underflow to zero, nonfinite exponents +and trailing text are rejected. + +IDs are in `[0, m)`. Uniform requests select each ID with probability `1/m`. +For Zipf, ID `k` has probability proportional to `(k + 1)^(-alpha)` over that +finite range. Requests are sampled with replacement; a finite trace need not +contain every object. + +Each request has size and cost 1, operation `OP_NOP`, no TTL, and timestamp 0, +matching a plain-text trace without timestamps. There is no arrival-time model; +time-based warmup and analysis are not meaningful for this source. Future-access +metadata is unavailable, so it cannot directly drive offline Belady algorithms. +Reader options for columns, file offsets, headers, block splitting and string +object IDs do not apply. Ordinary spatial sampling and request limits apply to +the generated stream. A positive request limit counts source reads, including +requests discarded by sampling; zero and negative limits mean unlimited. + +Fractional/automatic cache capacities retain the existing working-set scan. +That scan consumes time and memory proportional to the observed workload; use +absolute capacities when avoiding that preprocessing is important. Generation +itself keeps constant-size state, without a probability table or trace buffer. + +## Library use and reproducibility + +```c +reader_t *reader = setup_reader("Zipf_1000000_10000_0.9_42", + SYNTHETIC_TRACE, NULL); +request_t *req = new_request(); +while (read_one_req(reader, req) == 0) { + /* Consume req. */ +} +free_request(req); +close_reader(reader); +``` + +Reset replays the same sequence. A clone starts at the beginning with the same +configuration and independent state, following the existing reader convention. +The source cursor supports skipping, fractional seeking and reverse traversal +without replaying the prefix. First/last-request helpers leave the original +reader untouched. Counting requests returns the declared source length. + +The generator uses Philox2x64-10 with the source index and rejection-draw index +as its counter and the seed as its key. It never consumes the eviction +algorithms' random state. Changing worker count or algorithm order does not +change the workload, and increasing `n` preserves its prefix. Randomized cache +algorithms may still produce different simulation results. + +Uniform sampling removes modulo bias. Bounded Zipf uses rejection-inversion, +with stable integral/inverse calculations around exponent 1, adapted from +Apache Commons RNG. The object-count bound keeps the ranks within the reference +algorithm's supported integer range. Floating-point precision limits very small +tail probabilities. Zipf output is reproducible within the same implementation +and numerical environment; identical output across different math libraries or +future generator changes is not guaranteed. Materialize a trace when exact +long-term replay across environments is required. + +Spatial sampling preserves source-index navigation. Stateful temporal sampling +supports forward reads, reset and clones; seek/skip/reverse move the source +cursor, not the temporal sampler's history. Reset before a new forward pass. + +## Validation + +`testSyntheticReader` covers published RNG vectors, reset/clone/navigation, +sampling and limits, finite-distribution checks, and equality with a materialized +trace for LRU, FIFO and S3FIFO in serial and parallel simulations. `testCLI` +covers synthetic syntax and invalid inputs. + +```sh +ctest --test-dir _build --output-on-failure +``` + +Reader throughput includes generation cost. For benchmarks, report generation +and simulation separately; replaying an existing binary file can be faster when +many experiments reuse a workload. diff --git a/libCacheSim/bin/cachesim/cli_parser.c b/libCacheSim/bin/cachesim/cli_parser.c index aa4376e7d..28b14ea92 100644 --- a/libCacheSim/bin/cachesim/cli_parser.c +++ b/libCacheSim/bin/cachesim/cli_parser.c @@ -215,7 +215,7 @@ static char doc[] = "example: ./cachesim /trace/path csv LRU 100MB\n\n" "trace can be zstd compressed\n" "cache_size is in byte, but also support KB/MB/GB\n" - "supported trace_type: txt/csv/twr/vscsi/oracleGeneralBin\n" + "supported trace_type: txt/csv/twr/vscsi/oracleGeneralBin/syn\n" "supported eviction_algo: LRU/LFU/FIFO/ARC/LeCaR/Cacheus/Merlin\n" "print-head-req: Print the first few requests when simulating start\n"; @@ -276,7 +276,11 @@ void free_arg(struct arguments *args) { // args->caches[i]->cache_free(args->caches[i]); // } + /* setup_reader clones the sampler; keep the original alive for reader + * clones, then release the CLI-owned instance when simulation is done. */ + sampler_t *sampler = args->reader->init_params.sampler; close_reader(args->reader); + if (sampler != NULL) sampler->free(sampler); } /** diff --git a/libCacheSim/bin/cli_reader_utils.c b/libCacheSim/bin/cli_reader_utils.c index 4b36a6fdf..bbbce92b3 100644 --- a/libCacheSim/bin/cli_reader_utils.c +++ b/libCacheSim/bin/cli_reader_utils.c @@ -29,6 +29,8 @@ trace_type_e trace_type_str_to_enum(const char *trace_type_str, trace_path); } return trace_type; + } else if (strcasecmp(trace_type_str, "syn") == 0) { + return SYNTHETIC_TRACE; } else if (strcasecmp(trace_type_str, "txt") == 0) { return PLAIN_TXT_TRACE; } else if (strcasecmp(trace_type_str, "csv") == 0) { diff --git a/libCacheSim/include/libCacheSim/enum.h b/libCacheSim/include/libCacheSim/enum.h index 3e1d90827..30d230cfc 100644 --- a/libCacheSim/include/libCacheSim/enum.h +++ b/libCacheSim/include/libCacheSim/enum.h @@ -19,7 +19,8 @@ typedef enum { BINARY_TRACE_FORMAT, TXT_TRACE_FORMAT, - INVALID_TRACE_FORMAT + INVALID_TRACE_FORMAT, + SYNTHETIC_TRACE_FORMAT } trace_format_e; // trace type @@ -43,6 +44,7 @@ typedef enum { VALPIN_TRACE, UNKNOWN_TRACE, + SYNTHETIC_TRACE, } __attribute__((__packed__)) trace_type_e; static const char* const g_trace_type_name[UNKNOWN_TRACE + 2] = { @@ -63,6 +65,7 @@ static const char* const g_trace_type_name[UNKNOWN_TRACE + 2] = { "VALPIN_TRACE", "UNKNOWN_TRACE", + "SYNTHETIC_TRACE", }; typedef enum { diff --git a/libCacheSim/include/libCacheSim/reader.h b/libCacheSim/include/libCacheSim/reader.h index f7db21916..cee403719 100644 --- a/libCacheSim/include/libCacheSim/reader.h +++ b/libCacheSim/include/libCacheSim/reader.h @@ -180,7 +180,8 @@ static inline reader_init_param_t default_reader_init_params(void) { /** * setup a reader for reading trace - * @param trace_path path to the trace + * @param trace_path path to the trace, or Uniform_n_m[_seed] / + * Zipf_n_m_alpha[_seed] for SYNTHETIC_TRACE (see doc/synthetic_traces.md) * @param trace_type CSV_TRACE, PLAIN_TXT_TRACE, BIN_TRACE, VSCSI_TRACE, * TWR_BIN_TRACE, see libCacheSim/enum.h for more * @param reader_init_param some initialization parameters used by csv and diff --git a/libCacheSim/traceReader/CMakeLists.txt b/libCacheSim/traceReader/CMakeLists.txt index 6e1b68ced..630cd84ed 100644 --- a/libCacheSim/traceReader/CMakeLists.txt +++ b/libCacheSim/traceReader/CMakeLists.txt @@ -5,10 +5,12 @@ set(traceReader_sources_c generalReader/binary.c generalReader/csv.c + generalReader/synthetic.c generalReader/txt.c generalReader/libcsv.c customizedReader/lcs.c reader.c + readerRequest.c sampling/spatial.c sampling/temporal.c ) diff --git a/libCacheSim/traceReader/generalReader/synthetic.c b/libCacheSim/traceReader/generalReader/synthetic.c new file mode 100644 index 000000000..bf134e5a7 --- /dev/null +++ b/libCacheSim/traceReader/generalReader/synthetic.c @@ -0,0 +1,188 @@ +/* + * Bounded Zipf sampling adapted from Apache Commons RNG's + * RejectionInversionZipfSampler (Apache License 2.0): + * https://commons.apache.org/proper/commons-rng/ + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "synthetic.h" + +#include +#include + +#include "syntheticRng.h" + +/* Stable limits at alpha=1 for H(x)=(x^(1-alpha)-1)/(1-alpha) + * and its inverse. These also avoid cancellation near alpha=1. */ +static double expm1_over_x(double x) { + if (fabs(x) > 1e-8) return expm1(x) / x; + return 1 + x * 0.5 * (1 + x / 3 * (1 + x * 0.25)); +} + +static double log1p_over_x(double x) { + if (fabs(x) > 1e-8) return log1p(x) / x; + return 1 - x * (0.5 - x * (1.0 / 3 - x * 0.25)); +} + +static double h_integral(double x, double alpha) { + double log_x = log(x); + /* expm1(t) rounds to -1 below -40. Avoid overflowing t for large alpha. */ + if (1 - alpha < -40 / log_x) return -1 / (1 - alpha); + return expm1_over_x((1 - alpha) * log_x) * log_x; +} + +static double h_inverse(double x, double alpha) { + /* Roundoff can put t just below -1 at the upper endpoint. */ + double t = fmax(-1, x * (1 - alpha)); + if (t == -1) return INFINITY; + return exp(log1p_over_x(t) * x); +} + +static bool parse_uint(const char *text, uint64_t max, uint64_t *value) { + if (*text == '\0') return false; + for (const char *p = text; *p != '\0'; p++) { + if (!g_ascii_isdigit(*p)) return false; + } + errno = 0; + *value = strtoull(text, NULL, 10); + return errno != ERANGE && *value <= max; +} + +int synthetic_setup(reader_t *reader) { + synthetic_params_t params = {.seed = 1}; + uint64_t n_requests = 0; + char **parts = g_strsplit(reader->trace_path, "_", -1); + size_t count = g_strv_length(parts); + const char *error = NULL; + bool zipf = count > 0 && g_ascii_strcasecmp(parts[0], "Zipf") == 0; + size_t n_fields = zipf ? 4 : 3; + if ((!zipf && (count == 0 || g_ascii_strcasecmp(parts[0], "Uniform") != 0)) || + (count != n_fields && count != n_fields + 1)) { + error = "expected Uniform_n_m[_seed] or Zipf_n_m_alpha[_seed]"; + } else if (!parse_uint(parts[1], INT64_MAX, &n_requests) || n_requests == 0) { + error = "n must be an integer in [1, INT64_MAX]"; + } else if (!parse_uint(parts[2], INT32_MAX, ¶ms.n_objects) || + params.n_objects == 0) { + error = "m must be an integer in [1, INT32_MAX]"; + } else if (count == n_fields + 1 && + !parse_uint(parts[n_fields], UINT64_MAX, ¶ms.seed)) { + error = "seed must be an integer in [0, UINT64_MAX]"; + } else if (zipf) { + char *end; + errno = 0; + params.alpha = g_ascii_strtod(parts[3], &end); + if ((!g_ascii_isdigit(parts[3][0]) && parts[3][0] != '.') || + strspn(parts[3], "0123456789.eE+-") != strlen(parts[3]) || + *end != '\0' || (errno == ERANGE && params.alpha == 0) || + !isfinite(params.alpha) || params.alpha < 0) { + error = "alpha must be a finite nonnegative number"; + } + } + g_strfreev(parts); + + const reader_init_param_t *init = &reader->init_params; + if (error == NULL && + (init->trace_start_offset != 0 || init->binary_fmt_str != NULL || + init->block_size > 1 || init->time_field != 0 || + init->obj_id_field != 0 || init->obj_size_field != 0 || + init->obj_cost_field != 0 || init->op_field != 0 || + init->ttl_field != 0 || init->cnt_field != 0 || + init->tenant_field != 0 || init->next_access_vtime_field != 0 || + init->n_feature_fields != 0 || init->has_header || + (init->obj_id_is_num_set && !init->obj_id_is_num))) { + error = + "synthetic traces do not support file fields, headers, offsets, " + "block splitting or string object IDs"; + } + if (error != NULL) { + fprintf(stderr, "error: invalid synthetic trace '%s': %s\n", + reader->trace_path, error); + return 1; + } + + if (params.alpha > 0 && params.n_objects > 1) { + params.h_integral_min = h_integral(1.5, params.alpha) - 1; + params.h_integral_max = h_integral(params.n_objects + 0.5, params.alpha); + params.squeeze = 2 - h_inverse(h_integral(2.5, params.alpha) - + exp(-params.alpha * log(2)), + params.alpha); + } + synthetic_params_t *state = malloc(sizeof(*state)); + if (state == NULL) { + fprintf(stderr, "error: cannot allocate synthetic reader\n"); + return 1; + } + *state = params; + reader->reader_params = state; + reader->trace_format = SYNTHETIC_TRACE_FORMAT; + reader->n_total_req = (int64_t)n_requests; + reader->obj_id_is_num = true; + return 0; +} + +static uint64_t synthetic_object(const synthetic_params_t *params) { + if (params->n_objects == 1) return 0; + uint64_t draw = 0; + if (params->alpha == 0) { + /* Discard the incomplete residue group to avoid modulo bias. */ + uint64_t threshold = -params->n_objects % params->n_objects; + uint64_t word; + do { + word = synthetic_random(params->position, draw++, params->seed); + } while (word < threshold); + return word % params->n_objects; + } + + /* Hormann & Derflinger, rejection-inversion for monotone discrete + * distributions (1996), with H shifted to support alpha <= 1 as well. + * Rank k has probability proportional to k^-alpha; IDs are k-1. */ + for (;;) { + uint64_t word = synthetic_random(params->position, draw++, params->seed); + double u = params->h_integral_max + + (double)(word >> 11) * 0x1.0p-53 * + (params->h_integral_min - params->h_integral_max); + double x = h_inverse(u, params->alpha); + /* Clamp before the integer conversion, including rounded endpoints. */ + double k = fmin((double)params->n_objects, fmax(1, floor(x + 0.5))); + if (k - x <= params->squeeze || u >= h_integral(k + 0.5, params->alpha) - + exp(-params->alpha * log(k))) { + return (uint64_t)k - 1; + } + } +} + +bool synthetic_has_next(const reader_t *reader) { + const synthetic_params_t *params = reader->reader_params; + return params->position < (uint64_t)reader->n_total_req && + (reader->cap_at_n_req <= 0 || + reader->n_read_req < reader->cap_at_n_req); +} + +int synthetic_read_one_req(reader_t *reader, request_t *req) { + synthetic_params_t *params = reader->reader_params; + req->obj_id = synthetic_object(params); + req->clock_time = 0; + req->op = OP_NOP; + req->next_access_vtime = -2; + req->tenant_id = 0; + req->ns = 0; + req->n_features = 0; + req->kv.key_size = 0; + req->kv.val_size = 0; + params->position++; + return 0; +} diff --git a/libCacheSim/traceReader/generalReader/synthetic.h b/libCacheSim/traceReader/generalReader/synthetic.h new file mode 100644 index 000000000..cc8dfbc99 --- /dev/null +++ b/libCacheSim/traceReader/generalReader/synthetic.h @@ -0,0 +1,18 @@ +#pragma once + +#include "libCacheSim/reader.h" + +/* Private reader state. Position counts source requests, before sampling. */ +typedef struct { + uint64_t position; + uint64_t n_objects; + uint64_t seed; + double alpha; + double h_integral_max; + double h_integral_min; + double squeeze; +} synthetic_params_t; + +int synthetic_setup(reader_t *reader); +bool synthetic_has_next(const reader_t *reader); +int synthetic_read_one_req(reader_t *reader, request_t *req); diff --git a/libCacheSim/traceReader/generalReader/syntheticRng.h b/libCacheSim/traceReader/generalReader/syntheticRng.h new file mode 100644 index 000000000..43d8f2bff --- /dev/null +++ b/libCacheSim/traceReader/generalReader/syntheticRng.h @@ -0,0 +1,47 @@ +/* + * Philox2x64-10, adapted from Random123's philox.h to return one word. + * https://github.com/DEShawResearch/random123 + * + * Copyright 2010-2011, D. E. Shaw Research. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions, and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions, and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * * Neither the name of D. E. Shaw Research nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +#pragma once + +#include + +/* Each source position has its own draw counter. Rejection sampling never + * changes subsequent requests, and seeking needs no replay or shared RNG. */ +static inline uint64_t synthetic_random(uint64_t position, uint64_t draw, + uint64_t seed) { + for (int round = 0; round < 10; round++) { + __uint128_t product = (__uint128_t)position * UINT64_C(0xd2b74407b1ce6e93); + position = (uint64_t)(product >> 64) ^ seed ^ draw; + draw = (uint64_t)product; + seed += UINT64_C(0x9e3779b97f4a7c15); + } + return position; +} diff --git a/libCacheSim/traceReader/reader.c b/libCacheSim/traceReader/reader.c index 7705e57b7..39235bfaf 100644 --- a/libCacheSim/traceReader/reader.c +++ b/libCacheSim/traceReader/reader.c @@ -7,6 +7,7 @@ // #include +#include #include "customizedReader/lcs.h" #include "customizedReader/oracle/oracleGeneralBin.h" @@ -17,6 +18,7 @@ #include "customizedReader/valpinBin.h" #include "customizedReader/vscsi.h" #include "generalReader/libcsv.h" +#include "generalReader/synthetic.h" #include "libCacheSim/macro.h" #include "readerInternal.h" @@ -56,7 +58,8 @@ reader_t *setup_reader(const char *const trace_path, reader->zstd_reader_p = NULL; #ifdef SUPPORT_ZSTD_TRACE size_t slen = strlen(trace_path); - if (strncmp(trace_path + (slen - 4), ".zst", 4) == 0) { + if (trace_type != SYNTHETIC_TRACE && slen >= 4 && + strncmp(trace_path + (slen - 4), ".zst", 4) == 0) { reader->is_zstd_file = true; reader->zstd_reader_p = create_zstd_reader(trace_path); if (!_info_printed) { @@ -105,6 +108,14 @@ reader_t *setup_reader(const char *const trace_path, assert(trace_path != NULL); reader->trace_path = strdup(trace_path); + if (trace_type == SYNTHETIC_TRACE) { + if (synthetic_setup(reader) != 0) { + close_reader(reader); + exit(EXIT_FAILURE); + } + return reader; + } + if ((fd = open(trace_path, O_RDONLY)) < 0) { ERROR("Unable to open '%s', %s\n", trace_path, strerror(errno)); exit(1); @@ -226,127 +237,6 @@ reader_t *setup_reader(const char *const trace_path, return reader; } -/** - * @brief read one request from trace file - * - * @param reader - * @param req - * @return 0 if success, 1 if end of file - */ -int read_one_req(reader_t *const reader, request_t *const req) { - if (reader->mmap_offset >= reader->file_size) { - DEBUG("read_one_req: end of file, current mmap_offset %zu, file size %zu\n", - reader->mmap_offset, reader->file_size); - req->valid = false; - return 1; - } - - if (reader->cap_at_n_req > 1 && reader->n_read_req >= reader->cap_at_n_req) { - DEBUG("read_one_req: processed %ld requests capped by the user\n", - (long)reader->n_read_req); - req->valid = false; - return 1; - } - - int status = 0; - size_t offset_before_read = reader->mmap_offset; - if (reader->n_req_left > 0) { - reader->n_req_left -= 1; - req->clock_time = reader->last_req_clock_time; - - } else { - reader->n_read_req += 1; - req->hv = 0; - req->ttl = 0; - req->obj_size = 1; - req->obj_cost = 1; - req->valid = true; - - switch (reader->trace_type) { - case CSV_TRACE: - offset_before_read = ftell(reader->file); - status = csv_read_one_req(reader, req); - break; - case PLAIN_TXT_TRACE:; - offset_before_read = ftell(reader->file); - status = txt_read_one_req(reader, req); - break; - case BIN_TRACE: - status = binary_read_one_req(reader, req); - break; - case VSCSI_TRACE: - status = vscsi_read_one_req(reader, req); - break; - case TWR_TRACE: - status = twr_read_one_req(reader, req); - break; - case TWRNS_TRACE: - status = twrNS_read_one_req(reader, req); - break; - case ORACLE_GENERAL_TRACE: - status = oracleGeneralBin_read_one_req(reader, req); - break; - case ORACLE_SIM_TWR_TRACE: - status = oracleSimTwrBin_read_one_req(reader, req); - break; - case ORACLE_SYS_TWRNS_TRACE: - status = oracleSysTwrNSBin_read_one_req(reader, req); - break; - case ORACLE_SIM_TWRNS_TRACE: - status = oracleSimTwrNSBin_read_one_req(reader, req); - break; - case LCS_TRACE: - status = lcs_read_one_req(reader, req); - break; - case VALPIN_TRACE: - status = valpin_read_one_req(reader, req); - break; - default: - ERROR( - "cannot recognize reader trace_type, given reader trace_type: " - "%c\n", - reader->trace_type); - abort(); - } - } - - if (reader->sampler != NULL) { - /* we use this complex solution rather than simply recursive calls - because recursive calls can lead to stack overflow */ - sampler_t *sampler = reader->sampler; - reader->sampler = NULL; - while (!sampler->sample(sampler, req)) { - VERBOSE("skip one req: time %lu, obj_id %lu, size %lu at offset %zu\n", - req->clock_time, req->obj_id, req->obj_size, offset_before_read); - if (reader->read_direction == READ_FORWARD) { - status = read_one_req(reader, req); - } else { - status = read_one_req_above(reader, req); - } - if (status != 0) { - reader->sampler = sampler; - return status; - } - } - reader->sampler = sampler; - } - - if (reader->ignore_obj_size) { - req->obj_size = 1; - } - - if (reader->init_params.obj_cost_field <= 0) { - req->obj_cost = 1; - } - - VERBOSE( - "read one req: time %lu, obj_id %lu, size %lu, cost %lu at offset %zu\n", - req->clock_time, req->obj_id, req->obj_size, req->obj_cost, - offset_before_read); - - return status; -} - /** * @brief from current line/request, go back one, the next read will * get the current request @@ -356,6 +246,12 @@ int read_one_req(reader_t *const reader, request_t *const req) { */ int go_back_one_req(reader_t *const reader) { switch (reader->trace_format) { + case SYNTHETIC_TRACE_FORMAT: { + synthetic_params_t *params = reader->reader_params; + if (params->position == 0) return 1; + params->position--; + return 0; + } case TXT_TRACE_FORMAT:; ssize_t curr_offset = ftell(reader->file); if (curr_offset <= reader->trace_start_offset) { @@ -466,7 +362,13 @@ int skip_n_req(reader_t *reader, const int N) { char **buf = &reader->line_buf; size_t *buf_size_ptr = &reader->line_buf_size; - if (reader->trace_format == TXT_TRACE_FORMAT) { + if (reader->trace_format == SYNTHETIC_TRACE_FORMAT) { + synthetic_params_t *params = reader->reader_params; + if (N <= 0) return 0; + uint64_t left = (uint64_t)reader->n_total_req - params->position; + count = (int)MIN((uint64_t)N, left); + params->position += count; + } else if (reader->trace_format == TXT_TRACE_FORMAT) { for (int i = 0; i < N; i++) { if (getline(buf, buf_size_ptr, reader->file) == -1) { WARN("try to skip %d requests, but only %d requests left\n", N, i); @@ -496,6 +398,16 @@ void reset_reader(reader_t *const reader) { long curr_offset = 0; reader->n_read_req = 0; + if (reader->trace_type == SYNTHETIC_TRACE) { + ((synthetic_params_t *)reader->reader_params)->position = 0; + if (reader->sampler != NULL) { + sampler_t *sampler = reader->sampler; + reader->sampler = sampler->clone(sampler); + sampler->free(sampler); + } + return; + } + #ifdef SUPPORT_ZSTD_TRACE if (reader->is_zstd_file) { reset_zstd_reader(reader->zstd_reader_p); @@ -542,11 +454,15 @@ int64_t get_num_of_req(reader_t *const reader) { } reader_t *clone_reader(const reader_t *const reader_in) { - reader_t *reader = setup_reader(reader_in->trace_path, reader_in->trace_type, - &reader_in->init_params); + reader_init_param_t init = reader_in->init_params; + if (reader_in->trace_type == SYNTHETIC_TRACE) { + init.sampler = reader_in->sampler; + } + reader_t *reader = + setup_reader(reader_in->trace_path, reader_in->trace_type, &init); reader->n_total_req = reader_in->n_total_req; - if (reader->trace_format != TXT_TRACE_FORMAT) { + if (reader->trace_format == BINARY_TRACE_FORMAT) { munmap(reader->mapped_file, reader->file_size); reader->mapped_file = reader_in->mapped_file; } @@ -562,7 +478,9 @@ int close_reader(reader_t *const reader) { indicate the error. In either case no further access to the stream is possible.*/ - if (reader->trace_type == PLAIN_TXT_TRACE) { + if (reader->trace_type == SYNTHETIC_TRACE) { + free(reader->init_params.binary_fmt_str); + } else if (reader->trace_type == PLAIN_TXT_TRACE) { fclose(reader->file); free(reader->line_buf); } else if (reader->trace_type == CSV_TRACE) { @@ -598,7 +516,11 @@ int close_reader(reader_t *const reader) { } if (reader->sampler != NULL) { - free(reader->sampler); + if (reader->trace_type == SYNTHETIC_TRACE) { + reader->sampler->free(reader->sampler); + } else { + free(reader->sampler); + } } free(reader->trace_path); @@ -613,6 +535,17 @@ int close_reader(reader_t *const reader) { * this function should not be used in the middle of reading the trace */ void reader_set_read_pos(reader_t *const reader, double pos) { + if (reader->trace_type == SYNTHETIC_TRACE) { + synthetic_params_t *params = reader->reader_params; + if (isnan(pos) || pos <= 0) { + params->position = 0; + } else if (pos >= 1) { + params->position = (uint64_t)reader->n_total_req; + } else { + params->position = (uint64_t)((long double)reader->n_total_req * pos); + } + return; + } /* jason (202004): this may not work for CSV */ if (pos > 1) pos = 1; @@ -640,6 +573,12 @@ void reader_set_read_pos(reader_t *const reader, double pos) { } void read_first_req(reader_t *reader, request_t *req) { + if (reader->trace_type == SYNTHETIC_TRACE) { + reader_t *copy = clone_reader(reader); + read_one_req(copy, req); + close_reader(copy); + return; + } uint64_t offset = reader->mmap_offset; reset_reader(reader); read_one_req(reader, req); @@ -647,6 +586,15 @@ void read_first_req(reader_t *reader, request_t *req) { } void read_last_req(reader_t *reader, request_t *req) { + if (reader->trace_type == SYNTHETIC_TRACE) { + reader_t *copy = clone_reader(reader); + reader_set_read_pos(copy, 1); + go_back_one_req(copy); + copy->read_direction = READ_BACKWARD; + read_one_req(copy, req); + close_reader(copy); + return; + } uint64_t offset = reader->mmap_offset; reset_reader(reader); reader_set_read_pos(reader, 1.0); diff --git a/libCacheSim/traceReader/readerRequest.c b/libCacheSim/traceReader/readerRequest.c new file mode 100644 index 000000000..9a50b048d --- /dev/null +++ b/libCacheSim/traceReader/readerRequest.c @@ -0,0 +1,145 @@ +// +// readerRequest.c +// libCacheSim +// +// Created by Juncheng on 5/25/16. +// Copyright © 2016 Juncheng. All rights reserved. +// + +#include "customizedReader/lcs.h" +#include "customizedReader/oracle/oracleGeneralBin.h" +#include "customizedReader/oracle/oracleTwrBin.h" +#include "customizedReader/oracle/oracleTwrNSBin.h" +#include "customizedReader/twrBin.h" +#include "customizedReader/twrNSBin.h" +#include "customizedReader/valpinBin.h" +#include "customizedReader/vscsi.h" +#include "generalReader/libcsv.h" +#include "generalReader/synthetic.h" +#include "libCacheSim/macro.h" +#include "readerInternal.h" + +/** + * @brief read the next request from the source + * + * @param reader + * @param req + * @return 0 if success, 1 if end of file + */ +int read_one_req(reader_t *const reader, request_t *const req) { + if (reader->mmap_offset >= reader->file_size && + (reader->trace_type != SYNTHETIC_TRACE || !synthetic_has_next(reader))) { + DEBUG("read_one_req: end of file, current mmap_offset %zu, file size %zu\n", + reader->mmap_offset, reader->file_size); + req->valid = false; + return 1; + } + + if (reader->cap_at_n_req > 1 && reader->n_read_req >= reader->cap_at_n_req) { + DEBUG("read_one_req: processed %ld requests capped by the user\n", + (long)reader->n_read_req); + req->valid = false; + return 1; + } + + int status = 0; + size_t offset_before_read = reader->mmap_offset; + if (reader->n_req_left > 0) { + reader->n_req_left -= 1; + req->clock_time = reader->last_req_clock_time; + + } else { + reader->n_read_req += 1; + req->hv = 0; + req->ttl = 0; + req->obj_size = 1; + req->obj_cost = 1; + req->valid = true; + + switch (reader->trace_type) { + case CSV_TRACE: + offset_before_read = ftell(reader->file); + status = csv_read_one_req(reader, req); + break; + case PLAIN_TXT_TRACE:; + offset_before_read = ftell(reader->file); + status = txt_read_one_req(reader, req); + break; + case BIN_TRACE: + status = binary_read_one_req(reader, req); + break; + case VSCSI_TRACE: + status = vscsi_read_one_req(reader, req); + break; + case TWR_TRACE: + status = twr_read_one_req(reader, req); + break; + case TWRNS_TRACE: + status = twrNS_read_one_req(reader, req); + break; + case ORACLE_GENERAL_TRACE: + status = oracleGeneralBin_read_one_req(reader, req); + break; + case ORACLE_SIM_TWR_TRACE: + status = oracleSimTwrBin_read_one_req(reader, req); + break; + case ORACLE_SYS_TWRNS_TRACE: + status = oracleSysTwrNSBin_read_one_req(reader, req); + break; + case ORACLE_SIM_TWRNS_TRACE: + status = oracleSimTwrNSBin_read_one_req(reader, req); + break; + case LCS_TRACE: + status = lcs_read_one_req(reader, req); + break; + case VALPIN_TRACE: + status = valpin_read_one_req(reader, req); + break; + case SYNTHETIC_TRACE: + status = synthetic_read_one_req(reader, req); + break; + default: + ERROR( + "cannot recognize reader trace_type, given reader trace_type: " + "%c\n", + reader->trace_type); + abort(); + } + } + + if (reader->sampler != NULL) { + /* we use this complex solution rather than simply recursive calls + because recursive calls can lead to stack overflow */ + sampler_t *sampler = reader->sampler; + reader->sampler = NULL; + while (!sampler->sample(sampler, req)) { + VERBOSE("skip one req: time %lu, obj_id %lu, size %lu at offset %zu\n", + req->clock_time, req->obj_id, req->obj_size, offset_before_read); + if (reader->read_direction == READ_FORWARD) { + status = read_one_req(reader, req); + } else { + status = read_one_req_above(reader, req); + } + if (status != 0) { + reader->sampler = sampler; + return status; + } + } + reader->sampler = sampler; + } + + if (reader->ignore_obj_size) { + req->obj_size = 1; + } + + if (reader->init_params.obj_cost_field <= 0) { + req->obj_cost = 1; + } + + VERBOSE( + "read one req: time %lu, obj_id %lu, size %lu, cost %lu at offset %zu\n", + req->clock_time, req->obj_id, req->obj_size, req->obj_cost, + offset_before_read); + + return status; +} diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 196ad7ad3..22b636cf9 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -33,6 +33,7 @@ endfunction() # Create C test executables add_test_executable(testReader test_traceReader.c) +add_test_executable(testSyntheticReader test_syntheticReader.c) add_test_executable(testDistUtils test_dist.c) add_test_executable(testProfilerLRU test_profilerLRU.c) add_test_executable(testSimulator test_simulator.c) @@ -51,6 +52,7 @@ add_test_executable(testUtils test_utils.c) add_test(NAME testReader COMMAND testReader WORKING_DIRECTORY .) +add_test(NAME testSyntheticReader COMMAND testSyntheticReader WORKING_DIRECTORY .) add_test(NAME testDistUtils COMMAND testDistUtils WORKING_DIRECTORY .) add_test(NAME testProfilerLRU COMMAND testProfilerLRU WORKING_DIRECTORY .) add_test(NAME testSimulator COMMAND testSimulator WORKING_DIRECTORY .) diff --git a/test/test_cli.sh b/test/test_cli.sh index ed8d434d4..51f052404 100755 --- a/test/test_cli.sh +++ b/test/test_cli.sh @@ -157,6 +157,44 @@ expect_output "cachesim hashes string object ids" "miss ratio 0\.6667" \ "${BIN_DIR}/cachesim" str-ids.csv csv lru 1mb \ -t "time-col=1,obj-id-col=2,obj-size-col=3,has-header=true" +echo "running synthetic reader tests" +for spec in Uniform_1000_10 Zipf_1000_10_0 Zipf_1000_10_1 \ + Zipf_1000_10_1.2_42 Zipf_1000_10_.5 Zipf_1000_10_1e-310; do + expect_output "synthetic ${spec}" '1000 req, miss ratio 0\.0100' \ + "${BIN_DIR}/cachesim" "${spec}" syn lru 10 --hashpower=8 --num-thread=1 +done +expect_output "synthetic cap of one" '1 req, miss ratio 1\.0000' \ + "${BIN_DIR}/cachesim" Uniform_100_10 syn lru 10 --num-req=1 --hashpower=8 +expect_output "synthetic large source count with cap" '10 req' \ + "${BIN_DIR}/cachesim" Uniform_4294967306_10 syn lru 10 --num-req=10 --hashpower=8 +expect_output "synthetic observed working set" 'working set size: 1 object 1 byte' \ + "${BIN_DIR}/cachesim" Uniform_1_1000 syn lru 0.5 --hashpower=8 +expect_ok "synthetic parallel caches" \ + "${BIN_DIR}/cachesim" Zipf_1000_100_0.9_42 syn lru,fifo 10,50 --num-thread=2 --hashpower=8 +expect_ok "synthetic spatial sampling" \ + "${BIN_DIR}/cachesim" Zipf_1000_100_0.9 syn lru 10 --sample-ratio=0.5 --hashpower=8 +for spec in '' x Uniform Uniform_0_10 Uniform_10_0 Uniform_-1_10 \ + Uniform_1x_10 Uniform_1_10x Uniform_1_2147483648 \ + Uniform_9223372036854775808_1 Uniform_1_1_18446744073709551616 \ + Uniform_1_1_-1 Uniform_1_1_ Uniform_1_1_2_extra \ + Zipf_10_10 Zipf_10_10_ Zipf_10_10_-1 Zipf_10_10_nan \ + Zipf_10_10_inf Zipf_10_10_1junk Zipf_10_10_0x1p0 \ + Zipf_10_10_1e999 Zipf_10_10_1e-999 Zipf_10_10_1_2_extra; do + # Require exit 1 explicitly: ERROR()/abort must not count as validation. + out=$("${BIN_DIR}/cachesim" "${spec}" syn lru 10 --hashpower=8 2>&1) + rc=$? + if [[ ${rc} -eq 1 ]] && grep -q 'error: invalid synthetic trace' <<<"${out}" && + ! grep -qE 'Sanitizer|runtime error:' <<<"${out}"; then + _report 0 "" + else + _report 1 "synthetic rejects '${spec}' cleanly (exit ${rc})" + fi +done +for param in time-col=1 obj-size-col=1 has-header=true obj-id-is-num=false; do + expect_clean_error "synthetic rejects ${param}" \ + "${BIN_DIR}/cachesim" Uniform_100_10 syn lru 10 -t "${param}" --hashpower=8 +done + echo "running -e print tests" # `-e print` runs before the cache is fully built, so the reporting path must diff --git a/test/test_syntheticReader.c b/test/test_syntheticReader.c new file mode 100644 index 000000000..119a28bc2 --- /dev/null +++ b/test/test_syntheticReader.c @@ -0,0 +1,375 @@ +#include +#include + +#include "common.h" +#include "traceReader/generalReader/syntheticRng.h" + +static reader_t *open_synthetic(const char *spec) { + return setup_reader(spec, SYNTHETIC_TRACE, NULL); +} + +static void read_id(reader_t *reader, request_t *req, uint64_t expected) { + g_assert_cmpint(read_one_req(reader, req), ==, 0); + g_assert_true(req->valid); + g_assert_cmpuint(req->obj_id, ==, expected); + g_assert_cmpint(req->obj_size, ==, 1); + g_assert_cmpint(req->obj_cost, ==, 1); + g_assert_cmpint(req->clock_time, ==, 0); + g_assert_cmpint(req->next_access_vtime, ==, -2); +} + +static void test_random_vectors(void) { + /* Published Philox2x64-10 known-answer vectors from + * Random123/tests/kat_vectors. Check the first output word, including + * full-width counter and key values. */ + g_assert_cmpuint(synthetic_random(0, 0, 0), ==, UINT64_C(0xca00a0459843d731)); + g_assert_cmpuint(synthetic_random(UINT64_MAX, UINT64_MAX, UINT64_MAX), ==, + UINT64_C(0x65b021d60cd8310f)); + g_assert_cmpuint(synthetic_random(UINT64_C(0x243f6a8885a308d3), + UINT64_C(0x13198a2e03707344), + UINT64_C(0xa4093822299f31d0)), + ==, UINT64_C(0x0a5e742c2997341c)); +} + +static void test_navigation(gconstpointer data) { + reader_t *reader = open_synthetic(data); + request_t *req = new_request(); + uint64_t ids[1000]; + g_assert_cmpint(get_num_of_req(reader), ==, 1000); + g_assert_null(reader->mapped_file); + g_assert_null(reader->file); + g_assert_cmpuint(reader->file_size, ==, 0); + for (size_t i = 0; i < G_N_ELEMENTS(ids); i++) { + g_assert_cmpint(read_one_req(reader, req), ==, 0); + g_assert_cmpuint(req->obj_id, <, 100); + ids[i] = req->obj_id; + } + for (int i = 0; i < 3; i++) { + g_assert_cmpint(read_one_req(reader, req), ==, 1); + g_assert_false(req->valid); + } + reset_reader(reader); + for (size_t i = 0; i < G_N_ELEMENTS(ids); i++) { + /* Consumers may reuse request objects; source fields must be restored. */ + req->clock_time = 100; + req->obj_size = 999; + req->obj_cost = 99; + req->next_access_vtime = 123; + next_rand(); + read_id(reader, req, ids[i]); + } + reset_reader(reader); + g_assert_cmpint(skip_n_req(reader, 123), ==, 123); + reader_t *clone = clone_reader(reader); + read_id(clone, req, ids[0]); + read_id(reader, req, ids[123]); + read_first_req(reader, req); + g_assert_cmpuint(req->obj_id, ==, ids[0]); + read_last_req(reader, req); + g_assert_cmpuint(req->obj_id, ==, ids[999]); + g_assert_cmpint(reader->n_read_req, ==, 1); + read_id(reader, req, ids[124]); + reader_set_read_pos(reader, 0.5); + read_id(reader, req, ids[500]); + reader_set_read_pos(reader, 1); + g_assert_cmpint(go_back_one_req(reader), ==, 0); + read_id(reader, req, ids[999]); + for (int i = 998; i >= 0; i--) { + g_assert_cmpint(read_one_req_above(reader, req), ==, 0); + g_assert_cmpuint(req->obj_id, ==, ids[i]); + } + g_assert_cmpint(read_one_req_above(reader, req), ==, 1); + reset_reader(reader); + g_assert_cmpint(skip_n_req(reader, 2000), ==, 1000); + g_assert_cmpint(skip_n_req(reader, 1), ==, 0); + g_assert_cmpint(skip_n_req(reader, -1), ==, 0); + reader_set_read_pos(reader, -1); + read_id(reader, req, ids[0]); + reader_set_read_pos(reader, NAN); + read_id(reader, req, ids[0]); + reader_set_read_pos(reader, INFINITY); + g_assert_cmpint(read_one_req(reader, req), ==, 1); + close_reader(reader); + /* A clone must own its state, including after the original is closed. */ + read_id(clone, req, ids[1]); + close_reader(clone); + free_request(req); +} + +static void test_parameters(void) { + request_t *req = new_request(); + reader_t *short_trace = open_synthetic("Uniform_100_17"); + reader_t *long_trace = open_synthetic("Zipf_200_17_0_1"); + reader_t *other_seed = open_synthetic("Uniform_100_17_2"); + request_t *other = new_request(); + bool different = false; + for (int i = 0; i < 100; i++) { + g_assert_cmpint(read_one_req(short_trace, req), ==, 0); + read_id(long_trace, other, req->obj_id); + read_one_req(other_seed, other); + different |= other->obj_id != req->obj_id; + } + g_assert_true(different); + close_reader(short_trace); + close_reader(long_trace); + close_reader(other_seed); + + reader_t *large = open_synthetic( + "Uniform_9223372036854775807_2147483647_18446744073709551615"); + g_assert_cmpint(get_num_of_req(large), ==, INT64_MAX); + reader_set_read_pos(large, 1); + g_assert_cmpint(go_back_one_req(large), ==, 0); + g_assert_cmpint(read_one_req(large, req), ==, 0); + g_assert_cmpuint(req->obj_id, <, INT32_MAX); + g_assert_cmpint(read_one_req(large, req), ==, 1); + close_reader(large); + + const char *edges[] = {"Zipf_100_1_1", "Zipf_100_2_100", + "Zipf_100_2147483647_1e300", + "Zipf_100_2147483647_1.7976931348623157e308"}; + for (size_t i = 0; i < G_N_ELEMENTS(edges); i++) { + feclearexcept(FE_ALL_EXCEPT); + reader_t *reader = open_synthetic(edges[i]); + for (int j = 0; j < 100; j++) read_id(reader, req, 0); + g_assert_cmpint(fetestexcept(FE_INVALID | FE_DIVBYZERO | FE_OVERFLOW), ==, + 0); + close_reader(reader); + } + free_request(other); + free_request(req); +} + +static void test_limits_and_sampling(void) { + const int caps[] = {-1, 0, 1, 2, 99, 100, 101}; + request_t *req = new_request(); + for (size_t i = 0; i < G_N_ELEMENTS(caps); i++) { + reader_init_param_t init = default_reader_init_params(); + init.cap_at_n_req = caps[i]; + reader_t *reader = setup_reader("Uniform_100_10", SYNTHETIC_TRACE, &init); + int expected = caps[i] > 0 ? MIN(caps[i], 100) : 100; + for (int pass = 0; pass < 2; pass++) { + int n = 0; + while (read_one_req(reader, req) == 0) n++; + g_assert_cmpint(n, ==, expected); + reset_reader(reader); + } + close_reader(reader); + } + + for (int temporal = 0; temporal < 2; temporal++) { + sampler_t *sampler = + temporal ? create_temporal_sampler(0.25) : create_spatial_sampler(0.25); + reader_init_param_t init = default_reader_init_params(); + init.sampler = sampler; + reader_t *sampled = + setup_reader("Zipf_1000_100_0.9", SYNTHETIC_TRACE, &init); + reader_t *raw = open_synthetic("Zipf_1000_100_0.9"); + uint64_t ids[1000]; + int count = 0; + while (read_one_req(raw, req) == 0) { + if (sampler->sample(sampler, req)) ids[count++] = req->obj_id; + } + sampler->free(sampler); + close_reader(raw); + g_assert_cmpint(count, >, 0); + for (int pass = 0; pass < 2; pass++) { + for (int i = 0; i < count; i++) read_id(sampled, req, ids[i]); + g_assert_cmpint(read_one_req(sampled, req), ==, 1); + reset_reader(sampled); + } + if (!temporal) { + read_id(sampled, req, ids[0]); + int64_t n_read = sampled->n_read_req; + read_first_req(sampled, req); + g_assert_cmpuint(req->obj_id, ==, ids[0]); + read_last_req(sampled, req); + g_assert_cmpuint(req->obj_id, ==, ids[count - 1]); + g_assert_cmpint(sampled->n_read_req, ==, n_read); + read_id(sampled, req, ids[1]); + reader_set_read_pos(sampled, 1); + sampled->read_direction = READ_BACKWARD; + g_assert_cmpint(go_back_one_req(sampled), ==, 0); + read_id(sampled, req, ids[count - 1]); + for (int i = count - 2; i >= 0; i--) { + g_assert_cmpint(read_one_req_above(sampled, req), ==, 0); + g_assert_cmpuint(req->obj_id, ==, ids[i]); + } + g_assert_cmpint(read_one_req_above(sampled, req), ==, 1); + sampled->read_direction = READ_FORWARD; + reset_reader(sampled); + } + reader_t *clone = clone_reader(sampled); + close_reader(sampled); + for (int i = 0; i < count; i++) read_id(clone, req, ids[i]); + close_reader(clone); + } + free_request(req); +} + +static void test_sampled_boundaries(void) { + request_t *req = new_request(); + sampler_t *sampler = create_spatial_sampler(0.25); + req->obj_id = 0; + req->hv = 0; + while (sampler->sample(sampler, req)) sampler->sampling_salt++; + reader_init_param_t init = default_reader_init_params(); + init.sampler = sampler; + reader_t *reader = setup_reader("Uniform_100_1", SYNTHETIC_TRACE, &init); + g_assert_cmpint(read_one_req(reader, req), ==, 1); + g_assert_false(req->valid); + read_first_req(reader, req); + g_assert_false(req->valid); + read_last_req(reader, req); + g_assert_false(req->valid); + close_reader(reader); + + const int caps[] = {1, 2, 17, 100}; + for (size_t i = 0; i < G_N_ELEMENTS(caps); i++) { + init.cap_at_n_req = caps[i]; + reader = setup_reader("Uniform_100_10", SYNTHETIC_TRACE, &init); + reader_t *raw = open_synthetic("Uniform_100_10"); + uint64_t ids[100]; + int count = 0; + for (int j = 0; j < caps[i]; j++) { + g_assert_cmpint(read_one_req(raw, req), ==, 0); + if (sampler->sample(sampler, req)) ids[count++] = req->obj_id; + } + for (int pass = 0; pass < 2; pass++) { + for (int j = 0; j < count; j++) read_id(reader, req, ids[j]); + g_assert_cmpint(read_one_req(reader, req), ==, 1); + g_assert_cmpint(reader->n_read_req, ==, caps[i]); + reset_reader(reader); + } + close_reader(raw); + close_reader(reader); + } + sampler->free(sampler); + free_request(req); +} + +static void test_distribution(void) { + /* Independent finite-PMF calculation, not the sampler's integral formula. + * Fixed seeds and generous 7-sigma bounds avoid flaky tail-bin tests. */ + const double alphas[] = {0, 1e-10, 0.8, 1 - 1e-9, 1, 1 + 1e-9, 1.2, 2, 8}; + const int n = 200000, m = 16; + request_t *req = new_request(); + for (size_t a = 0; a < G_N_ELEMENTS(alphas); a++) { + char spec[128]; + snprintf(spec, sizeof(spec), "Zipf_%d_%d_%.17g_42", n, m, alphas[a]); + reader_t *reader = open_synthetic(spec); + int counts[16] = {0}; + int same = 0; + uint64_t previous = UINT64_MAX; + for (int i = 0; i < n; i++) { + g_assert_cmpint(read_one_req(reader, req), ==, 0); + g_assert_cmpuint(req->obj_id, <, m); + counts[req->obj_id]++; + same += req->obj_id == previous; + previous = req->obj_id; + } + long double weights[16], sum = 0; + for (int k = 0; k < m; k++) { + weights[k] = powl(k + 1, -alphas[a]); + sum += weights[k]; + } + long double collision = 0; + int tail_count = 0; + long double tail_prob = 0; + for (int k = 0; k < m; k++) { + long double p = weights[k] / sum; + collision += p * p; + if (n * p < 100) { + tail_count += counts[k]; + tail_prob += p; + } else { + g_assert_true(fabsl(counts[k] - n * p) < + 7 * sqrtl(n * p * (1 - p)) + 2); + } + } + g_assert_true(fabsl(tail_count - n * tail_prob) < + 7 * sqrtl(n * tail_prob * (1 - tail_prob)) + 2); + /* Adjacent collision events overlap: use a conservative O(sqrt(n)) + * tolerance rather than assuming independent Bernoulli observations. */ + g_assert_true(fabsl(same - (n - 1) * collision) < 10 * sqrtl(n)); + close_reader(reader); + } + free_request(req); +} + +static void compare_stats(const cache_stat_t *a, const cache_stat_t *b) { + g_assert_cmpuint(a->n_req, ==, b->n_req); + g_assert_cmpuint(a->n_miss, ==, b->n_miss); + g_assert_cmpuint(a->n_req_byte, ==, b->n_req_byte); + g_assert_cmpuint(a->n_miss_byte, ==, b->n_miss_byte); +} + +static void test_materialized(gconstpointer data) { + reader_t *synthetic = open_synthetic(data); + request_t *req = new_request(); + char *path = NULL; + int fd = g_file_open_tmp("libcachesim-synthetic-XXXXXX", &path, NULL); + g_assert_cmpint(fd, >=, 0); + FILE *file = fdopen(fd, "w"); + g_assert_nonnull(file); + while (read_one_req(synthetic, req) == 0) { + fprintf(file, "%" PRIu64 "\n", req->obj_id); + } + g_assert_cmpint(fclose(file), ==, 0); + reader_init_param_t init = default_reader_init_params(); + reader_t *materialized = setup_reader(path, PLAIN_TXT_TRACE, &init); + request_t *other = new_request(); + reset_reader(synthetic); + while (read_one_req(synthetic, req) == 0) { + read_id(materialized, other, req->obj_id); + } + g_assert_cmpint(read_one_req(materialized, other), ==, 1); + + const char *algorithms[] = {"lru", "fifo", "s3fifo"}; + uint64_t sizes[] = {10, 20, 50, 100}; + common_cache_params_t cc = { + .cache_size = 10, .hashpower = 8, .consider_obj_metadata = false}; + for (size_t a = 0; a < G_N_ELEMENTS(algorithms); a++) { + cache_t *cache = create_cache_by_name(algorithms[a], cc, NULL); + for (int warmup = 0; warmup < 2; warmup++) { + double fraction = warmup ? 0.1 : 0; + cache_stat_t *serial = simulate_at_multi_sizes( + synthetic, cache, 4, sizes, NULL, fraction, 0, 1, false); + cache_stat_t *parallel = simulate_at_multi_sizes( + synthetic, cache, 4, sizes, NULL, fraction, 0, 4, false); + cache_stat_t *disk = simulate_at_multi_sizes( + materialized, cache, 4, sizes, NULL, fraction, 0, 2, false); + for (int i = 0; i < 4; i++) { + compare_stats(&serial[i], ¶llel[i]); + compare_stats(&serial[i], &disk[i]); + } + free(serial); + free(parallel); + free(disk); + } + cache->cache_free(cache); + } + free_request(other); + free_request(req); + close_reader(synthetic); + close_reader(materialized); + g_assert_cmpint(unlink(path), ==, 0); + g_free(path); +} + +int main(int argc, char **argv) { + g_test_init(&argc, &argv, NULL); + g_test_add_func("/synthetic/random-vectors", test_random_vectors); + g_test_add_data_func("/synthetic/uniform-navigation", "Uniform_1000_100_7", + test_navigation); + g_test_add_data_func("/synthetic/zipf-navigation", "Zipf_1000_100_0.9_7", + test_navigation); + g_test_add_func("/synthetic/parameters", test_parameters); + g_test_add_func("/synthetic/sampling-and-limits", test_limits_and_sampling); + g_test_add_func("/synthetic/sampled-boundaries", test_sampled_boundaries); + g_test_add_func("/synthetic/distributions", test_distribution); + g_test_add_data_func("/synthetic/uniform-materialized", "Uniform_5000_100_7", + test_materialized); + g_test_add_data_func("/synthetic/zipf-materialized", "Zipf_5000_100_0.9_7", + test_materialized); + return g_test_run(); +}