feat: scaffold FCES-native C++ project with libtorch integration
- CMakeLists.txt with libtorch, GoogleTest, GoogleBenchmark, OpenMP, pybind11 - Header files: config, controller, population, fitness, evolution, spectral, oscillation, telemetry, optimizer - Source implementations: controller (full micro-MLP forward pass, mutation, crossover), fitness (Welford's algorithm), oscillation (DFT), spectral (SVD rank), optimizer (sign-SGD stub) - Tests: controller, population, fitness, optimizer (Google Test) - Benchmarks: evolve throughput, optimizer step (Google Benchmark) - Examples: simple optimization, PyTorch/libtorch integration - Python extension: pybind11 bindings with setup.py - README with architecture diagram and build instructions
This commit is contained in:
46
.gitignore
vendored
Normal file
46
.gitignore
vendored
Normal file
@@ -0,0 +1,46 @@
|
||||
# Build directories
|
||||
build/
|
||||
cmake-build-*/
|
||||
out/
|
||||
|
||||
# Compiled objects
|
||||
*.o
|
||||
*.obj
|
||||
*.so
|
||||
*.dylib
|
||||
*.dll
|
||||
*.a
|
||||
*.lib
|
||||
*.exe
|
||||
|
||||
# IDE
|
||||
.vscode/
|
||||
.idea/
|
||||
*.swp
|
||||
*.swo
|
||||
*~
|
||||
.vs/
|
||||
|
||||
# CMake
|
||||
CMakeCache.txt
|
||||
CMakeFiles/
|
||||
cmake_install.cmake
|
||||
Makefile
|
||||
install_manifest.txt
|
||||
compile_commands.json
|
||||
CTestTestfile.cmake
|
||||
_deps/
|
||||
|
||||
# Python extension build
|
||||
python/build/
|
||||
python/dist/
|
||||
python/*.egg-info/
|
||||
__pycache__/
|
||||
*.pyc
|
||||
|
||||
# OS
|
||||
.DS_Store
|
||||
Thumbs.db
|
||||
|
||||
# libtorch download
|
||||
libtorch/
|
||||
167
CMakeLists.txt
Normal file
167
CMakeLists.txt
Normal file
@@ -0,0 +1,167 @@
|
||||
cmake_minimum_required(VERSION 3.18 FATAL_ERROR)
|
||||
project(fces-native
|
||||
VERSION 0.1.0
|
||||
DESCRIPTION "High-performance C++ FCES optimizer with libtorch integration"
|
||||
LANGUAGES CXX
|
||||
)
|
||||
|
||||
# ============================================================================
|
||||
# Build Configuration
|
||||
# ============================================================================
|
||||
set(CMAKE_CXX_STANDARD 17)
|
||||
set(CMAKE_CXX_STANDARD_REQUIRED ON)
|
||||
set(CMAKE_CXX_EXTENSIONS OFF)
|
||||
set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
|
||||
|
||||
# Options
|
||||
option(FCES_BUILD_TESTS "Build unit tests" ON)
|
||||
option(FCES_BUILD_BENCHMARKS "Build performance benchmarks" ON)
|
||||
option(FCES_BUILD_EXAMPLES "Build examples" ON)
|
||||
option(FCES_BUILD_PYTHON "Build Python extension" OFF)
|
||||
option(FCES_ENABLE_OPENMP "Enable OpenMP for parallel evolution" ON)
|
||||
|
||||
# ============================================================================
|
||||
# Dependencies
|
||||
# ============================================================================
|
||||
|
||||
# libtorch (PyTorch C++ API)
|
||||
find_package(Torch REQUIRED)
|
||||
|
||||
# OpenMP (optional, for parallel population evaluation)
|
||||
if(FCES_ENABLE_OPENMP)
|
||||
find_package(OpenMP)
|
||||
if(OpenMP_CXX_FOUND)
|
||||
message(STATUS "OpenMP found — parallel evolution enabled")
|
||||
else()
|
||||
message(STATUS "OpenMP not found — falling back to serial execution")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
# ============================================================================
|
||||
# FCES Core Library
|
||||
# ============================================================================
|
||||
add_library(fces STATIC
|
||||
src/config.cpp
|
||||
src/controller.cpp
|
||||
src/population.cpp
|
||||
src/fitness.cpp
|
||||
src/evolution.cpp
|
||||
src/spectral.cpp
|
||||
src/oscillation.cpp
|
||||
src/optimizer.cpp
|
||||
src/telemetry.cpp
|
||||
)
|
||||
|
||||
target_include_directories(fces
|
||||
PUBLIC
|
||||
$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
|
||||
$<INSTALL_INTERFACE:include>
|
||||
)
|
||||
|
||||
target_link_libraries(fces PUBLIC ${TORCH_LIBRARIES})
|
||||
|
||||
if(OpenMP_CXX_FOUND)
|
||||
target_link_libraries(fces PUBLIC OpenMP::OpenMP_CXX)
|
||||
target_compile_definitions(fces PUBLIC FCES_USE_OPENMP)
|
||||
endif()
|
||||
|
||||
# MSVC-specific flags
|
||||
if(MSVC)
|
||||
target_compile_options(fces PRIVATE /W4 /permissive-)
|
||||
else()
|
||||
target_compile_options(fces PRIVATE -Wall -Wextra -Wpedantic -O2)
|
||||
endif()
|
||||
|
||||
# ============================================================================
|
||||
# Tests (Google Test)
|
||||
# ============================================================================
|
||||
if(FCES_BUILD_TESTS)
|
||||
enable_testing()
|
||||
|
||||
include(FetchContent)
|
||||
FetchContent_Declare(
|
||||
googletest
|
||||
URL https://github.com/google/googletest/archive/refs/tags/v1.14.0.zip
|
||||
)
|
||||
# For Windows: Prevent overriding the parent project's compiler/linker settings
|
||||
set(gtest_force_shared_crt ON CACHE BOOL "" FORCE)
|
||||
FetchContent_MakeAvailable(googletest)
|
||||
|
||||
add_executable(fces_tests
|
||||
tests/test_controller.cpp
|
||||
tests/test_population.cpp
|
||||
tests/test_optimizer.cpp
|
||||
tests/test_fitness.cpp
|
||||
)
|
||||
|
||||
target_link_libraries(fces_tests PRIVATE fces GTest::gtest_main)
|
||||
|
||||
include(GoogleTest)
|
||||
gtest_discover_tests(fces_tests)
|
||||
endif()
|
||||
|
||||
# ============================================================================
|
||||
# Benchmarks (Google Benchmark)
|
||||
# ============================================================================
|
||||
if(FCES_BUILD_BENCHMARKS)
|
||||
include(FetchContent)
|
||||
FetchContent_Declare(
|
||||
benchmark
|
||||
URL https://github.com/google/benchmark/archive/refs/tags/v1.8.3.zip
|
||||
)
|
||||
set(BENCHMARK_ENABLE_TESTING OFF CACHE BOOL "" FORCE)
|
||||
FetchContent_MakeAvailable(benchmark)
|
||||
|
||||
add_executable(fces_bench
|
||||
benchmarks/bench_evolve.cpp
|
||||
benchmarks/bench_step.cpp
|
||||
)
|
||||
|
||||
target_link_libraries(fces_bench PRIVATE fces benchmark::benchmark_main)
|
||||
endif()
|
||||
|
||||
# ============================================================================
|
||||
# Examples
|
||||
# ============================================================================
|
||||
if(FCES_BUILD_EXAMPLES)
|
||||
add_executable(simple_optimization examples/simple_optimization.cpp)
|
||||
target_link_libraries(simple_optimization PRIVATE fces)
|
||||
|
||||
add_executable(pytorch_integration examples/pytorch_integration.cpp)
|
||||
target_link_libraries(pytorch_integration PRIVATE fces)
|
||||
endif()
|
||||
|
||||
# ============================================================================
|
||||
# Python Extension (pybind11)
|
||||
# ============================================================================
|
||||
if(FCES_BUILD_PYTHON)
|
||||
find_package(pybind11 REQUIRED)
|
||||
pybind11_add_module(fces_native python/fces_native.cpp)
|
||||
target_link_libraries(fces_native PRIVATE fces)
|
||||
endif()
|
||||
|
||||
# ============================================================================
|
||||
# Install
|
||||
# ============================================================================
|
||||
install(TARGETS fces
|
||||
EXPORT fcesTargets
|
||||
LIBRARY DESTINATION lib
|
||||
ARCHIVE DESTINATION lib
|
||||
RUNTIME DESTINATION bin
|
||||
INCLUDES DESTINATION include
|
||||
)
|
||||
|
||||
install(DIRECTORY include/fces DESTINATION include)
|
||||
|
||||
message(STATUS "")
|
||||
message(STATUS "=== FCES Native Build Configuration ===")
|
||||
message(STATUS " Version: ${PROJECT_VERSION}")
|
||||
message(STATUS " C++ Standard: ${CMAKE_CXX_STANDARD}")
|
||||
message(STATUS " Tests: ${FCES_BUILD_TESTS}")
|
||||
message(STATUS " Benchmarks: ${FCES_BUILD_BENCHMARKS}")
|
||||
message(STATUS " Examples: ${FCES_BUILD_EXAMPLES}")
|
||||
message(STATUS " Python: ${FCES_BUILD_PYTHON}")
|
||||
message(STATUS " OpenMP: ${OpenMP_CXX_FOUND}")
|
||||
message(STATUS " Torch: ${Torch_VERSION}")
|
||||
message(STATUS "========================================")
|
||||
message(STATUS "")
|
||||
21
LICENSE
Normal file
21
LICENSE
Normal file
@@ -0,0 +1,21 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2026 Sven
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
114
README.md
Normal file
114
README.md
Normal file
@@ -0,0 +1,114 @@
|
||||
# FCES-native
|
||||
|
||||
**High-performance C++ reimplementation of the Fuzzy Controlled Evolutionary Search (FCES) optimizer.**
|
||||
|
||||
FCES is a zero-memory evolutionary optimizer that replaces AdamW for neural network training, saving 100% of optimizer VRAM. This repository provides a native C++ implementation with libtorch integration for maximum performance.
|
||||
|
||||
## Features
|
||||
|
||||
- **Zero-State Overhead**: No per-parameter momentum/variance buffers (unlike Adam)
|
||||
- **Neuro-Evolutionary Search**: Population of fuzzy controllers evolved via genetic algorithms
|
||||
- **Spectral Sensing**: Grokking-aware rank minimization
|
||||
- **Born Quantized**: Quantization-aware evolutionary training
|
||||
- **libtorch Integration**: Drop-in replacement for PyTorch optimizers
|
||||
- **Python Extension**: pip-installable via pybind11 for seamless PyTorch interop
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
┌──────────────────────────────────────────────────────┐
|
||||
│ FCESOptimizer │
|
||||
│ ┌────────────────┐ ┌─────────────────────────────┐ │
|
||||
│ │ EvolutionMgr │ │ Parameter Update (libtorch) │ │
|
||||
│ │ ┌────────────┐ │ │ • sign(grad) │ │
|
||||
│ │ │ Population │ │ │ • trust region clipping │ │
|
||||
│ │ │ ┌────────┐ │ │ │ • weight decay │ │
|
||||
│ │ │ │Ctrl[0] │ │ │ └─────────────────────────────┘ │
|
||||
│ │ │ │Ctrl[1] │ │ │ ┌─────────────────────────────┐ │
|
||||
│ │ │ │ ... │ │ │ │ SpectralSensor │ │
|
||||
│ │ │ │Ctrl[N] │ │ │ │ • rank tracking │ │
|
||||
│ │ │ └────────┘ │ │ │ • grokking detection │ │
|
||||
│ │ └────────────┘ │ └─────────────────────────────┘ │
|
||||
│ │ • crossover │ ┌─────────────────────────────┐ │
|
||||
│ │ • mutation │ │ OscillationDetector (FFT) │ │
|
||||
│ │ • selection │ └─────────────────────────────┘ │
|
||||
│ └────────────────┘ │
|
||||
└──────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
## Building
|
||||
|
||||
### Prerequisites
|
||||
|
||||
- C++17 compiler (GCC 9+, Clang 10+, MSVC 2019+)
|
||||
- CMake 3.18+
|
||||
- libtorch (PyTorch C++ distribution)
|
||||
|
||||
### Build Steps
|
||||
|
||||
```bash
|
||||
# Download libtorch (Linux, CUDA 12.1)
|
||||
wget https://download.pytorch.org/libtorch/cu121/libtorch-cxx11-abi-shared-with-deps-2.3.0%2Bcu121.zip
|
||||
unzip libtorch-*.zip
|
||||
|
||||
# Configure & Build
|
||||
mkdir build && cd build
|
||||
cmake .. -DCMAKE_PREFIX_PATH=/path/to/libtorch
|
||||
cmake --build . --config Release -j$(nproc)
|
||||
|
||||
# Run tests
|
||||
ctest --output-on-failure
|
||||
```
|
||||
|
||||
### Windows (MSVC)
|
||||
|
||||
```powershell
|
||||
mkdir build; cd build
|
||||
cmake .. -DCMAKE_PREFIX_PATH="C:/path/to/libtorch" -G "Visual Studio 17 2022"
|
||||
cmake --build . --config Release
|
||||
ctest -C Release --output-on-failure
|
||||
```
|
||||
|
||||
## Python Extension
|
||||
|
||||
```bash
|
||||
cd python
|
||||
pip install .
|
||||
```
|
||||
|
||||
```python
|
||||
import torch
|
||||
from fces_native import FCESOptimizer
|
||||
|
||||
model = MyModel()
|
||||
optimizer = FCESOptimizer(model.parameters(), lr=1.6e-3, population_size=200)
|
||||
|
||||
for batch in dataloader:
|
||||
loss = model(batch)
|
||||
loss.backward()
|
||||
optimizer.step()
|
||||
optimizer.update_fitness(loss.item())
|
||||
optimizer.zero_grad()
|
||||
```
|
||||
|
||||
## Performance
|
||||
|
||||
Expected speedup over the Python implementation:
|
||||
|
||||
| Component | Python | C++ | Speedup |
|
||||
|-----------|--------|-----|---------|
|
||||
| `evolve()` (200 controllers) | ~2ms | ~20μs | ~100x |
|
||||
| `decide_update()` | ~0.5ms | ~5μs | ~100x |
|
||||
| End-to-end `step()` overhead | ~4ms | ~0.15ms | ~25x |
|
||||
|
||||
## Scientific Background
|
||||
|
||||
See [FCES SCIENCE.md](https://git.zky.de/sven/FCES/src/branch/main/SCIENCE.md) for the full scientific log documenting the evolution from V1.0 (hardcoded fuzzy rules) through V49.0 (born-quantized training).
|
||||
|
||||
## License
|
||||
|
||||
MIT License — See [LICENSE](LICENSE) for details.
|
||||
|
||||
## Related
|
||||
|
||||
- [FCES (Python)](https://git.zky.de/sven/FCES) — Original Python implementation
|
||||
45
benchmarks/bench_evolve.cpp
Normal file
45
benchmarks/bench_evolve.cpp
Normal file
@@ -0,0 +1,45 @@
|
||||
#include <benchmark/benchmark.h>
|
||||
#include "fces/population.hpp"
|
||||
#include "fces/controller.hpp"
|
||||
|
||||
using namespace fces;
|
||||
|
||||
static void BM_ControllerDecideUpdate(benchmark::State& state) {
|
||||
FuzzyController ctrl;
|
||||
std::vector<std::vector<float>> stats(state.range(0), {0.1f, 0.2f, 0.3f, 0.4f, 0.5f});
|
||||
|
||||
for (auto _ : state) {
|
||||
auto actions = ctrl.decide_update(stats, 0.0f, 0.5f, 0.0f, 0.1f, 0.0f, 0.0f, 1.0f, 0.0f);
|
||||
benchmark::DoNotOptimize(actions);
|
||||
}
|
||||
}
|
||||
BENCHMARK(BM_ControllerDecideUpdate)->Arg(10)->Arg(50)->Arg(200);
|
||||
|
||||
static void BM_Evolve(benchmark::State& state) {
|
||||
Population pop(state.range(0));
|
||||
|
||||
for (auto _ : state) {
|
||||
pop.evolve(2.0f, -0.01f, 0.5f);
|
||||
}
|
||||
}
|
||||
BENCHMARK(BM_Evolve)->Arg(50)->Arg(100)->Arg(200);
|
||||
|
||||
static void BM_Mutation(benchmark::State& state) {
|
||||
FuzzyController ctrl;
|
||||
|
||||
for (auto _ : state) {
|
||||
auto child = ctrl.mutate(2.0f, 1.0f);
|
||||
benchmark::DoNotOptimize(child);
|
||||
}
|
||||
}
|
||||
BENCHMARK(BM_Mutation);
|
||||
|
||||
static void BM_Crossover(benchmark::State& state) {
|
||||
FuzzyController a, b;
|
||||
|
||||
for (auto _ : state) {
|
||||
auto child = a.crossover(b);
|
||||
benchmark::DoNotOptimize(child);
|
||||
}
|
||||
}
|
||||
BENCHMARK(BM_Crossover);
|
||||
25
benchmarks/bench_step.cpp
Normal file
25
benchmarks/bench_step.cpp
Normal file
@@ -0,0 +1,25 @@
|
||||
#include <benchmark/benchmark.h>
|
||||
#include <torch/torch.h>
|
||||
#include "fces/optimizer.hpp"
|
||||
|
||||
using namespace fces;
|
||||
|
||||
static void BM_OptimizerStep(benchmark::State& state) {
|
||||
auto model = torch::nn::Linear(state.range(0), state.range(0) / 2);
|
||||
std::vector<torch::Tensor> params;
|
||||
for (auto& p : model->parameters()) params.push_back(p);
|
||||
|
||||
FCESOptimizer opt(params, FCESConfig{}.set_lr(1e-3f));
|
||||
|
||||
auto x = torch::randn({8, state.range(0)});
|
||||
|
||||
for (auto _ : state) {
|
||||
auto y = model->forward(x);
|
||||
auto loss = y.sum();
|
||||
loss.backward();
|
||||
opt.step();
|
||||
opt.zero_grad();
|
||||
benchmark::DoNotOptimize(loss);
|
||||
}
|
||||
}
|
||||
BENCHMARK(BM_OptimizerStep)->Arg(64)->Arg(256)->Arg(1024);
|
||||
61
examples/pytorch_integration.cpp
Normal file
61
examples/pytorch_integration.cpp
Normal file
@@ -0,0 +1,61 @@
|
||||
/**
|
||||
* @file pytorch_integration.cpp
|
||||
* @brief Example: train a small neural network with FCES via libtorch.
|
||||
*/
|
||||
|
||||
#include <iostream>
|
||||
#include <torch/torch.h>
|
||||
#include "fces/optimizer.hpp"
|
||||
|
||||
struct TinyNet : torch::nn::Module {
|
||||
torch::nn::Linear fc1{nullptr}, fc2{nullptr};
|
||||
|
||||
TinyNet() {
|
||||
fc1 = register_module("fc1", torch::nn::Linear(10, 32));
|
||||
fc2 = register_module("fc2", torch::nn::Linear(32, 1));
|
||||
}
|
||||
|
||||
torch::Tensor forward(torch::Tensor x) {
|
||||
x = torch::relu(fc1->forward(x));
|
||||
return fc2->forward(x);
|
||||
}
|
||||
};
|
||||
|
||||
int main() {
|
||||
auto model = std::make_shared<TinyNet>();
|
||||
|
||||
std::vector<torch::Tensor> params;
|
||||
for (auto& p : model->parameters()) params.push_back(p);
|
||||
|
||||
fces::FCESOptimizer optimizer(
|
||||
params,
|
||||
fces::FCESConfig{}
|
||||
.set_lr(1.6e-3f)
|
||||
.set_population_size(200)
|
||||
.set_total_steps(1000)
|
||||
);
|
||||
|
||||
// Generate synthetic regression data
|
||||
auto x_train = torch::randn({100, 10});
|
||||
auto y_train = torch::sin(x_train.sum(1, true));
|
||||
|
||||
for (int epoch = 0; epoch < 100; ++epoch) {
|
||||
optimizer.zero_grad();
|
||||
auto pred = model->forward(x_train);
|
||||
auto loss = torch::mse_loss(pred, y_train);
|
||||
loss.backward();
|
||||
optimizer.step();
|
||||
optimizer.update_fitness(loss.item<float>());
|
||||
|
||||
if (epoch % 10 == 0) {
|
||||
std::cout << "Epoch " << epoch
|
||||
<< " | Loss: " << loss.item<float>() << std::endl;
|
||||
}
|
||||
}
|
||||
|
||||
std::cout << "\nTraining complete. Final loss: "
|
||||
<< torch::mse_loss(model->forward(x_train), y_train).item<float>()
|
||||
<< std::endl;
|
||||
|
||||
return 0;
|
||||
}
|
||||
36
examples/simple_optimization.cpp
Normal file
36
examples/simple_optimization.cpp
Normal file
@@ -0,0 +1,36 @@
|
||||
/**
|
||||
* @file simple_optimization.cpp
|
||||
* @brief Minimal example: optimize a quadratic function with FCES.
|
||||
*/
|
||||
|
||||
#include <iostream>
|
||||
#include <torch/torch.h>
|
||||
#include "fces/optimizer.hpp"
|
||||
|
||||
int main() {
|
||||
// Target: minimize f(x) = ||x - target||^2
|
||||
auto target = torch::tensor({1.0f, 2.0f, 3.0f, 4.0f, 5.0f});
|
||||
auto x = torch::randn({5}, torch::requires_grad());
|
||||
|
||||
std::vector<torch::Tensor> params = {x};
|
||||
fces::FCESOptimizer optimizer(params, fces::FCESConfig{}.set_lr(1e-2f));
|
||||
|
||||
for (int step = 0; step < 500; ++step) {
|
||||
optimizer.zero_grad();
|
||||
auto loss = (x - target).pow(2).sum();
|
||||
loss.backward();
|
||||
optimizer.step();
|
||||
optimizer.update_fitness(loss.item<float>());
|
||||
|
||||
if (step % 50 == 0) {
|
||||
std::cout << "Step " << step
|
||||
<< " | Loss: " << loss.item<float>()
|
||||
<< " | x: " << x << std::endl;
|
||||
}
|
||||
}
|
||||
|
||||
std::cout << "\nFinal x: " << x << std::endl;
|
||||
std::cout << "Target: " << target << std::endl;
|
||||
|
||||
return 0;
|
||||
}
|
||||
82
include/fces/config.hpp
Normal file
82
include/fces/config.hpp
Normal file
@@ -0,0 +1,82 @@
|
||||
#pragma once
|
||||
|
||||
/**
|
||||
* @file config.hpp
|
||||
* @brief FCES Configuration — compile-time defaults and runtime overrides.
|
||||
*
|
||||
* Maps directly from Python's FCESConfig (Pydantic model) to a C++ struct
|
||||
* with constexpr defaults and builder-pattern construction.
|
||||
*/
|
||||
|
||||
#include <cstdint>
|
||||
#include <string>
|
||||
|
||||
namespace fces {
|
||||
|
||||
/**
|
||||
* Core configuration for the FCES optimizer.
|
||||
* All fields have sensible defaults matching the Python V49.0 implementation.
|
||||
*/
|
||||
struct FCESConfig {
|
||||
// Learning rate (V49 optimal default)
|
||||
float lr = 1.6e-3f;
|
||||
|
||||
// Weight decay coefficient
|
||||
float weight_decay = 0.0f;
|
||||
|
||||
// Population size for evolutionary search
|
||||
int population_size = 200;
|
||||
|
||||
// Total training steps (for progress-aware scheduling)
|
||||
int total_steps = 5000;
|
||||
|
||||
// Signal mode for loss velocity calculation
|
||||
std::string signal_mode = "relative";
|
||||
|
||||
// Grokking awareness coefficient (0.0 = disabled)
|
||||
float grokking_coefficient = 0.1f;
|
||||
|
||||
// Spectral sensing frequency (every N steps)
|
||||
int spectral_frequency = 10;
|
||||
|
||||
// Curriculum Spectral Regularization
|
||||
bool csr_enabled = false;
|
||||
int csr_warmup_steps = 500;
|
||||
int csr_ramp_steps = 1000;
|
||||
|
||||
// Trust region clipping
|
||||
float trust_region_clip = 0.01f;
|
||||
|
||||
// Rollback threshold
|
||||
float rollback_threshold = 1.5f;
|
||||
|
||||
// Adaptive weight decay
|
||||
bool adaptive_wd = false;
|
||||
|
||||
// Parasitic mode (gradient alignment reward)
|
||||
bool parasitic_mode = false;
|
||||
|
||||
// Ablation mode: "", "force_sign", "force_grad"
|
||||
std::string ablation_mode = "";
|
||||
|
||||
// Fractional factorial scoring (CRO trick)
|
||||
bool use_fractional_scoring = false;
|
||||
|
||||
// Direct construction mode (pop_size=1)
|
||||
bool direct_construction = false;
|
||||
|
||||
// Banach-Tarski fission
|
||||
bool use_banach_fission = false;
|
||||
|
||||
// Auto-population (stabilize on divergence)
|
||||
bool auto_population = false;
|
||||
|
||||
// Builder pattern
|
||||
FCESConfig& set_lr(float v) { lr = v; return *this; }
|
||||
FCESConfig& set_population_size(int v) { population_size = v; return *this; }
|
||||
FCESConfig& set_total_steps(int v) { total_steps = v; return *this; }
|
||||
FCESConfig& set_grokking_coefficient(float v) { grokking_coefficient = v; return *this; }
|
||||
FCESConfig& set_direct_construction(bool v) { direct_construction = v; return *this; }
|
||||
};
|
||||
|
||||
} // namespace fces
|
||||
145
include/fces/controller.hpp
Normal file
145
include/fces/controller.hpp
Normal file
@@ -0,0 +1,145 @@
|
||||
#pragma once
|
||||
|
||||
/**
|
||||
* @file controller.hpp
|
||||
* @brief FuzzyController and Genome — the decision-making units of FCES.
|
||||
*
|
||||
* Each controller contains a Genome (neural network weights) that maps
|
||||
* layer statistics to update decisions (multiplier, sign_gate, wd_mult).
|
||||
*
|
||||
* Port of: packages/fces/core/controller.py
|
||||
*/
|
||||
|
||||
#include <array>
|
||||
#include <cstdint>
|
||||
#include <memory>
|
||||
#include <random>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include <torch/torch.h>
|
||||
|
||||
namespace fces {
|
||||
|
||||
// Controller input dimension (layer stats features)
|
||||
constexpr int GENOME_INPUT_DIM = 9;
|
||||
// Controller hidden dimension
|
||||
constexpr int GENOME_HIDDEN_DIM = 16;
|
||||
// Controller output dimension: [multiplier, sign_gate, wd_mult]
|
||||
constexpr int GENOME_OUTPUT_DIM = 3;
|
||||
// Total genome size: input->hidden weights + hidden biases + hidden->output weights + output biases
|
||||
constexpr int GENOME_SIZE =
|
||||
(GENOME_INPUT_DIM * GENOME_HIDDEN_DIM) + // input -> hidden weights
|
||||
GENOME_HIDDEN_DIM + // hidden biases
|
||||
(GENOME_HIDDEN_DIM * GENOME_OUTPUT_DIM) + // hidden -> output weights
|
||||
GENOME_OUTPUT_DIM; // output biases
|
||||
|
||||
/**
|
||||
* Genome — the "DNA" of a fuzzy controller.
|
||||
* A flat array of floats encoding a micro-MLP.
|
||||
*/
|
||||
struct Genome {
|
||||
std::array<float, GENOME_SIZE> weights{};
|
||||
std::array<float, GENOME_SIZE> gene_success{};
|
||||
float sigma_gene = 0.1f;
|
||||
float plasticity = 1.0f;
|
||||
|
||||
/// Initialize with random weights from a normal distribution
|
||||
void randomize(std::mt19937& rng);
|
||||
|
||||
/// Deep copy
|
||||
Genome clone() const;
|
||||
};
|
||||
|
||||
/**
|
||||
* FuzzyController — a single agent in the evolutionary population.
|
||||
*
|
||||
* Lifecycle:
|
||||
* 1. Created via random initialization or crossover/mutation
|
||||
* 2. Activated for `selection_interval` steps
|
||||
* 3. Evaluated based on loss improvement during its tenure
|
||||
* 4. Evolved (crossover/mutation) or culled based on fitness
|
||||
*/
|
||||
class FuzzyController {
|
||||
public:
|
||||
/// Unique identifier
|
||||
uint64_t id;
|
||||
|
||||
/// The neural genome
|
||||
Genome genome;
|
||||
|
||||
/// Fitness scores
|
||||
float fitness = 0.0f;
|
||||
float lifetime_fitness = 0.0f;
|
||||
float ema_fitness = 0.0f;
|
||||
int evaluation_count = 0;
|
||||
int age = 0;
|
||||
|
||||
/// Origin tracking
|
||||
std::string origin = "random";
|
||||
|
||||
/// Trust region violation counter
|
||||
int trust_violations = 0;
|
||||
|
||||
/// Rolling fitness history (for Phase 23 strategies)
|
||||
std::vector<float> fitness_history;
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// Construction
|
||||
// ---------------------------------------------------------------
|
||||
|
||||
FuzzyController();
|
||||
explicit FuzzyController(Genome genome);
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// Core Operations
|
||||
// ---------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Forward pass through the micro-MLP to produce update decisions.
|
||||
*
|
||||
* @param layer_stats Vector of per-layer feature maps
|
||||
* @param loss_trend Current loss velocity
|
||||
* @param step_pct Training progress [0, 1]
|
||||
* @param rollback_rate Rolling average rollback frequency
|
||||
* @param grad_stability Gradient coefficient of variation
|
||||
* @param spectral_alpha Log spectral rank
|
||||
* @param stagnation_intensity Stagnation counter / 500
|
||||
* @param kzm_damping Kibble-Zurek damping factor
|
||||
* @param projected_drift Projected loss drift
|
||||
* @return Tensor of shape [num_groups, 3] — (mult, sign_gate, wd_mult)
|
||||
*/
|
||||
torch::Tensor decide_update(
|
||||
const std::vector<std::vector<float>>& layer_stats,
|
||||
float loss_trend,
|
||||
float step_pct,
|
||||
float rollback_rate,
|
||||
float grad_stability,
|
||||
float spectral_alpha,
|
||||
float stagnation_intensity,
|
||||
float kzm_damping,
|
||||
float projected_drift
|
||||
);
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// Evolutionary Operators
|
||||
// ---------------------------------------------------------------
|
||||
|
||||
/// Create a mutated child
|
||||
FuzzyController mutate(float current_loss, float sigma_scale = 1.0f) const;
|
||||
|
||||
/// Crossover with another controller
|
||||
FuzzyController crossover(const FuzzyController& partner, bool use_alignment = true) const;
|
||||
|
||||
/// Create an orthogonal counter-strategy (Phoenix Rebirth)
|
||||
FuzzyController create_orthogonal_child(float intensity = 1.0f) const;
|
||||
|
||||
/// Banach-Tarski fission: split into two complementary children
|
||||
std::pair<FuzzyController, FuzzyController> banach_tarski_fission(float intensity = 1.0f) const;
|
||||
|
||||
private:
|
||||
static std::atomic<uint64_t> next_id_;
|
||||
static thread_local std::mt19937 rng_;
|
||||
};
|
||||
|
||||
} // namespace fces
|
||||
54
include/fces/evolution.hpp
Normal file
54
include/fces/evolution.hpp
Normal file
@@ -0,0 +1,54 @@
|
||||
#pragma once
|
||||
|
||||
/**
|
||||
* @file evolution.hpp
|
||||
* @brief EvolutionManager — orchestrates population dynamics.
|
||||
*
|
||||
* Port of: packages/fces/core/evolution_manager.py
|
||||
*/
|
||||
|
||||
#include "population.hpp"
|
||||
|
||||
namespace fces {
|
||||
|
||||
/**
|
||||
* EvolutionManager — controls the scheduling of evolutionary operations.
|
||||
*
|
||||
* Responsibilities:
|
||||
* - Controller rotation (sticky selection with interval)
|
||||
* - Population dynamics (auto-sizing, lockdown)
|
||||
* - Triggering evolution at intervals
|
||||
*/
|
||||
class EvolutionManager {
|
||||
public:
|
||||
explicit EvolutionManager(
|
||||
Population& population,
|
||||
int selection_interval = 50,
|
||||
bool auto_population = false,
|
||||
bool direct_construction = false
|
||||
);
|
||||
|
||||
/// Get the currently active controller
|
||||
FuzzyController& get_active_controller();
|
||||
|
||||
/// Update population dynamics based on current training state
|
||||
void update_population_dynamics(
|
||||
float loss_velocity,
|
||||
float ema_loss,
|
||||
int step_counter,
|
||||
int total_steps
|
||||
);
|
||||
|
||||
/// Steps the active controller has been in control
|
||||
int steps_active = 0;
|
||||
|
||||
/// Selection interval (how long a controller stays active)
|
||||
int selection_interval;
|
||||
|
||||
private:
|
||||
Population& population_;
|
||||
bool auto_population_;
|
||||
bool direct_construction_;
|
||||
};
|
||||
|
||||
} // namespace fces
|
||||
90
include/fces/fitness.hpp
Normal file
90
include/fces/fitness.hpp
Normal file
@@ -0,0 +1,90 @@
|
||||
#pragma once
|
||||
|
||||
/**
|
||||
* @file fitness.hpp
|
||||
* @brief Fitness evaluation — loss signal processing and multi-objective evaluation.
|
||||
*
|
||||
* Port of: packages/fces/core/fitness_engine.py + fitness.py
|
||||
*/
|
||||
|
||||
#include <cmath>
|
||||
#include <vector>
|
||||
|
||||
namespace fces {
|
||||
|
||||
/**
|
||||
* Running statistics tracker (Welford's algorithm).
|
||||
* Thread-safe, O(1) memory, numerically stable.
|
||||
*/
|
||||
class RunningStats {
|
||||
public:
|
||||
void update(float value);
|
||||
float z_score(float value) const;
|
||||
float get_mean() const { return mean_; }
|
||||
float get_std() const;
|
||||
int get_count() const { return count_; }
|
||||
|
||||
void reset();
|
||||
|
||||
private:
|
||||
int count_ = 0;
|
||||
float mean_ = 0.0f;
|
||||
float m2_ = 0.0f;
|
||||
};
|
||||
|
||||
/**
|
||||
* FitnessEngine — processes raw loss values into controller fitness signals.
|
||||
*/
|
||||
class FitnessEngine {
|
||||
public:
|
||||
explicit FitnessEngine(float grokking_coefficient = 0.1f);
|
||||
|
||||
/**
|
||||
* Calculate loss velocity signal.
|
||||
*
|
||||
* @param current_loss Current step loss
|
||||
* @param ema_loss Exponential moving average loss
|
||||
* @param mode "relative" or "absolute"
|
||||
* @return Velocity signal (negative = improving)
|
||||
*/
|
||||
float calculate_loss_signal(float current_loss, float ema_loss, const std::string& mode = "relative") const;
|
||||
|
||||
/**
|
||||
* Compute Kibble-Zurek Mechanism damping factor.
|
||||
* Prevents topological defects during phase transitions.
|
||||
*/
|
||||
float compute_kzm_damping(float spectral_alpha) const;
|
||||
|
||||
private:
|
||||
float grokking_coefficient_;
|
||||
};
|
||||
|
||||
/**
|
||||
* Fitness metrics for multi-objective evaluation.
|
||||
*/
|
||||
struct FitnessMetrics {
|
||||
float loss_improvement = 0.0f;
|
||||
float sparsity_score = 0.0f;
|
||||
float stability_score = 0.0f;
|
||||
float novelty_score = 0.0f;
|
||||
|
||||
/// Weighted combination
|
||||
float total(float alpha = 0.7f, float beta = 0.3f) const {
|
||||
return alpha * loss_improvement + beta * sparsity_score;
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* FuzzyFitnessEvaluator — multi-objective fitness evaluation with fuzzy weighting.
|
||||
*/
|
||||
class FuzzyFitnessEvaluator {
|
||||
public:
|
||||
FitnessMetrics evaluate(
|
||||
float loss_before,
|
||||
float loss_after,
|
||||
float sparsity = 0.0f,
|
||||
float val_loss = -1.0f
|
||||
) const;
|
||||
};
|
||||
|
||||
} // namespace fces
|
||||
87
include/fces/optimizer.hpp
Normal file
87
include/fces/optimizer.hpp
Normal file
@@ -0,0 +1,87 @@
|
||||
#pragma once
|
||||
|
||||
/**
|
||||
* @file optimizer.hpp
|
||||
* @brief FCESOptimizer — the main entry point. libtorch-compatible optimizer.
|
||||
*/
|
||||
|
||||
#include <torch/torch.h>
|
||||
#include <memory>
|
||||
#include <vector>
|
||||
#include <string>
|
||||
|
||||
#include "config.hpp"
|
||||
#include "population.hpp"
|
||||
#include "fitness.hpp"
|
||||
#include "evolution.hpp"
|
||||
#include "spectral.hpp"
|
||||
#include "oscillation.hpp"
|
||||
#include "telemetry.hpp"
|
||||
|
||||
namespace fces {
|
||||
|
||||
/**
|
||||
* FCESOptimizer — Fuzzy Controlled Evolutionary Search V49.0 (C++ Port).
|
||||
*
|
||||
* Usage:
|
||||
* auto optimizer = FCESOptimizer(model->parameters(), FCESConfig{}.set_lr(1.6e-3));
|
||||
* // In training loop:
|
||||
* optimizer.zero_grad();
|
||||
* auto loss = model->forward(input);
|
||||
* loss.backward();
|
||||
* optimizer.step();
|
||||
* optimizer.update_fitness(loss.item<float>());
|
||||
*/
|
||||
class FCESOptimizer : public torch::optim::Optimizer {
|
||||
public:
|
||||
explicit FCESOptimizer(
|
||||
std::vector<torch::Tensor> params,
|
||||
FCESConfig config = FCESConfig{}
|
||||
);
|
||||
|
||||
/// Perform a single optimization step
|
||||
torch::Tensor step(LossClosure closure = nullptr) override;
|
||||
|
||||
/// Update evolutionary fitness with current loss
|
||||
void update_fitness(float loss);
|
||||
|
||||
/// Backup model weights to CPU RAM
|
||||
void backup_to_ram();
|
||||
|
||||
/// Restore model weights from CPU RAM backup
|
||||
void restore_from_ram();
|
||||
|
||||
/// Get current step count
|
||||
int step_count() const { return step_counter_; }
|
||||
|
||||
/// Calculate model sparsity
|
||||
float calculate_sparsity() const;
|
||||
|
||||
private:
|
||||
FCESConfig config_;
|
||||
Population population_;
|
||||
FitnessEngine fitness_engine_;
|
||||
FuzzyFitnessEvaluator fitness_evaluator_;
|
||||
std::unique_ptr<EvolutionManager> evolution_manager_;
|
||||
OscillationDetector oscillation_detector_;
|
||||
RunningStats grad_norm_tracker_;
|
||||
|
||||
// State
|
||||
int step_counter_ = 0;
|
||||
float ema_loss_ = 0.0f;
|
||||
float last_step_loss_ = 0.0f;
|
||||
float best_loss_window_ = std::numeric_limits<float>::infinity();
|
||||
float rollback_ema_ = 0.0f;
|
||||
int stagnation_counter_ = 0;
|
||||
float last_loss_velocity_ = 0.0f;
|
||||
|
||||
// RAM backup
|
||||
std::vector<torch::Tensor> ram_backup_;
|
||||
|
||||
// Internal methods
|
||||
void gather_stats();
|
||||
void apply_parameter_updates(const torch::Tensor& actions);
|
||||
void handle_rollback();
|
||||
};
|
||||
|
||||
} // namespace fces
|
||||
29
include/fces/oscillation.hpp
Normal file
29
include/fces/oscillation.hpp
Normal file
@@ -0,0 +1,29 @@
|
||||
#pragma once
|
||||
|
||||
/**
|
||||
* @file oscillation.hpp
|
||||
* @brief FFT-based oscillation detection (Phase 25).
|
||||
*/
|
||||
|
||||
#include <vector>
|
||||
|
||||
namespace fces {
|
||||
|
||||
class OscillationDetector {
|
||||
public:
|
||||
static constexpr int WINDOW_SIZE = 64;
|
||||
static constexpr float POWER_THRESHOLD = 0.5f;
|
||||
|
||||
void update(float loss);
|
||||
bool detect() const;
|
||||
float get_score() const;
|
||||
float get_variance_50() const;
|
||||
void reset();
|
||||
|
||||
private:
|
||||
std::vector<float> loss_history_;
|
||||
static std::vector<float> detrend(const std::vector<float>& signal);
|
||||
static std::vector<float> compute_power_spectrum(const std::vector<float>& signal);
|
||||
};
|
||||
|
||||
} // namespace fces
|
||||
165
include/fces/population.hpp
Normal file
165
include/fces/population.hpp
Normal file
@@ -0,0 +1,165 @@
|
||||
#pragma once
|
||||
|
||||
/**
|
||||
* @file population.hpp
|
||||
* @brief Population management — the evolutionary ecosystem.
|
||||
*
|
||||
* Manages a population of FuzzyControllers with:
|
||||
* - Elitism (protected top-N)
|
||||
* - Tournament selection with age weighting
|
||||
* - Crossover, mutation, and Phoenix rebirth
|
||||
* - Island migration (optional)
|
||||
* - Novelty search (optional)
|
||||
* - Phase 23 stale elite mitigation strategies
|
||||
* - Phase 24 violator synchronization
|
||||
*
|
||||
* Port of: packages/fces/core/population.py (~1260 LOC)
|
||||
*/
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include <optional>
|
||||
|
||||
#include "controller.hpp"
|
||||
|
||||
namespace fces {
|
||||
|
||||
/**
|
||||
* Elite selection strategy for stale elite mitigation (Phase 23).
|
||||
*/
|
||||
enum class EliteStrategy {
|
||||
Cumulative, // Raw cumulative fitness
|
||||
EMA, // Exponential moving average
|
||||
Rolling, // Rolling window average
|
||||
Reset, // Periodic reset every 500 steps
|
||||
AgePenalty // fitness / log(age + 2)
|
||||
};
|
||||
|
||||
/**
|
||||
* Population — manages the ecosystem of FuzzyControllers.
|
||||
*/
|
||||
class Population {
|
||||
public:
|
||||
// Configuration constants
|
||||
static constexpr int ELITE_COUNT = 2;
|
||||
static constexpr float NOVELTY_WEIGHT = 0.1f;
|
||||
static constexpr float ISLAND_MIGRATION_RATE = 0.05f;
|
||||
static constexpr int BEHAVIORAL_ARCHIVE_SIZE = 100;
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// Construction
|
||||
// ---------------------------------------------------------------
|
||||
|
||||
explicit Population(
|
||||
int active_size = 75,
|
||||
int repo_size = 10000,
|
||||
EliteStrategy elite_strategy = EliteStrategy::Cumulative,
|
||||
bool link_mutation = false,
|
||||
bool link_elite = false,
|
||||
bool link_violator = false,
|
||||
bool use_fuzzy_pacer = false,
|
||||
bool use_fuzzy_importance = false,
|
||||
bool direct_construction = false,
|
||||
bool use_banach_fission = false
|
||||
);
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// Core API
|
||||
// ---------------------------------------------------------------
|
||||
|
||||
/// Get the currently active controller (sticky selection)
|
||||
FuzzyController& get_active_controller();
|
||||
|
||||
/// Select a controller via fitness-weighted tournament
|
||||
FuzzyController& select_weighted();
|
||||
|
||||
/// Get the best controller in the active population
|
||||
FuzzyController& get_best_active();
|
||||
|
||||
/// Get the worst non-elite controller
|
||||
FuzzyController& get_worst_active();
|
||||
|
||||
/// Remove a specific controller (unless elite)
|
||||
void kill(FuzzyController& controller);
|
||||
|
||||
/// Update a controller's fitness
|
||||
void update_controller_fitness(FuzzyController& controller, float reward, bool increment_eval = true);
|
||||
|
||||
/// Mark a controller as a violator (rollback)
|
||||
void mark_violated(FuzzyController& controller);
|
||||
|
||||
/// Get the effective fitness considering elite strategy and training progress
|
||||
float get_effective_fitness(const FuzzyController& controller, float training_progress) const;
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// Evolution
|
||||
// ---------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Evolve the population: select parents, crossover/mutate, replace worst.
|
||||
*
|
||||
* @param current_loss Current training loss
|
||||
* @param velocity Loss velocity
|
||||
* @param training_progress Training progress [0, 1]
|
||||
*/
|
||||
void evolve(float current_loss, float velocity = 0.0f, float training_progress = 0.0f);
|
||||
|
||||
/// Resize the population (dynamic expansion/contraction)
|
||||
void resize(int target_size, float training_progress = 0.5f);
|
||||
|
||||
/// Reduce mutation variance after rollback
|
||||
void calm_down();
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// Accessors
|
||||
// ---------------------------------------------------------------
|
||||
|
||||
int size() const { return static_cast<int>(gladiators_.size()); }
|
||||
float global_sigma_modifier() const { return global_sigma_modifier_; }
|
||||
|
||||
/// Compute diversity index (behavioral spread)
|
||||
float get_diversity_index() const;
|
||||
|
||||
/// Serialization
|
||||
// TODO: state_dict / load_state_dict
|
||||
|
||||
private:
|
||||
std::vector<FuzzyController> gladiators_;
|
||||
std::vector<FuzzyController> repository_;
|
||||
std::vector<FuzzyController> violated_controllers_;
|
||||
|
||||
float global_sigma_modifier_ = 1.0f;
|
||||
|
||||
// Sticky controller selection
|
||||
FuzzyController* active_controller_ = nullptr;
|
||||
int steps_active_ = 0;
|
||||
int selection_interval_ = 20;
|
||||
|
||||
// Configuration
|
||||
EliteStrategy elite_strategy_;
|
||||
bool link_mutation_;
|
||||
bool link_elite_;
|
||||
bool link_violator_;
|
||||
bool use_fuzzy_pacer_;
|
||||
bool use_fuzzy_importance_;
|
||||
bool direct_construction_;
|
||||
bool use_banach_fission_;
|
||||
|
||||
// Novelty search
|
||||
std::vector<std::vector<float>> behavioral_archive_;
|
||||
|
||||
// Fitness history for fuzzy pacer
|
||||
std::vector<float> fitness_history_;
|
||||
|
||||
// Phase 23: periodic reset counter
|
||||
int reset_step_counter_ = 0;
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// Internal
|
||||
// ---------------------------------------------------------------
|
||||
|
||||
std::vector<FuzzyController*> get_elites();
|
||||
void add_to_repository(const FuzzyController& controller);
|
||||
};
|
||||
|
||||
} // namespace fces
|
||||
53
include/fces/spectral.hpp
Normal file
53
include/fces/spectral.hpp
Normal file
@@ -0,0 +1,53 @@
|
||||
#pragma once
|
||||
|
||||
/**
|
||||
* @file spectral.hpp
|
||||
* @brief Spectral Sensor and Controller — grokking awareness via rank tracking.
|
||||
*
|
||||
* Port of: packages/fces/core/spectral_sensor.py + spectral_controller.py
|
||||
*/
|
||||
|
||||
#include <string>
|
||||
#include <unordered_map>
|
||||
#include <vector>
|
||||
|
||||
#include <torch/torch.h>
|
||||
|
||||
namespace fces {
|
||||
|
||||
/**
|
||||
* SpectralSensor — tracks the effective rank of weight matrices.
|
||||
*
|
||||
* Used for grokking detection: when the model transitions from
|
||||
* memorization to generalization, the effective rank drops.
|
||||
*/
|
||||
class SpectralSensor {
|
||||
public:
|
||||
explicit SpectralSensor(torch::nn::Module& model);
|
||||
|
||||
/// Track a layer's weight tensor
|
||||
void track_layer(const std::string& name, const torch::Tensor& weight);
|
||||
|
||||
/// Get the global (average) effective rank
|
||||
float get_global_rank() const;
|
||||
|
||||
/// Reset all tracked layers
|
||||
void reset();
|
||||
|
||||
private:
|
||||
std::unordered_map<std::string, float> layer_ranks_;
|
||||
|
||||
/// Compute effective rank via SVD
|
||||
static float compute_effective_rank(const torch::Tensor& weight);
|
||||
};
|
||||
|
||||
/**
|
||||
* SpectralController — converts spectral rank into optimizer decisions.
|
||||
*/
|
||||
class SpectralController {
|
||||
public:
|
||||
/// Compute the spectral alpha (gating factor for rank-aware updates)
|
||||
float compute_alpha(float global_rank, float grokking_coefficient) const;
|
||||
};
|
||||
|
||||
} // namespace fces
|
||||
26
include/fces/telemetry.hpp
Normal file
26
include/fces/telemetry.hpp
Normal file
@@ -0,0 +1,26 @@
|
||||
#pragma once
|
||||
|
||||
/**
|
||||
* @file telemetry.hpp
|
||||
* @brief Structured logging and telemetry for FCES.
|
||||
*/
|
||||
|
||||
#include <string>
|
||||
|
||||
namespace fces {
|
||||
|
||||
class Telemetry {
|
||||
public:
|
||||
static Telemetry& get();
|
||||
|
||||
void info(const std::string& event, const std::string& detail = "");
|
||||
void warning(const std::string& event, const std::string& detail = "");
|
||||
void error(const std::string& event, const std::string& detail = "");
|
||||
|
||||
void push_to_remote();
|
||||
|
||||
private:
|
||||
Telemetry() = default;
|
||||
};
|
||||
|
||||
} // namespace fces
|
||||
51
python/fces_native.cpp
Normal file
51
python/fces_native.cpp
Normal file
@@ -0,0 +1,51 @@
|
||||
/**
|
||||
* @file fces_native.cpp
|
||||
* @brief Python bindings for FCES-native via pybind11.
|
||||
*
|
||||
* Exposes FCESOptimizer as a drop-in replacement for the Python implementation.
|
||||
*
|
||||
* Usage:
|
||||
* from fces_native import FCESOptimizer
|
||||
* opt = FCESOptimizer(model.parameters(), lr=1.6e-3, population_size=200)
|
||||
*/
|
||||
|
||||
#include <pybind11/pybind11.h>
|
||||
#include <pybind11/stl.h>
|
||||
#include <torch/extension.h>
|
||||
|
||||
#include "fces/optimizer.hpp"
|
||||
#include "fces/config.hpp"
|
||||
|
||||
namespace py = pybind11;
|
||||
|
||||
PYBIND11_MODULE(fces_native, m) {
|
||||
m.doc() = "FCES-native: High-performance C++ FCES optimizer";
|
||||
|
||||
py::class_<fces::FCESConfig>(m, "FCESConfig")
|
||||
.def(py::init<>())
|
||||
.def_readwrite("lr", &fces::FCESConfig::lr)
|
||||
.def_readwrite("population_size", &fces::FCESConfig::population_size)
|
||||
.def_readwrite("total_steps", &fces::FCESConfig::total_steps)
|
||||
.def_readwrite("grokking_coefficient", &fces::FCESConfig::grokking_coefficient)
|
||||
.def_readwrite("direct_construction", &fces::FCESConfig::direct_construction);
|
||||
|
||||
py::class_<fces::FCESOptimizer>(m, "FCESOptimizer")
|
||||
.def(py::init<std::vector<torch::Tensor>, fces::FCESConfig>(),
|
||||
py::arg("params"),
|
||||
py::arg("config") = fces::FCESConfig{})
|
||||
.def("step", &fces::FCESOptimizer::step)
|
||||
.def("update_fitness", &fces::FCESOptimizer::update_fitness)
|
||||
.def("backup_to_ram", &fces::FCESOptimizer::backup_to_ram)
|
||||
.def("restore_from_ram", &fces::FCESOptimizer::restore_from_ram)
|
||||
.def("step_count", &fces::FCESOptimizer::step_count)
|
||||
.def("calculate_sparsity", &fces::FCESOptimizer::calculate_sparsity)
|
||||
.def("zero_grad", [](fces::FCESOptimizer& self) {
|
||||
for (auto& group : self.param_groups()) {
|
||||
for (auto& p : group.params()) {
|
||||
if (p.grad().defined()) {
|
||||
p.grad().zero_();
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
27
python/setup.py
Normal file
27
python/setup.py
Normal file
@@ -0,0 +1,27 @@
|
||||
from setuptools import setup
|
||||
from torch.utils.cpp_extension import BuildExtension, CppExtension
|
||||
|
||||
setup(
|
||||
name="fces_native",
|
||||
version="0.1.0",
|
||||
description="High-performance C++ FCES optimizer (Python bindings)",
|
||||
ext_modules=[
|
||||
CppExtension(
|
||||
name="fces_native",
|
||||
sources=[
|
||||
"fces_native.cpp",
|
||||
"../src/config.cpp",
|
||||
"../src/controller.cpp",
|
||||
"../src/population.cpp",
|
||||
"../src/fitness.cpp",
|
||||
"../src/evolution.cpp",
|
||||
"../src/spectral.cpp",
|
||||
"../src/oscillation.cpp",
|
||||
"../src/optimizer.cpp",
|
||||
"../src/telemetry.cpp",
|
||||
],
|
||||
include_dirs=["../include"],
|
||||
),
|
||||
],
|
||||
cmdclass={"build_ext": BuildExtension},
|
||||
)
|
||||
4
src/config.cpp
Normal file
4
src/config.cpp
Normal file
@@ -0,0 +1,4 @@
|
||||
#include "fces/config.hpp"
|
||||
|
||||
// Config is header-only (all defaults in struct).
|
||||
// This file reserved for future runtime config loading (YAML/JSON).
|
||||
174
src/controller.cpp
Normal file
174
src/controller.cpp
Normal file
@@ -0,0 +1,174 @@
|
||||
#include "fces/controller.hpp"
|
||||
#include <algorithm>
|
||||
#include <cmath>
|
||||
#include <numeric>
|
||||
|
||||
namespace fces {
|
||||
|
||||
// Static members
|
||||
std::atomic<uint64_t> FuzzyController::next_id_{0};
|
||||
thread_local std::mt19937 FuzzyController::rng_{std::random_device{}()};
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// Genome
|
||||
// ---------------------------------------------------------------
|
||||
|
||||
void Genome::randomize(std::mt19937& rng) {
|
||||
std::normal_distribution<float> dist(0.0f, 0.5f);
|
||||
for (auto& w : weights) {
|
||||
w = dist(rng);
|
||||
}
|
||||
gene_success.fill(0.0f);
|
||||
}
|
||||
|
||||
Genome Genome::clone() const {
|
||||
return *this; // Copy all fields
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// FuzzyController
|
||||
// ---------------------------------------------------------------
|
||||
|
||||
FuzzyController::FuzzyController()
|
||||
: id(next_id_++), origin("random") {
|
||||
genome.randomize(rng_);
|
||||
// Bias output toward acceleration (V2.1 insight)
|
||||
// Set output biases (last GENOME_OUTPUT_DIM elements) to +2.0
|
||||
constexpr int bias_start = GENOME_SIZE - GENOME_OUTPUT_DIM;
|
||||
for (int i = bias_start; i < GENOME_SIZE; ++i) {
|
||||
genome.weights[i] = 2.0f;
|
||||
}
|
||||
}
|
||||
|
||||
FuzzyController::FuzzyController(Genome genome)
|
||||
: id(next_id_++), genome(std::move(genome)), origin("constructed") {}
|
||||
|
||||
torch::Tensor FuzzyController::decide_update(
|
||||
const std::vector<std::vector<float>>& layer_stats,
|
||||
float loss_trend,
|
||||
float step_pct,
|
||||
float rollback_rate,
|
||||
float grad_stability,
|
||||
float spectral_alpha,
|
||||
float stagnation_intensity,
|
||||
float kzm_damping,
|
||||
float projected_drift
|
||||
) {
|
||||
const int num_groups = static_cast<int>(layer_stats.size());
|
||||
auto actions = torch::zeros({num_groups, GENOME_OUTPUT_DIM});
|
||||
|
||||
// Extract weight views for the micro-MLP
|
||||
const float* w = genome.weights.data();
|
||||
|
||||
// Layer 1: input -> hidden
|
||||
const float* W1 = w; // [GENOME_INPUT_DIM x GENOME_HIDDEN_DIM]
|
||||
const float* b1 = w + (GENOME_INPUT_DIM * GENOME_HIDDEN_DIM); // [GENOME_HIDDEN_DIM]
|
||||
// Layer 2: hidden -> output
|
||||
const float* W2 = b1 + GENOME_HIDDEN_DIM; // [GENOME_HIDDEN_DIM x GENOME_OUTPUT_DIM]
|
||||
const float* b2 = W2 + (GENOME_HIDDEN_DIM * GENOME_OUTPUT_DIM); // [GENOME_OUTPUT_DIM]
|
||||
|
||||
for (int g = 0; g < num_groups; ++g) {
|
||||
// Build input vector
|
||||
std::array<float, GENOME_INPUT_DIM> input{};
|
||||
if (!layer_stats[g].empty()) {
|
||||
// Copy available stats, pad with context
|
||||
const int n = std::min(static_cast<int>(layer_stats[g].size()), GENOME_INPUT_DIM - 4);
|
||||
for (int i = 0; i < n; ++i) {
|
||||
input[i] = layer_stats[g][i];
|
||||
}
|
||||
}
|
||||
// Append global context
|
||||
input[GENOME_INPUT_DIM - 4] = loss_trend;
|
||||
input[GENOME_INPUT_DIM - 3] = step_pct;
|
||||
input[GENOME_INPUT_DIM - 2] = grad_stability;
|
||||
input[GENOME_INPUT_DIM - 1] = stagnation_intensity;
|
||||
|
||||
// Forward pass: hidden = tanh(W1 * input + b1)
|
||||
std::array<float, GENOME_HIDDEN_DIM> hidden{};
|
||||
for (int h = 0; h < GENOME_HIDDEN_DIM; ++h) {
|
||||
float sum = b1[h];
|
||||
for (int i = 0; i < GENOME_INPUT_DIM; ++i) {
|
||||
sum += W1[h * GENOME_INPUT_DIM + i] * input[i];
|
||||
}
|
||||
hidden[h] = std::tanh(sum);
|
||||
}
|
||||
|
||||
// Output = W2 * hidden + b2
|
||||
for (int o = 0; o < GENOME_OUTPUT_DIM; ++o) {
|
||||
float sum = b2[o];
|
||||
for (int h = 0; h < GENOME_HIDDEN_DIM; ++h) {
|
||||
sum += W2[o * GENOME_HIDDEN_DIM + h] * hidden[h];
|
||||
}
|
||||
actions[g][o] = sum;
|
||||
}
|
||||
}
|
||||
|
||||
return actions;
|
||||
}
|
||||
|
||||
FuzzyController FuzzyController::mutate(float current_loss, float sigma_scale) const {
|
||||
Genome child_genome = genome.clone();
|
||||
std::normal_distribution<float> noise(0.0f, genome.sigma_gene * sigma_scale);
|
||||
|
||||
for (size_t i = 0; i < child_genome.weights.size(); ++i) {
|
||||
child_genome.weights[i] += noise(rng_);
|
||||
}
|
||||
|
||||
FuzzyController child(child_genome);
|
||||
child.origin = "mutation";
|
||||
return child;
|
||||
}
|
||||
|
||||
FuzzyController FuzzyController::crossover(const FuzzyController& partner, bool use_alignment) const {
|
||||
Genome child_genome;
|
||||
std::uniform_real_distribution<float> coin(0.0f, 1.0f);
|
||||
|
||||
for (size_t i = 0; i < child_genome.weights.size(); ++i) {
|
||||
if (use_alignment && genome.gene_success[i] > partner.genome.gene_success[i]) {
|
||||
child_genome.weights[i] = genome.weights[i];
|
||||
} else if (use_alignment && partner.genome.gene_success[i] > genome.gene_success[i]) {
|
||||
child_genome.weights[i] = partner.genome.weights[i];
|
||||
} else {
|
||||
// Uniform crossover
|
||||
child_genome.weights[i] = (coin(rng_) < 0.5f)
|
||||
? genome.weights[i]
|
||||
: partner.genome.weights[i];
|
||||
}
|
||||
child_genome.gene_success[i] = 0.0f;
|
||||
}
|
||||
|
||||
FuzzyController child(child_genome);
|
||||
child.origin = "crossover";
|
||||
return child;
|
||||
}
|
||||
|
||||
FuzzyController FuzzyController::create_orthogonal_child(float intensity) const {
|
||||
Genome child_genome = genome.clone();
|
||||
// Negate a random subset of weights
|
||||
std::uniform_real_distribution<float> coin(0.0f, 1.0f);
|
||||
for (size_t i = 0; i < child_genome.weights.size(); ++i) {
|
||||
if (coin(rng_) < 0.3f * intensity) {
|
||||
child_genome.weights[i] = -child_genome.weights[i];
|
||||
}
|
||||
}
|
||||
|
||||
FuzzyController child(child_genome);
|
||||
child.origin = "phoenix_rebirth";
|
||||
return child;
|
||||
}
|
||||
|
||||
std::pair<FuzzyController, FuzzyController> FuzzyController::banach_tarski_fission(float intensity) const {
|
||||
Genome plus_genome = genome.clone();
|
||||
Genome minus_genome = genome.clone();
|
||||
std::normal_distribution<float> noise(0.0f, 0.1f * intensity);
|
||||
|
||||
for (size_t i = 0; i < genome.weights.size(); ++i) {
|
||||
float delta = noise(rng_);
|
||||
plus_genome.weights[i] += delta;
|
||||
minus_genome.weights[i] -= delta;
|
||||
}
|
||||
|
||||
return {FuzzyController(plus_genome), FuzzyController(minus_genome)};
|
||||
}
|
||||
|
||||
} // namespace fces
|
||||
24
src/evolution.cpp
Normal file
24
src/evolution.cpp
Normal file
@@ -0,0 +1,24 @@
|
||||
#include "fces/evolution.hpp"
|
||||
|
||||
namespace fces {
|
||||
|
||||
EvolutionManager::EvolutionManager(
|
||||
Population& population, int selection_interval,
|
||||
bool auto_population, bool direct_construction
|
||||
)
|
||||
: population_(population),
|
||||
selection_interval(selection_interval),
|
||||
auto_population_(auto_population),
|
||||
direct_construction_(direct_construction) {}
|
||||
|
||||
FuzzyController& EvolutionManager::get_active_controller() {
|
||||
return population_.get_active_controller();
|
||||
}
|
||||
|
||||
void EvolutionManager::update_population_dynamics(
|
||||
float loss_velocity, float ema_loss, int step_counter, int total_steps
|
||||
) {
|
||||
// TODO: Port full dynamics (auto-sizing, lockdown, evolution triggers)
|
||||
}
|
||||
|
||||
} // namespace fces
|
||||
71
src/fitness.cpp
Normal file
71
src/fitness.cpp
Normal file
@@ -0,0 +1,71 @@
|
||||
#include "fces/fitness.hpp"
|
||||
#include <cmath>
|
||||
#include <numeric>
|
||||
#include <algorithm>
|
||||
|
||||
namespace fces {
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// RunningStats (Welford's Online Algorithm)
|
||||
// ---------------------------------------------------------------
|
||||
|
||||
void RunningStats::update(float value) {
|
||||
count_++;
|
||||
float delta = value - mean_;
|
||||
mean_ += delta / static_cast<float>(count_);
|
||||
float delta2 = value - mean_;
|
||||
m2_ += delta * delta2;
|
||||
}
|
||||
|
||||
float RunningStats::z_score(float value) const {
|
||||
float s = get_std();
|
||||
if (s < 1e-8f) return 0.0f;
|
||||
return (value - mean_) / s;
|
||||
}
|
||||
|
||||
float RunningStats::get_std() const {
|
||||
if (count_ < 2) return 1.0f;
|
||||
return std::sqrt(m2_ / static_cast<float>(count_ - 1));
|
||||
}
|
||||
|
||||
void RunningStats::reset() {
|
||||
count_ = 0;
|
||||
mean_ = 0.0f;
|
||||
m2_ = 0.0f;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// FitnessEngine
|
||||
// ---------------------------------------------------------------
|
||||
|
||||
FitnessEngine::FitnessEngine(float grokking_coefficient)
|
||||
: grokking_coefficient_(grokking_coefficient) {}
|
||||
|
||||
float FitnessEngine::calculate_loss_signal(float current_loss, float ema_loss, const std::string& mode) const {
|
||||
if (ema_loss < 1e-8f) return 0.0f;
|
||||
|
||||
if (mode == "relative") {
|
||||
return (current_loss - ema_loss) / (ema_loss + 1e-8f);
|
||||
}
|
||||
return current_loss - ema_loss;
|
||||
}
|
||||
|
||||
float FitnessEngine::compute_kzm_damping(float spectral_alpha) const {
|
||||
// Kibble-Zurek damping: high spectral rank = more damping
|
||||
return 1.0f / (1.0f + grokking_coefficient_ * spectral_alpha);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// FuzzyFitnessEvaluator
|
||||
// ---------------------------------------------------------------
|
||||
|
||||
FitnessMetrics FuzzyFitnessEvaluator::evaluate(
|
||||
float loss_before, float loss_after, float sparsity, float val_loss
|
||||
) const {
|
||||
FitnessMetrics metrics;
|
||||
metrics.loss_improvement = loss_before - loss_after;
|
||||
metrics.sparsity_score = sparsity * sparsity; // Quadratic reward
|
||||
return metrics;
|
||||
}
|
||||
|
||||
} // namespace fces
|
||||
131
src/optimizer.cpp
Normal file
131
src/optimizer.cpp
Normal file
@@ -0,0 +1,131 @@
|
||||
#include "fces/optimizer.hpp"
|
||||
#include <cmath>
|
||||
#include <iostream>
|
||||
|
||||
namespace fces {
|
||||
|
||||
FCESOptimizer::FCESOptimizer(
|
||||
std::vector<torch::Tensor> params,
|
||||
FCESConfig config
|
||||
)
|
||||
: torch::optim::Optimizer(
|
||||
{torch::optim::OptimizerParamGroup(std::move(params))},
|
||||
std::make_unique<torch::optim::OptimizerOptions>(config.lr)
|
||||
),
|
||||
config_(std::move(config)),
|
||||
population_(config_.population_size, 10000,
|
||||
EliteStrategy::Cumulative,
|
||||
false, false, false, false, false,
|
||||
config_.direct_construction,
|
||||
config_.use_banach_fission),
|
||||
fitness_engine_(config_.grokking_coefficient) {
|
||||
|
||||
evolution_manager_ = std::make_unique<EvolutionManager>(
|
||||
population_, 50, config_.auto_population, config_.direct_construction
|
||||
);
|
||||
|
||||
// Initial RAM backup
|
||||
backup_to_ram();
|
||||
|
||||
Telemetry::get().info("optimizer_initialized",
|
||||
"version=0.1.0 pop_size=" + std::to_string(config_.population_size));
|
||||
}
|
||||
|
||||
torch::Tensor FCESOptimizer::step(LossClosure closure) {
|
||||
torch::NoGradGuard no_grad;
|
||||
step_counter_++;
|
||||
|
||||
torch::Tensor loss = {};
|
||||
if (closure) {
|
||||
loss = closure();
|
||||
}
|
||||
|
||||
// TODO: Port full step logic from Python:
|
||||
// 1. _gather_stats()
|
||||
// 2. get_active_controller()
|
||||
// 3. _get_actions()
|
||||
// 4. _apply_parameter_updates()
|
||||
// 5. Evolution & maintenance
|
||||
|
||||
// Minimal stub: apply sign-SGD update
|
||||
for (auto& group : param_groups()) {
|
||||
for (auto& p : group.params()) {
|
||||
if (!p.grad().defined()) continue;
|
||||
|
||||
auto update = torch::sign(p.grad());
|
||||
p.data().add_(update, -config_.lr);
|
||||
}
|
||||
}
|
||||
|
||||
if (step_counter_ % 50 == 0) {
|
||||
backup_to_ram();
|
||||
}
|
||||
|
||||
return loss;
|
||||
}
|
||||
|
||||
void FCESOptimizer::update_fitness(float loss) {
|
||||
// EMA loss tracking
|
||||
if (step_counter_ == 1) {
|
||||
ema_loss_ = loss;
|
||||
} else {
|
||||
ema_loss_ = 0.95f * ema_loss_ + 0.05f * loss;
|
||||
}
|
||||
last_step_loss_ = loss;
|
||||
|
||||
// Update best loss window
|
||||
if (loss < best_loss_window_) {
|
||||
best_loss_window_ = loss;
|
||||
stagnation_counter_ = 0;
|
||||
} else {
|
||||
stagnation_counter_++;
|
||||
}
|
||||
}
|
||||
|
||||
void FCESOptimizer::backup_to_ram() {
|
||||
ram_backup_.clear();
|
||||
for (auto& group : param_groups()) {
|
||||
for (auto& p : group.params()) {
|
||||
ram_backup_.push_back(p.data().clone().cpu());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void FCESOptimizer::restore_from_ram() {
|
||||
int idx = 0;
|
||||
for (auto& group : param_groups()) {
|
||||
for (auto& p : group.params()) {
|
||||
if (idx < static_cast<int>(ram_backup_.size())) {
|
||||
p.data().copy_(ram_backup_[idx].to(p.device()));
|
||||
idx++;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
float FCESOptimizer::calculate_sparsity() const {
|
||||
int64_t total = 0, zeros = 0;
|
||||
for (const auto& group : param_groups()) {
|
||||
for (const auto& p : group.params()) {
|
||||
total += p.numel();
|
||||
zeros += (p.data().abs() < 1e-5f).sum().item<int64_t>();
|
||||
}
|
||||
}
|
||||
return (total > 0) ? static_cast<float>(zeros) / total : 0.0f;
|
||||
}
|
||||
|
||||
void FCESOptimizer::gather_stats() {
|
||||
// TODO: Port _gather_stats from Python
|
||||
}
|
||||
|
||||
void FCESOptimizer::apply_parameter_updates(const torch::Tensor& /*actions*/) {
|
||||
// TODO: Port _apply_parameter_updates from Python
|
||||
}
|
||||
|
||||
void FCESOptimizer::handle_rollback() {
|
||||
restore_from_ram();
|
||||
population_.calm_down();
|
||||
rollback_ema_ = 0.9f * rollback_ema_ + 0.1f;
|
||||
}
|
||||
|
||||
} // namespace fces
|
||||
97
src/oscillation.cpp
Normal file
97
src/oscillation.cpp
Normal file
@@ -0,0 +1,97 @@
|
||||
#include "fces/oscillation.hpp"
|
||||
#include <cmath>
|
||||
#include <numeric>
|
||||
#include <algorithm>
|
||||
|
||||
namespace fces {
|
||||
|
||||
void OscillationDetector::update(float loss) {
|
||||
loss_history_.push_back(loss);
|
||||
if (static_cast<int>(loss_history_.size()) > WINDOW_SIZE) {
|
||||
loss_history_.erase(loss_history_.begin());
|
||||
}
|
||||
}
|
||||
|
||||
bool OscillationDetector::detect() const {
|
||||
return get_score() > POWER_THRESHOLD;
|
||||
}
|
||||
|
||||
float OscillationDetector::get_score() const {
|
||||
if (static_cast<int>(loss_history_.size()) < WINDOW_SIZE) return 0.0f;
|
||||
|
||||
auto detrended = detrend(loss_history_);
|
||||
auto power = compute_power_spectrum(detrended);
|
||||
|
||||
// Sum power in oscillation bands (periods 4-16)
|
||||
float osc_power = 0.0f;
|
||||
float total_power = 0.0f;
|
||||
int n = static_cast<int>(power.size());
|
||||
for (int i = 1; i < n; ++i) {
|
||||
total_power += power[i];
|
||||
int period = n / i;
|
||||
if (period >= MIN_PERIOD && period <= MAX_PERIOD) {
|
||||
osc_power += power[i];
|
||||
}
|
||||
}
|
||||
|
||||
if (total_power < 1e-8f) return 0.0f;
|
||||
return osc_power / total_power;
|
||||
}
|
||||
|
||||
float OscillationDetector::get_variance_50() const {
|
||||
if (loss_history_.size() < 50) return 0.0f;
|
||||
auto start = loss_history_.end() - 50;
|
||||
float mean = std::accumulate(start, loss_history_.end(), 0.0f) / 50.0f;
|
||||
float var = 0.0f;
|
||||
for (auto it = start; it != loss_history_.end(); ++it) {
|
||||
float d = *it - mean;
|
||||
var += d * d;
|
||||
}
|
||||
return var / 50.0f;
|
||||
}
|
||||
|
||||
void OscillationDetector::reset() {
|
||||
loss_history_.clear();
|
||||
}
|
||||
|
||||
std::vector<float> OscillationDetector::detrend(const std::vector<float>& signal) {
|
||||
int n = static_cast<int>(signal.size());
|
||||
if (n < 2) return signal;
|
||||
|
||||
// Remove linear trend via least squares
|
||||
float sum_x = 0, sum_y = 0, sum_xy = 0, sum_xx = 0;
|
||||
for (int i = 0; i < n; ++i) {
|
||||
sum_x += i;
|
||||
sum_y += signal[i];
|
||||
sum_xy += i * signal[i];
|
||||
sum_xx += i * i;
|
||||
}
|
||||
float slope = (n * sum_xy - sum_x * sum_y) / (n * sum_xx - sum_x * sum_x + 1e-8f);
|
||||
float intercept = (sum_y - slope * sum_x) / n;
|
||||
|
||||
std::vector<float> result(n);
|
||||
for (int i = 0; i < n; ++i) {
|
||||
result[i] = signal[i] - (slope * i + intercept);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
std::vector<float> OscillationDetector::compute_power_spectrum(const std::vector<float>& signal) {
|
||||
// Simple DFT (for WINDOW_SIZE=64, this is fast enough; upgrade to FFT if needed)
|
||||
int n = static_cast<int>(signal.size());
|
||||
int half = n / 2;
|
||||
std::vector<float> power(half);
|
||||
|
||||
for (int k = 0; k < half; ++k) {
|
||||
float re = 0.0f, im = 0.0f;
|
||||
for (int t = 0; t < n; ++t) {
|
||||
float angle = 2.0f * 3.14159265358979f * k * t / n;
|
||||
re += signal[t] * std::cos(angle);
|
||||
im -= signal[t] * std::sin(angle);
|
||||
}
|
||||
power[k] = re * re + im * im;
|
||||
}
|
||||
return power;
|
||||
}
|
||||
|
||||
} // namespace fces
|
||||
122
src/population.cpp
Normal file
122
src/population.cpp
Normal file
@@ -0,0 +1,122 @@
|
||||
#include "fces/population.hpp"
|
||||
#include <algorithm>
|
||||
#include <random>
|
||||
|
||||
namespace fces {
|
||||
|
||||
Population::Population(
|
||||
int active_size, int repo_size, EliteStrategy elite_strategy,
|
||||
bool link_mutation, bool link_elite, bool link_violator,
|
||||
bool use_fuzzy_pacer, bool use_fuzzy_importance,
|
||||
bool direct_construction, bool use_banach_fission
|
||||
)
|
||||
: elite_strategy_(elite_strategy),
|
||||
link_mutation_(link_mutation),
|
||||
link_elite_(link_elite),
|
||||
link_violator_(link_violator),
|
||||
use_fuzzy_pacer_(use_fuzzy_pacer),
|
||||
use_fuzzy_importance_(use_fuzzy_importance),
|
||||
direct_construction_(direct_construction),
|
||||
use_banach_fission_(use_banach_fission) {
|
||||
|
||||
if (direct_construction) active_size = 1;
|
||||
|
||||
gladiators_.reserve(active_size);
|
||||
for (int i = 0; i < active_size; ++i) {
|
||||
gladiators_.emplace_back();
|
||||
}
|
||||
repository_.reserve(repo_size);
|
||||
}
|
||||
|
||||
FuzzyController& Population::get_active_controller() {
|
||||
// TODO: Implement sticky selection with interval
|
||||
if (active_controller_ == nullptr || steps_active_ >= selection_interval_) {
|
||||
active_controller_ = &select_weighted();
|
||||
steps_active_ = 0;
|
||||
}
|
||||
steps_active_++;
|
||||
return *active_controller_;
|
||||
}
|
||||
|
||||
FuzzyController& Population::select_weighted() {
|
||||
// TODO: Implement tournament selection with age weighting
|
||||
static thread_local std::mt19937 rng{std::random_device{}()};
|
||||
std::uniform_int_distribution<int> dist(0, static_cast<int>(gladiators_.size()) - 1);
|
||||
return gladiators_[dist(rng)];
|
||||
}
|
||||
|
||||
FuzzyController& Population::get_best_active() {
|
||||
return *std::max_element(gladiators_.begin(), gladiators_.end(),
|
||||
[](const FuzzyController& a, const FuzzyController& b) {
|
||||
return a.fitness < b.fitness;
|
||||
});
|
||||
}
|
||||
|
||||
FuzzyController& Population::get_worst_active() {
|
||||
return *std::min_element(gladiators_.begin(), gladiators_.end(),
|
||||
[](const FuzzyController& a, const FuzzyController& b) {
|
||||
return a.fitness < b.fitness;
|
||||
});
|
||||
}
|
||||
|
||||
void Population::kill(FuzzyController& controller) {
|
||||
// TODO: Elite protection
|
||||
auto it = std::find_if(gladiators_.begin(), gladiators_.end(),
|
||||
[&](const FuzzyController& c) { return c.id == controller.id; });
|
||||
if (it != gladiators_.end()) {
|
||||
gladiators_.erase(it);
|
||||
if (gladiators_.empty()) {
|
||||
gladiators_.emplace_back();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void Population::update_controller_fitness(FuzzyController& controller, float reward, bool increment_eval) {
|
||||
controller.fitness += reward;
|
||||
controller.lifetime_fitness += reward;
|
||||
if (increment_eval) controller.evaluation_count++;
|
||||
controller.age++;
|
||||
}
|
||||
|
||||
void Population::mark_violated(FuzzyController& controller) {
|
||||
// Deduplicate
|
||||
auto it = std::find_if(violated_controllers_.begin(), violated_controllers_.end(),
|
||||
[&](const FuzzyController& c) { return c.id == controller.id; });
|
||||
if (it == violated_controllers_.end()) {
|
||||
violated_controllers_.push_back(controller);
|
||||
}
|
||||
}
|
||||
|
||||
float Population::get_effective_fitness(const FuzzyController& controller, float training_progress) const {
|
||||
// TODO: Implement recency-weighted effective fitness (V42.5)
|
||||
return controller.fitness;
|
||||
}
|
||||
|
||||
void Population::evolve(float current_loss, float velocity, float training_progress) {
|
||||
// TODO: Port full evolution logic from Python
|
||||
}
|
||||
|
||||
void Population::resize(int target_size, float training_progress) {
|
||||
// TODO: Port resize logic
|
||||
}
|
||||
|
||||
void Population::calm_down() {
|
||||
global_sigma_modifier_ *= 0.8f;
|
||||
global_sigma_modifier_ = std::max(0.1f, global_sigma_modifier_);
|
||||
}
|
||||
|
||||
float Population::get_diversity_index() const {
|
||||
// TODO: Compute behavioral spread
|
||||
return 0.5f;
|
||||
}
|
||||
|
||||
std::vector<FuzzyController*> Population::get_elites() {
|
||||
// TODO: Implement strategy-aware elite selection
|
||||
return {};
|
||||
}
|
||||
|
||||
void Population::add_to_repository(const FuzzyController& controller) {
|
||||
// TODO: Maintain sorted repository
|
||||
}
|
||||
|
||||
} // namespace fces
|
||||
40
src/spectral.cpp
Normal file
40
src/spectral.cpp
Normal file
@@ -0,0 +1,40 @@
|
||||
#include "fces/spectral.hpp"
|
||||
#include <numeric>
|
||||
|
||||
namespace fces {
|
||||
|
||||
SpectralSensor::SpectralSensor(torch::nn::Module& /*model*/) {}
|
||||
|
||||
void SpectralSensor::track_layer(const std::string& name, const torch::Tensor& weight) {
|
||||
if (weight.dim() >= 2) {
|
||||
layer_ranks_[name] = compute_effective_rank(weight);
|
||||
}
|
||||
}
|
||||
|
||||
float SpectralSensor::get_global_rank() const {
|
||||
if (layer_ranks_.empty()) return 0.0f;
|
||||
float sum = 0.0f;
|
||||
for (const auto& [_, rank] : layer_ranks_) {
|
||||
sum += rank;
|
||||
}
|
||||
return sum / static_cast<float>(layer_ranks_.size());
|
||||
}
|
||||
|
||||
void SpectralSensor::reset() {
|
||||
layer_ranks_.clear();
|
||||
}
|
||||
|
||||
float SpectralSensor::compute_effective_rank(const torch::Tensor& weight) {
|
||||
// SVD-based effective rank (Shannon entropy of normalized singular values)
|
||||
auto svd = torch::linalg::svdvals(weight.to(torch::kFloat32));
|
||||
auto s = svd / svd.sum();
|
||||
auto log_s = torch::log(s + 1e-10f);
|
||||
float entropy = -(s * log_s).sum().item<float>();
|
||||
return std::exp(entropy);
|
||||
}
|
||||
|
||||
float SpectralController::compute_alpha(float global_rank, float grokking_coefficient) const {
|
||||
return global_rank * grokking_coefficient;
|
||||
}
|
||||
|
||||
} // namespace fces
|
||||
34
src/telemetry.cpp
Normal file
34
src/telemetry.cpp
Normal file
@@ -0,0 +1,34 @@
|
||||
#include "fces/telemetry.hpp"
|
||||
#include <iostream>
|
||||
#include <chrono>
|
||||
|
||||
namespace fces {
|
||||
|
||||
Telemetry& Telemetry::get() {
|
||||
static Telemetry instance;
|
||||
return instance;
|
||||
}
|
||||
|
||||
void Telemetry::info(const std::string& event, const std::string& detail) {
|
||||
std::cout << "[INFO] " << event;
|
||||
if (!detail.empty()) std::cout << " | " << detail;
|
||||
std::cout << std::endl;
|
||||
}
|
||||
|
||||
void Telemetry::warning(const std::string& event, const std::string& detail) {
|
||||
std::cerr << "[WARN] " << event;
|
||||
if (!detail.empty()) std::cerr << " | " << detail;
|
||||
std::cerr << std::endl;
|
||||
}
|
||||
|
||||
void Telemetry::error(const std::string& event, const std::string& detail) {
|
||||
std::cerr << "[ERROR] " << event;
|
||||
if (!detail.empty()) std::cerr << " | " << detail;
|
||||
std::cerr << std::endl;
|
||||
}
|
||||
|
||||
void Telemetry::push_to_remote() {
|
||||
// TODO: Implement telemetry push (Git sync, file export, etc.)
|
||||
}
|
||||
|
||||
} // namespace fces
|
||||
58
tests/test_controller.cpp
Normal file
58
tests/test_controller.cpp
Normal file
@@ -0,0 +1,58 @@
|
||||
#include <gtest/gtest.h>
|
||||
#include "fces/controller.hpp"
|
||||
|
||||
using namespace fces;
|
||||
|
||||
TEST(ControllerTest, Construction) {
|
||||
FuzzyController ctrl;
|
||||
EXPECT_GT(ctrl.id, 0u);
|
||||
EXPECT_EQ(ctrl.fitness, 0.0f);
|
||||
EXPECT_EQ(ctrl.origin, "random");
|
||||
}
|
||||
|
||||
TEST(ControllerTest, GenomeSize) {
|
||||
FuzzyController ctrl;
|
||||
EXPECT_EQ(ctrl.genome.weights.size(), static_cast<size_t>(GENOME_SIZE));
|
||||
}
|
||||
|
||||
TEST(ControllerTest, Mutation) {
|
||||
FuzzyController parent;
|
||||
auto child = parent.mutate(1.0f);
|
||||
EXPECT_NE(child.id, parent.id);
|
||||
EXPECT_EQ(child.origin, "mutation");
|
||||
// Child should differ from parent
|
||||
bool differs = false;
|
||||
for (size_t i = 0; i < parent.genome.weights.size(); ++i) {
|
||||
if (parent.genome.weights[i] != child.genome.weights[i]) {
|
||||
differs = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
EXPECT_TRUE(differs);
|
||||
}
|
||||
|
||||
TEST(ControllerTest, Crossover) {
|
||||
FuzzyController a, b;
|
||||
auto child = a.crossover(b);
|
||||
EXPECT_EQ(child.origin, "crossover");
|
||||
}
|
||||
|
||||
TEST(ControllerTest, DecideUpdate) {
|
||||
FuzzyController ctrl;
|
||||
std::vector<std::vector<float>> stats = {{0.1f, 0.2f, 0.3f, 0.4f, 0.5f}};
|
||||
auto actions = ctrl.decide_update(stats, 0.0f, 0.5f, 0.0f, 0.1f, 0.0f, 0.0f, 1.0f, 0.0f);
|
||||
EXPECT_EQ(actions.size(0), 1);
|
||||
EXPECT_EQ(actions.size(1), GENOME_OUTPUT_DIM);
|
||||
}
|
||||
|
||||
TEST(ControllerTest, OrthogonalChild) {
|
||||
FuzzyController parent;
|
||||
auto child = parent.create_orthogonal_child(1.0f);
|
||||
EXPECT_EQ(child.origin, "phoenix_rebirth");
|
||||
}
|
||||
|
||||
TEST(ControllerTest, BanachFission) {
|
||||
FuzzyController parent;
|
||||
auto [plus, minus] = parent.banach_tarski_fission(1.0f);
|
||||
EXPECT_NE(plus.id, minus.id);
|
||||
}
|
||||
33
tests/test_fitness.cpp
Normal file
33
tests/test_fitness.cpp
Normal file
@@ -0,0 +1,33 @@
|
||||
#include <gtest/gtest.h>
|
||||
#include "fces/fitness.hpp"
|
||||
|
||||
using namespace fces;
|
||||
|
||||
TEST(RunningStatsTest, BasicUpdate) {
|
||||
RunningStats stats;
|
||||
stats.update(1.0f);
|
||||
stats.update(2.0f);
|
||||
stats.update(3.0f);
|
||||
EXPECT_NEAR(stats.get_mean(), 2.0f, 1e-5f);
|
||||
EXPECT_GT(stats.get_std(), 0.0f);
|
||||
}
|
||||
|
||||
TEST(RunningStatsTest, ZScore) {
|
||||
RunningStats stats;
|
||||
for (int i = 0; i < 100; ++i) stats.update(static_cast<float>(i));
|
||||
float z = stats.z_score(50.0f);
|
||||
EXPECT_NEAR(z, 0.0f, 0.1f);
|
||||
}
|
||||
|
||||
TEST(FitnessEngineTest, LossSignal) {
|
||||
FitnessEngine engine;
|
||||
float sig = engine.calculate_loss_signal(1.0f, 2.0f, "relative");
|
||||
EXPECT_LT(sig, 0.0f); // Improving
|
||||
}
|
||||
|
||||
TEST(FitnessEngineTest, KZMDamping) {
|
||||
FitnessEngine engine(0.1f);
|
||||
float d = engine.compute_kzm_damping(5.0f);
|
||||
EXPECT_GT(d, 0.0f);
|
||||
EXPECT_LT(d, 1.0f);
|
||||
}
|
||||
42
tests/test_optimizer.cpp
Normal file
42
tests/test_optimizer.cpp
Normal file
@@ -0,0 +1,42 @@
|
||||
#include <gtest/gtest.h>
|
||||
#include <torch/torch.h>
|
||||
#include "fces/optimizer.hpp"
|
||||
|
||||
using namespace fces;
|
||||
|
||||
TEST(OptimizerTest, Construction) {
|
||||
auto model = torch::nn::Linear(10, 5);
|
||||
std::vector<torch::Tensor> params;
|
||||
for (auto& p : model->parameters()) params.push_back(p);
|
||||
|
||||
FCESOptimizer opt(params, FCESConfig{}.set_lr(1e-3f));
|
||||
EXPECT_EQ(opt.step_count(), 0);
|
||||
}
|
||||
|
||||
TEST(OptimizerTest, StepUpdatesCounter) {
|
||||
auto model = torch::nn::Linear(10, 5);
|
||||
std::vector<torch::Tensor> params;
|
||||
for (auto& p : model->parameters()) params.push_back(p);
|
||||
|
||||
FCESOptimizer opt(params, FCESConfig{}.set_lr(1e-3f));
|
||||
|
||||
// Simulate a training step
|
||||
auto x = torch::randn({2, 10});
|
||||
auto y = model->forward(x);
|
||||
auto loss = y.sum();
|
||||
loss.backward();
|
||||
opt.step();
|
||||
|
||||
EXPECT_EQ(opt.step_count(), 1);
|
||||
}
|
||||
|
||||
TEST(OptimizerTest, UpdateFitness) {
|
||||
auto model = torch::nn::Linear(10, 5);
|
||||
std::vector<torch::Tensor> params;
|
||||
for (auto& p : model->parameters()) params.push_back(p);
|
||||
|
||||
FCESOptimizer opt(params);
|
||||
opt.update_fitness(3.0f);
|
||||
opt.update_fitness(2.5f);
|
||||
// Should not crash
|
||||
}
|
||||
28
tests/test_population.cpp
Normal file
28
tests/test_population.cpp
Normal file
@@ -0,0 +1,28 @@
|
||||
#include <gtest/gtest.h>
|
||||
#include "fces/population.hpp"
|
||||
|
||||
using namespace fces;
|
||||
|
||||
TEST(PopulationTest, Construction) {
|
||||
Population pop(50);
|
||||
EXPECT_EQ(pop.size(), 50);
|
||||
}
|
||||
|
||||
TEST(PopulationTest, DirectConstruction) {
|
||||
Population pop(200, 10000, EliteStrategy::Cumulative,
|
||||
false, false, false, false, false, true);
|
||||
EXPECT_EQ(pop.size(), 1);
|
||||
}
|
||||
|
||||
TEST(PopulationTest, GetBestActive) {
|
||||
Population pop(10);
|
||||
auto& best = pop.get_best_active();
|
||||
// Should not crash
|
||||
EXPECT_GE(best.id, 0u);
|
||||
}
|
||||
|
||||
TEST(PopulationTest, CalmDown) {
|
||||
Population pop(10);
|
||||
pop.calm_down();
|
||||
EXPECT_LT(pop.global_sigma_modifier(), 1.0f);
|
||||
}
|
||||
Reference in New Issue
Block a user