diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 59fc3fb95f..ef303f5be6 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -54,6 +54,8 @@ jobs: steps: - uses: actions/checkout@v5 + with: + submodules: recursive - name: Set Additional Envs run: | echo "PYTHON_SUBVERSION=$(echo $PYTHON_VERSION | cut -c 3-)" >> $GITHUB_ENV @@ -114,6 +116,8 @@ jobs: steps: - uses: actions/checkout@v5 + with: + submodules: recursive - uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/test_backends.yml b/.github/workflows/test_backends.yml index 8cdd19cce2..ce956e563c 100644 --- a/.github/workflows/test_backends.yml +++ b/.github/workflows/test_backends.yml @@ -16,6 +16,8 @@ jobs: with: python-version: "3.12" - uses: actions/checkout@v5 + with: + submodules: recursive - name: Install cvxpy dependencies run: | pip install -e . diff --git a/.github/workflows/test_nlp_solvers.yml b/.github/workflows/test_nlp_solvers.yml new file mode 100644 index 0000000000..c0dd0f8e8e --- /dev/null +++ b/.github/workflows/test_nlp_solvers.yml @@ -0,0 +1,46 @@ +name: test_nlp_solvers + +on: + pull_request: + push: + branches: + - master + tags: + - '*' +jobs: + run_nlp_tests: + runs-on: ${{ matrix.os }} + defaults: + run: + shell: bash -l {0} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest] + steps: + - uses: actions/setup-python@v6 + with: + python-version: "3.12" + - uses: actions/checkout@v5 + with: + submodules: recursive + - name: Set up Conda + uses: conda-incubator/setup-miniconda@v3 + with: + activate-environment: nlp-test + python-version: 3.12 + channels: conda-forge + channel-priority: strict + auto-update-conda: true + miniforge-version: latest + use-mamba: true + - name: Install cyipopt via conda + run: | + conda install -y -c conda-forge cyipopt + - name: Install test dependencies + run: | + pip install -e . + pip install pytest hypothesis + - name: Run nlp tests + run: | + pytest -s -v cvxpy/tests/nlp_tests/. diff --git a/.github/workflows/test_optional_solvers.yml b/.github/workflows/test_optional_solvers.yml index 4fe0474dc7..bcb29632f8 100644 --- a/.github/workflows/test_optional_solvers.yml +++ b/.github/workflows/test_optional_solvers.yml @@ -31,7 +31,6 @@ jobs: with: python-version: 3.12 enable-cache: true - - uses: actions/checkout@v5 - name: Setup Julia uses: julia-actions/setup-julia@v2 with: diff --git a/CLAUDE.md b/CLAUDE.md index 9fcc7268e7..a294fdff2a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,32 +1,88 @@ -# CVXPY Development Guide +# CLAUDE.md -## Quick Reference +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Overview + +DNLP (Disciplined Nonlinear Programming) is an extension of CVXPY to general nonlinear programming. It allows smooth functions to be freely mixed with nonsmooth convex and concave functions, with rules governing how nonsmooth functions can be used. + +For theoretical foundation, see: [Disciplined Nonlinear Programming](https://web.stanford.edu/~boyd/papers/dnlp.html) + +## Build and Development Commands -### Commands ```bash -# Install in development mode +# Install IPOPT solver (required for NLP - use conda, NOT pip) +conda install -c conda-forge cyipopt + +# Install from source (development mode) pip install -e . -# Install pre-commit hooks (required) +# Install pre-commit hooks pip install pre-commit && pre-commit install # Run all tests pytest cvxpy/tests/ -# Run specific test -pytest cvxpy/tests/test_atoms.py::TestAtoms::test_norm_inf +# Run a specific test file +pytest cvxpy/tests/test_dgp.py + +# Run a specific test method +pytest cvxpy/tests/test_dgp.py::TestDgp::test_product + +# Run NLP-specific tests +pytest cvxpy/tests/nlp_tests/ + +# Lint with ruff +ruff check cvxpy + +# Auto-fix lint issues +ruff check --fix cvxpy + +# Build documentation +cd doc && make html +``` + +## Solving with DNLP + +```python +import cvxpy as cp +import numpy as np + +x = cp.Variable(n) +prob = cp.Problem(cp.Minimize(objective), constraints) + +# Initial point required for NLP solvers +x.value = np.ones(n) + +# Solve with nlp=True +prob.solve(nlp=True, solver=cp.IPOPT) + +# Optional: Run multiple solves with random initial points, return best +prob.solve(nlp=True, solver=cp.IPOPT, best_of=5) ``` +## Supported NLP Solvers + +| Solver | License | Installation | +|--------|---------|--------------| +| [IPOPT](https://github.com/coin-or/Ipopt) | EPL-2.0 | `conda install -c conda-forge cyipopt` | +| [Knitro](https://www.artelys.com/solvers/knitro/) | Commercial | `pip install knitro` (requires license) | +| [COPT](https://www.copt.de/) | Commercial | Requires license | +| [Uno](https://github.com/cuter-testing/uno) | Open source | See Uno documentation | + ## Code Style -- **Line length**: 100 characters +- Uses ruff for linting (configured in `pyproject.toml`) +- Target Python version: 3.11+ +- Line length: 100 characters - **IMPORTANT: IMPORTS AT THE TOP** of files - circular imports are the only exception -- **IMPORTANT:** Add Apache 2.0 license header to all new files -### License Header +## License Header + +New files should include the Apache 2.0 license header: ```python """ -Copyright, the CVXPY authors +Copyright 2025, the CVXPY developers Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -42,157 +98,86 @@ limitations under the License. """ ``` -## Project Structure - -``` -cvxpy/ -├── atoms/ # Mathematical functions (exp, log, norm, etc.) -│ ├── affine/ # Linear/affine ops (reshape, sum, trace, index) -│ └── elementwise/ # Element-wise ops (exp, log, abs, sqrt) -├── constraints/ # Constraint types (Zero, NonNeg, SOC, PSD) -├── expressions/ # Variable, Parameter, Constant, Expression -├── problems/ # Problem class and Minimize/Maximize -├── reductions/ # Problem transformations -│ ├── dcp2cone/ # DCP → conic canonicalizers -│ ├── dgp2dcp/ # DGP → DCP transforms -│ └── solvers/ # Solver interfaces -│ ├── conic_solvers/ -│ └── qp_solvers/ -├── lin_ops/ # Linear operator representation -│ └── backends/ # Canonicalization backends -├── utilities/ # Helpers, performance utils -└── tests/ # Unit tests (use pytest to run) -``` - ## Architecture -### Expression Hierarchy -``` -Expression (base) -├── Leaf (terminal nodes) -│ ├── Variable -│ ├── Parameter -│ └── Constant -└── Atom (function applications) - ├── AffineAtom - ├── Elementwise - └── AxisAtom -``` - -### Reduction Chain -Problems are transformed through a chain of reductions: -``` -Problem → [Dgp2Dcp] → [FlipObjective] → Dcp2Cone → CvxAttr2Constr → ConeMatrixStuffing → Solver -``` - -**Key reductions:** -- `Dgp2Dcp` - Converts DGP to DCP (if `gp=True`) -- `FlipObjective` - Converts Maximize to Minimize (negates objective) -- `Dcp2Cone` - Canonicalizes atoms to conic constraints (calls canonicalizers) -- `CvxAttr2Constr` - Converts variable attributes (e.g., `nonneg=True`) to constraints -- `ConeMatrixStuffing` - Extracts A, b, c matrices for solver - -Each reduction implements: -- `accepts(problem) → bool` - Can handle this problem? -- `apply(problem) → (new_problem, inverse_data)` - Transform -- `invert(solution, inverse_data) → solution` - Map solution back +### Expression System -See `cvxpy/reductions/solvers/solving_chain.py` for chain construction. +Expressions form an AST (Abstract Syntax Tree): +- **Expression** (base) → Variable, Parameter, Constant, Atom +- **Atom** subclasses implement mathematical functions (in `cvxpy/atoms/`) +- Each atom defines curvature, sign, and disciplined programming rules -### DCP Rules -Atoms define curvature via: -- `is_atom_convex()` / `is_atom_concave()` - Intrinsic curvature -- `is_incr(idx)` / `is_decr(idx)` - Monotonicity per argument +### Problem Types -### DGP (Disciplined Geometric Programming) -DGP problems use log-log curvature instead of standard curvature. Transformed to DCP via `dgp2dcp` reduction. +CVXPY supports multiple disciplined programming paradigms: +- **DCP** (Disciplined Convex Programming) - standard convex problems +- **DGP** (Disciplined Geometric Programming) - geometric programs +- **DQCP** (Disciplined Quasiconvex Programming) - quasiconvex programs +- **DNLP** (Disciplined Nonlinear Programming) - smooth nonlinear programs (this extension) -### DQCP (Disciplined Quasiconvex Programming) -DQCP extends DCP to quasiconvex functions. Solved via bisection on a parameter. Transformed via `dqcp2dcp` reduction. +### Reduction Pipeline -### DPP (Disciplined Parametrized Programming) -DPP enables efficient re-solving when only `Parameter` values change. CVXPY caches the canonicalization and reuses it. +Problems are transformed through a chain of reductions before solving: +``` +Problem → [Reductions] → Canonical Form → Solver +``` -**How it works**: Parameters are treated as affine (not constant) for curvature analysis. This means: -- `param * param` → NOT DPP (quadratic in params) -- `param * variable` → DPP (affine in params, params only in one factor) -- `cp.norm(param)` in constraint → NOT DPP (nonlinear in params) +Key reduction classes in `cvxpy/reductions/`: +- `Reduction` base class with `accepts()`, `apply()`, `invert()` methods +- `Chain` composes multiple reductions +- `SolvingChain` orchestrates the full solve process -Check with `problem.is_dpp()`. See `cvxpy/utilities/scopes.py` for implementation. +For DNLP: `CvxAttr2Constr` → `Dnlp2Smooth` → `NLPSolver` -## Implementing New Atoms +### Solver Categories -### 1. Create Atom Class -Location: `cvxpy/atoms/` or `cvxpy/atoms/elementwise/` +- **ConicSolvers** (`cvxpy/reductions/solvers/conic_solvers/`) - SCS, Clarabel, ECOS, etc. +- **QPSolvers** (`cvxpy/reductions/solvers/qp_solvers/`) - OSQP, ProxQP, etc. +- **NLPSolvers** (`cvxpy/reductions/solvers/nlp_solvers/`) - IPOPT, Knitro, COPT, Uno -```python -from typing import Tuple -from cvxpy.atoms.atom import Atom +### NLP System -class my_atom(Atom): - def __init__(self, x) -> None: - super().__init__(x) +The NLP infrastructure provides oracle-based interfaces for nonlinear solvers: +- `nlp_solver.py` - Base `NLPsolver` class with: + - `Bounds` class: extracts variable/constraint bounds from problem + - `Oracles` class: provides function and derivative oracles (objective, gradient, constraints, jacobian, hessian) +- `dnlp2smooth.py` - Transforms DNLP problems to smooth form via `Dnlp2Smooth` reduction +- DNLP validation: expressions must be smooth (ESR and HSR) +- Problem validity checked via `problem.is_dnlp()` method - def shape_from_args(self) -> Tuple[int, ...]: - return self.args[0].shape +### Diff Engine (SparseDiffPy) - def sign_from_args(self) -> Tuple[bool, bool]: - return (False, False) # (is_nonneg, is_nonpos) +The automatic differentiation engine is provided by the [SparseDiffPy](https://github.com/SparseDifferentiation/SparseDiffPy) package (`pip install sparsediffpy`), which wraps the [SparseDiffEngine](https://github.com/SparseDifferentiation/SparseDiffEngine) C library. It builds expression trees from CVXPY problems and computes derivatives (gradients, Jacobians, Hessians) for NLP solvers. New diff engine atoms require C-level additions in SparseDiffPy. - def is_atom_convex(self) -> bool: - return True +## Implementing New Atoms - def is_atom_concave(self) -> bool: - return False +### For DCP Atoms - def is_incr(self, idx: int) -> bool: - return True +1. Create atom class in `cvxpy/atoms/` or `cvxpy/atoms/elementwise/` +2. Implement: `shape_from_args()`, `sign_from_args()`, `is_atom_convex()`, `is_atom_concave()`, `is_incr()`, `is_decr()`, `numeric()` +3. Create canonicalizer in `cvxpy/reductions/dcp2cone/canonicalizers/` +4. Register in `canonicalizers/__init__.py` by adding to `CANON_METHODS` dict +5. Export in `cvxpy/atoms/__init__.py` - def is_decr(self, idx: int) -> bool: - return False +### For DNLP Support - def numeric(self, values): - return np.my_function(values[0]) -``` +1. Create a canonicalizer in `cvxpy/reductions/dnlp2smooth/canonicalizers/` +2. The canonicalizer converts non-smooth atoms to smooth equivalents using auxiliary variables +3. Register in `canonicalizers/__init__.py` by adding to `SMOOTH_CANON_METHODS` dict +4. Ensure the atom has proper `is_smooth()`, `is_esr()`, `is_hsr()` methods -### 2. Create Canonicalizer -Location: `cvxpy/reductions/dcp2cone/canonicalizers/` +### DNLP Rules (ESR/HSR) -```python -from cvxpy.expressions.variable import Variable -from cvxpy.utilities.solver_context import SolverInfo - -def my_atom_canon(expr, args, solver_context: SolverInfo | None = None): - x = args[0] - t = Variable(expr.shape) - # For CONVEX atoms: use t >= f(x) - # When minimizing, optimizer pushes t down to equality: t = f(x) - # For CONCAVE atoms: use t <= f(x) - # When maximizing, optimizer pushes t up to equality: t = f(x) - constraints = [t >= x] # Example for convex atom - return t, constraints -``` - -### 3. Register -In `cvxpy/reductions/dcp2cone/canonicalizers/__init__.py`: -```python -from cvxpy.atoms import my_atom -CANON_METHODS[my_atom] = my_atom_canon -``` +- **Smooth**: functions that are both ESR and HSR (analogous to affine in DCP) +- **ESR** (Essentially Smooth Respecting): can be minimized or appear in `<= 0` constraints +- **HSR** (Hierarchically Smooth Respecting): can be maximized or appear in `>= 0` constraints -### 4. Export -In `cvxpy/atoms/__init__.py`: -```python -from cvxpy.atoms.my_atom import my_atom -``` +Use `expr.is_smooth()`, `expr.is_esr()`, `expr.is_hsr()` to check expression properties. ## Testing -Tests should be **comprehensive but concise and focused**. Cover edge cases without unnecessary verbosity. +Tests should be comprehensive but concise. Use `solver=cp.CLARABEL` for tests that call `problem.solve()`. -**IMPORTANT:** Use `solver=cp.CLARABEL` for tests that call `problem.solve()` - it's the default open-source solver. - -### Base Test Pattern ```python from cvxpy.tests.base_test import BaseTest import cvxpy as cp @@ -200,47 +185,10 @@ import numpy as np class TestMyFeature(BaseTest): def test_basic(self) -> None: - x = cp.Variable(2) - atom = cp.my_atom(x) - - # Test DCP - self.assertTrue(atom.is_convex()) - - # Test numeric - x.value = np.array([1.0, 2.0]) - self.assertItemsAlmostEqual(atom.value, expected) - - def test_solve(self) -> None: x = cp.Variable(2) prob = cp.Problem(cp.Minimize(cp.sum(x)), [x >= 1]) prob.solve(solver=cp.CLARABEL) self.assertEqual(prob.status, cp.OPTIMAL) ``` -### Assertion Helpers -- `self.assertItemsAlmostEqual(a, b, places=5)` - Compare arrays -- `self.assertAlmostEqual(a, b, places=5)` - Compare scalars - -## Canon Backend Architecture - -Backends are critical to performance. They handle matrix construction during `ConeMatrixStuffing`. Located in `cvxpy/lin_ops/backends/`. - -**Backends:** -- `CPP` (default) - C++ implementation, fastest for problems with large expression trees -- `SCIPY` - Pure Python with SciPy sparse matrices, good for large problems -- `COO` - 3D COO tensor, better for large DPP problems with many parameters - -Select via `CVXPY_DEFAULT_CANON_BACKEND=CPP` (default), `SCIPY`, or `COO`. - -## Pull Requests - -Always use the PR template in `.github/` when opening PRs. Fill out all sections. **Never check an item in the Contribution Checklist that has not actually been done.** - -## Common Mistakes to Avoid - -1. **Forgetting to register canonicalizers** in `canonicalizers/__init__.py` -2. **Forgetting to export atoms** in `cvxpy/atoms/__init__.py` -3. Missing `is_incr`/`is_decr` methods in atoms (breaks DCP analysis) -4. Not testing with `Parameter` objects (DPP compliance) -5. Missing license headers on new files -6. **Forgetting to update documentation** - new features need docs at [cvxpy.org](https://www.cvxpy.org/) (see `doc/` folder) +NLP tests are in `cvxpy/tests/nlp_tests/` with Jacobian and Hessian verification tests. diff --git a/README.md b/README.md index e108d63187..0f1a8109f3 100644 --- a/README.md +++ b/README.md @@ -1,156 +1,60 @@ -CVXPY -===================== -[![Build Status](https://github.com/cvxpy/cvxpy/actions/workflows/build.yml/badge.svg?event=push)](https://github.com/cvxpy/cvxpy/actions/workflows/build.yml) -![PyPI - downloads](https://img.shields.io/pypi/dm/cvxpy.svg?label=Pypi%20downloads) -![Conda - downloads](https://img.shields.io/conda/dn/conda-forge/cvxpy.svg?label=Conda%20downloads) -[![Discord](https://img.shields.io/badge/Chat-Discord-Blue?color=5865f2)](https://discord.gg/4urRQeGBCr) -[![Benchmarks](http://img.shields.io/badge/benchmarked%20by-asv-blue.svg?style=flat)](https://cvxpy.github.io/benchmarks/) -[![OpenSSF Scorecard](https://api.securityscorecards.dev/projects/github.com/cvxpy/cvxpy/badge)](https://api.securityscorecards.dev/projects/github.com/cvxpy/cvxpy) - -**The CVXPY documentation is at [cvxpy.org](https://www.cvxpy.org/).** - -*We are building a CVXPY community on [Discord](https://discord.gg/4urRQeGBCr). Join the conversation! For issues and long-form discussions, use [Github Issues](https://github.com/cvxpy/cvxpy/issues) and [Github Discussions](https://github.com/cvxpy/cvxpy/discussions).* - -**Contents** -- [Installation](#installation) -- [Getting started](#getting-started) -- [Issues](#issues) -- [Community](#community) -- [Contributing](#contributing) -- [Team](#team) -- [Citing](#citing) - - -CVXPY is a Python-embedded modeling language for convex optimization problems. It allows you to express your problem in a natural way that follows the math, rather than in the restrictive standard form required by solvers. - -For example, the following code solves a least-squares problem where the variable is constrained by lower and upper bounds: - -```python3 -import cvxpy as cp -import numpy - -# Problem data. -m = 30 -n = 20 -numpy.random.seed(1) -A = numpy.random.randn(m, n) -b = numpy.random.randn(m) - -# Construct the problem. -x = cp.Variable(n) -objective = cp.Minimize(cp.sum_squares(A @ x - b)) -constraints = [0 <= x, x <= 1] -prob = cp.Problem(objective, constraints) - -# The optimal objective is returned by prob.solve(). -result = prob.solve() -# The optimal value for x is stored in x.value. -print(x.value) -# The optimal Lagrange multiplier for a constraint -# is stored in constraint.dual_value. -print(constraints[0].dual_value) -``` - -With CVXPY, you can model -* convex optimization problems, -* mixed-integer convex optimization problems, -* geometric programs, and -* quasiconvex programs. - -CVXPY is not a solver. It relies upon the open source solvers -[Clarabel](https://github.com/oxfordcontrol/Clarabel.rs), [SCS](https://github.com/bodono/scs-python), -[OSQP](https://github.com/oxfordcontrol/osqp) and [HiGHS](https://github.com/ERGO-Code/HiGHS). -Additional solvers are [available](https://www.cvxpy.org/tutorial/solvers/index.html#choosing-a-solver), -but must be installed separately. - -CVXPY began as a Stanford University research project. It is now developed by -many people, across many institutions and countries. - +# DNLP — Disciplined Nonlinear Programming +The DNLP package is an extension of [CVXPY](https://www.cvxpy.org/) to general nonlinear programming (NLP). +DNLP allows smooth functions to be freely mixed with nonsmooth convex and concave functions, +with some rules governing how the nonsmooth functions can be used. For details, see our paper [Disciplined Nonlinear Programming](https://web.stanford.edu/~boyd/papers/dnlp.html). +--- ## Installation -CVXPY is available on PyPI, and can be installed with -``` -pip install cvxpy -``` +The installation consists of two steps. -CVXPY can also be installed with conda, using +#### Step 1: Install IPOPT via Conda +DNLP requires an NLP solver. The recommended solver is [Ipopt](https://coin-or.github.io/Ipopt/), which can be installed together with its Python interface [cyipopt](https://github.com/mechmotum/cyipopt): +```bash +conda install -c conda-forge cyipopt ``` -conda install -c conda-forge cvxpy +Installing cyipopt via pip may lead to issues, so we strongly recommend using the Conda installation above, even if the rest of your environment uses pip. + +#### Step 2: Install DNLP +DNLP is installed by cloning this repository and installing it locally: +```bash +git clone https://github.com/cvxgrp/DNLP.git +cd DNLP +pip install . ``` -CVXPY has the following dependencies: - -- Python >= 3.11 -- Clarabel >= 0.5.0 -- OSQP >= 1.0.0 -- SCS >= 3.2.4.post1 -- NumPy >= 2.0.0 -- SciPy >= 1.13.0 -- highspy >= 1.11.0 - -For detailed instructions, see the [installation -guide](https://www.cvxpy.org/install/index.html). - -## Getting started -To get started with CVXPY, check out the following: -* [official CVXPY tutorial](https://www.cvxpy.org/tutorial/index.html) -* [example library](https://www.cvxpy.org/examples/index.html) -* [API reference](https://www.cvxpy.org/api_reference/cvxpy.html) - -## Issues -We encourage you to report issues using the [Github tracker](https://github.com/cvxpy/cvxpy/issues). We welcome all kinds of issues, especially those related to correctness, documentation, performance, and feature requests. - -For basic usage questions (e.g., "Why isn't my problem DCP?"), please use [StackOverflow](https://stackoverflow.com/questions/tagged/cvxpy) instead. - -## Community -The CVXPY community consists of researchers, data scientists, software engineers, and students from all over the world. We welcome you to join us! - -* To chat with the CVXPY community in real-time, join us on [Discord](https://discord.gg/4urRQeGBCr). -* To have longer, in-depth discussions with the CVXPY community, use [Github Discussions](https://github.com/cvxpy/cvxpy/discussions). -* To share feature requests and bug reports, use [Github Issues](https://github.com/cvxpy/cvxpy/issues). - -Please be respectful in your communications with the CVXPY community, and make sure to abide by our [code of conduct](https://github.com/cvxpy/cvxpy/blob/master/CODE_OF_CONDUCT.md). - -## Contributing -We appreciate all contributions. You don't need to be an expert in convex -optimization to help out. - -You should first -install [CVXPY from source](https://www.cvxpy.org/install/index.html#install-from-source). -Here are some simple ways to start contributing immediately: -* Read the CVXPY source code and improve the documentation, or address TODOs -* Enhance the [website documentation](https://github.com/cvxpy/cvxpy/tree/master/doc) -* Browse the [issue tracker](https://github.com/cvxpy/cvxpy/issues), and look for issues tagged as "help wanted" -* Polish the [example library](https://github.com/cvxpy/examples) -* Add a [benchmark](https://github.com/cvxpy/benchmarks) - -If you'd like to add a new example to our library, or implement a new feature, -please get in touch with us first to make sure that your priorities align with -ours. - -Contributions should be submitted as [pull requests](https://github.com/cvxpy/cvxpy/pulls). -A member of the CVXPY development team will review the pull request and guide -you through the contributing process. - -Before starting work on your contribution, please read the [contributing guide](https://github.com/cvxpy/cvxpy/blob/master/CONTRIBUTING.md). - -## Team -CVXPY is a community project, built from the contributions of many -researchers and engineers. +--- +## Example +Below we give a toy example where we maximize a convex quadratic function subject to a nonlinear equality constraint. Many more examples, including the ones in the paper, can be found at [DNLP-examples](https://github.com/cvxgrp/dnlp-examples). +```python +import cvxpy as cp +import numpy as np +import cvxpy as cp -CVXPY is developed and maintained by [Steven -Diamond](https://stevendiamond.me/), [Akshay -Agrawal](https://akshayagrawal.com), [Riley Murray](https://rileyjmurray.wordpress.com/), -[Philipp Schiele](https://www.philippschiele.com/), -[Bartolomeo Stellato](https://stellato.io/), -and [Parth Nobel](https://ptnobel.github.io), with many others contributing -significantly. -A non-exhaustive list of people who have shaped CVXPY over the -years includes Stephen Boyd, Eric Chu, Robin Verschueren, -Jaehyun Park, Enzo Busseti, AJ Friend, Judson Wilson, Chris Dembia, and -William Zhang. +# problem data +np.random.seed(0) +n = 3 +A = np.random.randn(n, n) +A = A.T @ A -For more information about the team and our processes, see our [governance document](https://github.com/cvxpy/org/blob/main/governance.md). +# formulate optimization problem +x = cp.Variable(n) +obj = cp.Maximize(cp.quad_form(x, A)) +constraints = [cp.sum_squares(x) == 1] + +# initialize and solve +x.value = np.ones(n) +prob = cp.Problem(obj, constraints) +prob.solve(nlp=True, verbose=True) +print("Optimal value from DNLP: ", prob.value) + +# the optimal value for this toy problem can also be found by computing the maximum eigenvalue of A +eigenvalues = np.linalg.eigvalsh(A) +print("Maximum eigenvalue: " , np.max(eigenvalues)) +``` -## Citing -If you use CVXPY for academic work, we encourage you to [cite our papers](https://www.cvxpy.org/resources/citing/index.html). If you use CVXPY in industry, we'd love to hear from you as well, on Discord or over email. +--- +## Supported Solvers +| Solver | License | Installation | +|--------|---------|--------------| +| [IPOPT](https://github.com/coin-or/Ipopt) | EPL-2.0 | `conda install -c conda-forge cyipopt` | +| [Knitro](https://www.artelys.com/solvers/knitro/) | Commercial | `pip install knitro` (requires license) | diff --git a/cvxpy/__init__.py b/cvxpy/__init__.py index 1c6d5d7a0a..56763a0668 100644 --- a/cvxpy/__init__.py +++ b/cvxpy/__init__.py @@ -112,6 +112,7 @@ USER_LIMIT as USER_LIMIT, XPRESS as XPRESS, HIGHS as HIGHS, + IPOPT as IPOPT, KNITRO as KNITRO, get_num_threads as get_num_threads, set_num_threads as set_num_threads, diff --git a/cvxpy/atoms/__init__.py b/cvxpy/atoms/__init__.py index e93102681d..f6954c4c39 100644 --- a/cvxpy/atoms/__init__.py +++ b/cvxpy/atoms/__init__.py @@ -72,6 +72,8 @@ from cvxpy.atoms.elementwise.sqrt import sqrt from cvxpy.atoms.elementwise.square import square from cvxpy.atoms.elementwise.xexp import xexp +from cvxpy.atoms.elementwise.trig import sin, cos, tan +from cvxpy.atoms.elementwise.hyperbolic import sinh, asinh, tanh, atanh from cvxpy.atoms.eye_minus_inv import eye_minus_inv, resolvent from cvxpy.atoms.gen_lambda_max import gen_lambda_max from cvxpy.atoms.geo_mean import GeoMean, GeoMeanApprox, geo_mean @@ -161,6 +163,12 @@ ptp ] +NON_SMOOTH_ATOMS = [ + abs, + maximum, + minimum, +] + # DGP atoms whose Dgp2Dcp canonicalization produces ExpCone-requiring DCP atoms. GP_EXP_ATOMS = [ AddExpression, diff --git a/cvxpy/atoms/affine/affine_atom.py b/cvxpy/atoms/affine/affine_atom.py index a89fcd83eb..9c9b494f2a 100644 --- a/cvxpy/atoms/affine/affine_atom.py +++ b/cvxpy/atoms/affine/affine_atom.py @@ -56,6 +56,16 @@ def is_atom_concave(self) -> bool: """Is the atom concave? """ return True + + def is_atom_esr(self) -> bool: + """Is the atom esr? + """ + return True + + def is_atom_hsr(self) -> bool: + """Is the atom hsr? + """ + return True def is_incr(self, idx) -> bool: """Is the composition non-decreasing in argument idx? @@ -109,58 +119,54 @@ def is_nsd(self) -> bool: return True def _grad(self, values) -> List[Any]: - """Gives the (sub/super)gradient of the atom w.r.t. each argument. + """Computes the gradient of the affine atom w.r.t. each argument. - Matrix expressions are vectorized, so the gradient is a matrix. + For affine atoms, the gradient is constant and independent of argument values. + We compute it by constructing the canonical matrix representation and extracting + the linear coefficients. Args: - values: A list of numeric values for the arguments. + values: Argument values (unused for affine atoms). Returns: - A list of SciPy CSC sparse matrices or None. + List of gradient matrices, one for each argument. """ - # TODO should be a simple function in cvxcore for this. - # Make a fake lin op tree for the function. + # Create fake variables for each non-constant argument to build the linear system fake_args = [] var_offsets = {} - offset = 0 + var_length = 0 + for idx, arg in enumerate(self.args): if arg.is_constant(): - fake_args += [Constant(arg.value).canonical_form[0]] + fake_args.append(Constant(arg.value).canonical_form[0]) else: - fake_args += [lu.create_var(arg.shape, idx)] - var_offsets[idx] = offset - offset += arg.size - var_length = offset - fake_expr, _ = self.graph_implementation(fake_args, self.shape, - self.get_data()) - param_to_size = {lo.CONSTANT_ID: 1} - param_to_col = {lo.CONSTANT_ID: 0} - # Get the matrix representation of the function. + fake_args.append(lu.create_var(arg.shape, idx)) + var_offsets[idx] = var_length + var_length += arg.size + + # Get the canonical matrix representation: f(x) = Ax + b + fake_expr, _ = self.graph_implementation(fake_args, self.shape, self.get_data()) canon_mat = canonInterface.get_problem_matrix( - [fake_expr], - var_length, - var_offsets, - param_to_size, - param_to_col, - self.size, + [fake_expr], var_length, var_offsets, + {lo.CONSTANT_ID: 1}, {lo.CONSTANT_ID: 0}, self.size ) - # HACK TODO TODO convert tensors back to vectors. - # COO = (V[lo.CONSTANT_ID][0], (J[lo.CONSTANT_ID][0], I[lo.CONSTANT_ID][0])) - shape = (var_length + 1, self.size) - stacked_grad = canon_mat.reshape(shape).tocsc()[:-1, :] - # Break up into per argument matrices. + + # Extract gradient matrix A (exclude constant offset b) + grad_matrix = canon_mat.reshape((var_length + 1, self.size)).tocsc()[:-1, :] + + # Split gradients by argument grad_list = [] - start = 0 + var_start = 0 for arg in self.args: if arg.is_constant(): - grad_shape = (arg.size, shape[1]) - if grad_shape == (1, 1): - grad_list += [0] - else: - grad_list += [sp.coo_matrix(grad_shape, dtype='float64')] + # Zero gradient for constants + grad_shape = (arg.size, self.size) + grad_list.append(0 if grad_shape == (1, 1) else + sp.coo_matrix(grad_shape, dtype='float64')) else: - stop = start + arg.size - grad_list += [stacked_grad[start:stop, :]] - start = stop - return grad_list + # Extract gradient block for this variable + var_end = var_start + arg.size + grad_list.append(grad_matrix[var_start:var_end, :]) + var_start = var_end + + return grad_list \ No newline at end of file diff --git a/cvxpy/atoms/affine/binary_operators.py b/cvxpy/atoms/affine/binary_operators.py index d4bd108cf8..519465ff5b 100644 --- a/cvxpy/atoms/affine/binary_operators.py +++ b/cvxpy/atoms/affine/binary_operators.py @@ -43,7 +43,6 @@ class BinaryOperator(AffAtom): """ Base class for expressions involving binary operators. (other than addition) - """ OP_NAME = 'BINARY_OP' @@ -555,6 +554,9 @@ def is_decr(self, idx) -> bool: return self.args[1].is_nonpos() else: return self.args[0].is_nonneg() + + def point_in_domain(self): + return np.ones(self.args[1].shape) def graph_implementation( self, arg_objs, shape: Tuple[int, ...], data=None diff --git a/cvxpy/atoms/affine/broadcast_to.py b/cvxpy/atoms/affine/broadcast_to.py index bcf34fe0ed..90bd8c7e20 100644 --- a/cvxpy/atoms/affine/broadcast_to.py +++ b/cvxpy/atoms/affine/broadcast_to.py @@ -30,6 +30,7 @@ class broadcast_to(AffAtom): def __init__(self, expr, shape) -> None: self.broadcast_shape = shape self._shape = expr.shape + self.broadcast_type = None super(broadcast_to, self).__init__(expr) def _supports_cpp(self) -> bool: diff --git a/cvxpy/atoms/affine/index.py b/cvxpy/atoms/affine/index.py index c39e4ccd11..1cbef3e291 100644 --- a/cvxpy/atoms/affine/index.py +++ b/cvxpy/atoms/affine/index.py @@ -141,11 +141,12 @@ def __init__(self, expr: Expression, key) -> None: self.key = key # Order the entries of expr and select them using key. expr = index.cast_to_const(expr) - idx_mat = np.arange(expr.size) - idx_mat = np.reshape(idx_mat, expr.shape, order='F') + idxs = np.arange(expr.size) + idx_mat = np.reshape(idxs, expr.shape, order='F') self._select_mat = idx_mat[key] self._shape = self._select_mat.shape super(special_index, self).__init__(expr) + self._jacobian_operator = None def is_atom_log_log_convex(self) -> bool: """Is the atom log-log convex? diff --git a/cvxpy/atoms/affine/transpose.py b/cvxpy/atoms/affine/transpose.py index 02f5ecce0f..7f5675988b 100644 --- a/cvxpy/atoms/affine/transpose.py +++ b/cvxpy/atoms/affine/transpose.py @@ -116,6 +116,7 @@ def graph_implementation( """ return (lu.transpose(arg_objs[0], self.axes), []) + def permute_dims(expr, axes: List[int]): """Permute the dimensions of the expression. diff --git a/cvxpy/atoms/atom.py b/cvxpy/atoms/atom.py index 025ce9558b..aca2dd6c3e 100644 --- a/cvxpy/atoms/atom.py +++ b/cvxpy/atoms/atom.py @@ -182,12 +182,24 @@ def is_atom_concave(self) -> bool: """Is the atom concave? """ raise NotImplementedError() - + def is_atom_affine(self) -> bool: """Is the atom affine? """ return self.is_atom_concave() and self.is_atom_convex() + def is_atom_esr(self) -> bool: + """Is the atom esr? + """ + raise NotImplementedError("is_atom_esr not implemented for %s." + % self.__class__.__name__) + + def is_atom_hsr(self) -> bool: + """Is the atom hsr? + """ + raise NotImplementedError("is_atom_hsr not implemented for %s." + % self.__class__.__name__) + def is_atom_log_log_convex(self) -> bool: """Is the atom log-log convex? """ @@ -258,6 +270,40 @@ def is_concave(self) -> bool: return True else: return False + + @perf.compute_once + def is_esr(self) -> bool: + """Is the expression epigraph smooth representable? + """ + # Applies DNLP composition rule. + if self.is_constant(): + return True + elif self.is_atom_esr(): + for idx, arg in enumerate(self.args): + if not (arg.is_smooth() or + (arg.is_esr() and self.is_incr(idx)) or + (arg.is_hsr() and self.is_decr(idx))): + return False + return True + else: + return False + + @perf.compute_once + def is_hsr(self) -> bool: + """Is the expression hypograph smooth representable? + """ + # Applies DNLP composition rule. + if self.is_constant(): + return True + elif self.is_atom_hsr(): + for idx, arg in enumerate(self.args): + if not (arg.is_smooth() or + (arg.is_hsr() and self.is_incr(idx)) or + (arg.is_esr() and self.is_decr(idx))): + return False + return True + else: + return False def is_dpp(self, context='dcp') -> bool: """The expression is a disciplined parameterized expression. @@ -511,6 +557,10 @@ def _domain(self) -> List['Constraint']: """ # Default is no constraints. return [] + + def point_in_domain(self) -> np.ndarray: + """default point in domain of zero""" + return np.zeros(self.shape) @staticmethod def numpy_numeric(numeric_func): diff --git a/cvxpy/atoms/elementwise/abs.py b/cvxpy/atoms/elementwise/abs.py index 6cfa1dc8af..8386b6f193 100644 --- a/cvxpy/atoms/elementwise/abs.py +++ b/cvxpy/atoms/elementwise/abs.py @@ -55,6 +55,16 @@ def is_atom_concave(self) -> bool: """Is the atom concave? """ return False + + def is_atom_esr(self) -> bool: + """Is the atom esr? + """ + return True + + def is_atom_hsr(self) -> bool: + """Is the atom hsr? + """ + return False def is_incr(self, idx) -> bool: """Is the composition non-decreasing in argument idx? @@ -90,3 +100,4 @@ def _grad(self, values): D += (values[0] > 0) D -= (values[0] < 0) return [abs.elemwise_grad_to_diag(D, rows, cols)] + diff --git a/cvxpy/atoms/elementwise/ceil.py b/cvxpy/atoms/elementwise/ceil.py index 5597a3f038..bbb7f3456e 100644 --- a/cvxpy/atoms/elementwise/ceil.py +++ b/cvxpy/atoms/elementwise/ceil.py @@ -172,4 +172,4 @@ def _grad(self, values): Returns: A list of SciPy CSC sparse matrices or None. """ - return sp.csc_array(self.args[0].shape) + return sp.csc_array(self.args[0].shape) \ No newline at end of file diff --git a/cvxpy/atoms/elementwise/entr.py b/cvxpy/atoms/elementwise/entr.py index 4c5ce262d3..fb2662e269 100644 --- a/cvxpy/atoms/elementwise/entr.py +++ b/cvxpy/atoms/elementwise/entr.py @@ -57,6 +57,16 @@ def is_atom_concave(self) -> bool: """Is the atom concave? """ return True + + def is_atom_esr(self) -> bool: + """Is the atom esr? + """ + return True + + def is_atom_hsr(self) -> bool: + """Is the atom hsr? + """ + return True def is_incr(self, idx) -> bool: """Is the composition non-decreasing in argument idx? @@ -93,3 +103,6 @@ def _domain(self) -> List[Constraint]: """Returns constraints describing the domain of the node. """ return [self.args[0] >= 0] + + def point_in_domain(self): + return np.ones(self.shape) diff --git a/cvxpy/atoms/elementwise/exp.py b/cvxpy/atoms/elementwise/exp.py index bf3cc52991..7f4c5a17dc 100644 --- a/cvxpy/atoms/elementwise/exp.py +++ b/cvxpy/atoms/elementwise/exp.py @@ -54,6 +54,16 @@ def is_atom_concave(self) -> bool: """Is the atom concave? """ return False + + def is_atom_esr(self) -> bool: + """Is the atom esr? + """ + return True + + def is_atom_hsr(self) -> bool: + """Is the atom hsr? + """ + return True def is_atom_log_log_convex(self) -> bool: """Is the atom log-log convex? diff --git a/cvxpy/atoms/elementwise/huber.py b/cvxpy/atoms/elementwise/huber.py index b19d1d85c0..d8bdbda03f 100644 --- a/cvxpy/atoms/elementwise/huber.py +++ b/cvxpy/atoms/elementwise/huber.py @@ -70,6 +70,16 @@ def is_atom_convex(self) -> bool: def is_atom_concave(self) -> bool: """Is the atom concave?""" return False + + def is_atom_esr(self) -> bool: + """Is the atom esr? + """ + return True + + def is_atom_hsr(self) -> bool: + """Is the atom hsr? + """ + return False def is_incr(self, idx) -> bool: """Is the composition non-decreasing in argument idx?""" @@ -113,3 +123,4 @@ def _grad(self, values): min_val = np.minimum(np.abs(values[0]), self.M.value) grad_vals = 2 * np.multiply(np.sign(values[0]), min_val) return [huber.elemwise_grad_to_diag(grad_vals, rows, cols)] + diff --git a/cvxpy/atoms/elementwise/hyperbolic.py b/cvxpy/atoms/elementwise/hyperbolic.py new file mode 100644 index 0000000000..2453cfa828 --- /dev/null +++ b/cvxpy/atoms/elementwise/hyperbolic.py @@ -0,0 +1,221 @@ +""" +Copyright 2025 CVXPY Developers + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +from typing import List, Tuple + +import numpy as np + +from cvxpy.atoms.elementwise.elementwise import Elementwise +from cvxpy.constraints.constraint import Constraint + + +class sinh(Elementwise): + """Elementwise :math:`\\sinh x`. + """ + + def __init__(self, x) -> None: + super(sinh, self).__init__(x) + + @Elementwise.numpy_numeric + def numeric(self, values): + """Returns the elementwise sinh of x. + """ + return np.sinh(values[0]) + + def sign_from_args(self) -> Tuple[bool, bool]: + """Returns sign (is positive, is negative) of the expression. + """ + # Always unknown. + raise NotImplementedError("sign_from_args not implemented for sinh.") + + def is_atom_convex(self) -> bool: + """Is the atom convex? + """ + return False + + def is_atom_concave(self) -> bool: + """Is the atom concave? + """ + return False + + def is_atom_esr(self) -> bool: + """Is the atom esr? + """ + return True + + def is_atom_hsr(self) -> bool: + """Is the atom hsr? + """ + return True + + def is_incr(self, idx) -> bool: + """Is the composition non-decreasing in argument idx? + """ + return True + + def is_decr(self, idx) -> bool: + """Is the composition non-increasing in argument idx? + """ + return False + + def _domain(self) -> List[Constraint]: + """Returns constraints describing the domain of the node. + """ + return [] + + def _grad(self, values) -> List[Constraint]: + raise NotImplementedError("Gradient not implemented for sinh.") + + +class tanh(Elementwise): + """Elementwise :math:`\\tan x`. + """ + + def __init__(self, x) -> None: + super(tanh, self).__init__(x) + + @Elementwise.numpy_numeric + def numeric(self, values): + """Returns the elementwise hyperbolic tangent of x. + """ + return np.tanh(values[0]) + + def sign_from_args(self) -> Tuple[bool, bool]: + """Returns sign (is positive, is negative) of the expression. + """ + # Always unknown. + raise NotImplementedError("sign_from_args not implemented for tanh.") + + def is_atom_convex(self) -> bool: + """Is the atom convex? + """ + return False + + def is_atom_concave(self) -> bool: + """Is the atom concave? + """ + return False + + def is_atom_esr(self) -> bool: + """Is the atom esr? + """ + return True + + def is_atom_hsr(self) -> bool: + """Is the atom hsr? + """ + return True + + def is_incr(self, idx) -> bool: + """Is the composition non-decreasing in argument idx? + """ + return True + + def is_decr(self, idx) -> bool: + """Is the composition non-increasing in argument idx? + """ + return False + + def _domain(self) -> List[Constraint]: + """Returns constraints describing the domain of the node. + """ + return [] + + def _grad(self, values) -> List[Constraint]: + raise NotImplementedError("Gradient not implemented for tanh.") + + +class asinh(Elementwise): + """Elementwise :math:`\\operatorname{asinh} x` (inverse hyperbolic sine). + """ + + def __init__(self, x) -> None: + super(asinh, self).__init__(x) + + @Elementwise.numpy_numeric + def numeric(self, values): + """Returns the elementwise inverse hyperbolic sine of x. + """ + return np.arcsinh(values[0]) + + def sign_from_args(self) -> Tuple[bool, bool]: + # Always unknown. + raise NotImplementedError("sign_from_args not implemented for asinh.") + + def is_atom_convex(self) -> bool: + return False + + def is_atom_concave(self) -> bool: + return False + + def is_atom_esr(self) -> bool: + return True + + def is_atom_hsr(self) -> bool: + return True + + def is_incr(self, idx) -> bool: + return True + + def is_decr(self, idx) -> bool: + return False + + def _domain(self) -> List[Constraint]: + return [] + + def _grad(self, values) -> List[Constraint]: + raise NotImplementedError("Gradient not implemented for asinh.") + + +class atanh(Elementwise): + """Elementwise :math:`\\operatorname{atanh} x` (inverse hyperbolic tangent). + """ + + def __init__(self, x) -> None: + super(atanh, self).__init__(x) + + @Elementwise.numpy_numeric + def numeric(self, values): + """Returns the elementwise inverse hyperbolic tangent of x. + """ + return np.arctanh(values[0]) + + def sign_from_args(self) -> Tuple[bool, bool]: + # Always unknown. + raise NotImplementedError("sign_from_args not implemented for atanh.") + + def is_atom_convex(self) -> bool: + return False + + def is_atom_concave(self) -> bool: + return False + + def is_atom_esr(self) -> bool: + return True + + def is_atom_hsr(self) -> bool: + return True + + def is_incr(self, idx) -> bool: + return True + + def is_decr(self, idx) -> bool: + return False + + def _domain(self) -> List[Constraint]: + return [self.args[0] < 1, self.args[0] > -1] + + def _grad(self, values) -> List[Constraint]: + raise NotImplementedError("Gradient not implemented for atanh.") \ No newline at end of file diff --git a/cvxpy/atoms/elementwise/kl_div.py b/cvxpy/atoms/elementwise/kl_div.py index 625fb509c9..53343114dc 100644 --- a/cvxpy/atoms/elementwise/kl_div.py +++ b/cvxpy/atoms/elementwise/kl_div.py @@ -54,6 +54,16 @@ def is_atom_concave(self) -> bool: """Is the atom concave? """ return False + + def is_atom_esr(self) -> bool: + """Is the atom esr? + """ + return True + + def is_atom_hsr(self) -> bool: + """Is the atom hsr? + """ + return True def is_incr(self, idx) -> bool: """Is the composition non-decreasing in argument idx? @@ -94,3 +104,6 @@ def _domain(self) -> List[Constraint]: """Returns constraints describing the domain of the node. """ return [self.args[0] >= 0, self.args[1] >= 0] + + def point_in_domain(self, argument=0): + return np.ones(self.args[argument].shape) diff --git a/cvxpy/atoms/elementwise/log.py b/cvxpy/atoms/elementwise/log.py index 1185953f6f..57ea15e208 100644 --- a/cvxpy/atoms/elementwise/log.py +++ b/cvxpy/atoms/elementwise/log.py @@ -55,6 +55,16 @@ def is_atom_concave(self) -> bool: """Is the atom concave? """ return True + + def is_atom_esr(self) -> bool: + """Is the atom esr? + """ + return True + + def is_atom_hsr(self) -> bool: + """Is the atom hsr? + """ + return True def is_atom_log_log_convex(self) -> bool: """Is the atom log-log convex? @@ -101,3 +111,9 @@ def _domain(self) -> List[Constraint]: """Returns constraints describing the domain of the node. """ return [self.args[0] >= 0] + + def point_in_domain(self) -> np.ndarray: + """Returns a point in the domain of the node. + """ + dim = (1, ) if self.size == 1 else self.shape + return np.ones(dim) diff --git a/cvxpy/atoms/elementwise/logistic.py b/cvxpy/atoms/elementwise/logistic.py index bfa9b9a59a..b86ba41245 100644 --- a/cvxpy/atoms/elementwise/logistic.py +++ b/cvxpy/atoms/elementwise/logistic.py @@ -52,6 +52,16 @@ def is_atom_concave(self) -> bool: """Is the atom concave? """ return False + + def is_atom_esr(self) -> bool: + """Is the atom esr? + """ + return True + + def is_atom_hsr(self) -> bool: + """Is the atom hsr? + """ + return True def is_incr(self, idx) -> bool: """Is the composition non-decreasing in argument idx? @@ -78,3 +88,6 @@ def _grad(self, values): cols = self.size grad_vals = np.exp(values[0] - np.logaddexp(0, values[0])) return [logistic.elemwise_grad_to_diag(grad_vals, rows, cols)] + + def point_in_domain(self): + return np.zeros(self.shape) diff --git a/cvxpy/atoms/elementwise/maximum.py b/cvxpy/atoms/elementwise/maximum.py index b6fdfdacf5..c39088c72d 100644 --- a/cvxpy/atoms/elementwise/maximum.py +++ b/cvxpy/atoms/elementwise/maximum.py @@ -66,6 +66,16 @@ def is_atom_concave(self) -> bool: """Is the atom concave? """ return False + + def is_atom_esr(self) -> bool: + """Is the atom esr? + """ + return True + + def is_atom_hsr(self) -> bool: + """Is the atom hsr? + """ + return False def is_atom_log_log_convex(self) -> bool: """Is the atom log-log convex? @@ -115,3 +125,4 @@ def _grad(self, values) -> List[Any]: grad_list += [maximum.elemwise_grad_to_diag(grad_vals, rows, cols)] return grad_list + diff --git a/cvxpy/atoms/elementwise/minimum.py b/cvxpy/atoms/elementwise/minimum.py index 1bd0386fdd..4c1847e8df 100644 --- a/cvxpy/atoms/elementwise/minimum.py +++ b/cvxpy/atoms/elementwise/minimum.py @@ -58,6 +58,16 @@ def is_atom_concave(self) -> bool: """Is the atom concave? """ return True + + def is_atom_esr(self) -> bool: + """Is the atom esr? + """ + return False + + def is_atom_hsr(self) -> bool: + """Is the atom hsr? + """ + return True def is_atom_log_log_convex(self) -> bool: """Is the atom log-log convex? diff --git a/cvxpy/atoms/elementwise/power.py b/cvxpy/atoms/elementwise/power.py index 5168c35860..00188501be 100644 --- a/cvxpy/atoms/elementwise/power.py +++ b/cvxpy/atoms/elementwise/power.py @@ -212,6 +212,16 @@ def is_atom_concave(self) -> bool: # p == 0 is affine here. return _is_const(self.p) and 0 <= self.p.value <= 1 + def is_atom_esr(self) -> bool: + """Is the atom esr? + """ + return _is_const(self.p) + + def is_atom_hsr(self) -> bool: + """Is the atom hsr? + """ + return _is_const(self.p) + def parameters(self): # This is somewhat of a hack. When checking DPP for DGP, # we need to know whether the exponent p is a parameter, because @@ -395,6 +405,11 @@ def _domain(self) -> List[Constraint]: return [self.args[0] >= 0] else: return [] + + def point_in_domain(self) -> np.ndarray: + """Returns a point in the domain of the node. + """ + return np.ones(self.shape) def get_data(self): return [self._p_orig, self.max_denom] diff --git a/cvxpy/atoms/elementwise/rel_entr.py b/cvxpy/atoms/elementwise/rel_entr.py index 76d6b939dd..30cbb21962 100644 --- a/cvxpy/atoms/elementwise/rel_entr.py +++ b/cvxpy/atoms/elementwise/rel_entr.py @@ -52,6 +52,16 @@ def is_atom_concave(self) -> bool: """Is the atom concave? """ return False + + def is_atom_esr(self) -> bool: + """Is the atom esr? + """ + return True + + def is_atom_hsr(self) -> bool: + """Is the atom hsr? + """ + return True def is_incr(self, idx) -> bool: """Is the composition non-decreasing in argument idx? @@ -95,3 +105,6 @@ def _domain(self): """Returns constraints describing the domain of the node. """ return [self.args[0] >= 0, self.args[1] >= 0] + + def point_in_domain(self, argument=0): + return np.ones(self.args[argument].shape) diff --git a/cvxpy/atoms/elementwise/trig.py b/cvxpy/atoms/elementwise/trig.py new file mode 100644 index 0000000000..224ba44455 --- /dev/null +++ b/cvxpy/atoms/elementwise/trig.py @@ -0,0 +1,210 @@ +""" +Copyright 2025 CVXPY Developers + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +from typing import List, Tuple + +import numpy as np + +from cvxpy.atoms.elementwise.elementwise import Elementwise +from cvxpy.constraints.constraint import Constraint + + +class sin(Elementwise): + """Elementwise :math:`\\sin x`. + """ + + def __init__(self, x) -> None: + super(sin, self).__init__(x) + + @Elementwise.numpy_numeric + def numeric(self, values): + """Returns the elementwise sine of x. + """ + return np.sin(values[0]) + + def sign_from_args(self) -> Tuple[bool, bool]: + """Returns sign (is positive, is negative) of the expression. + """ + # Always unknown. + return (False, False) + + def is_atom_convex(self) -> bool: + """Is the atom convex? + """ + return False + + def is_atom_concave(self) -> bool: + """Is the atom concave? + """ + return False + + def is_atom_esr(self) -> bool: + """Is the atom esr? + """ + return True + + def is_atom_hsr(self) -> bool: + """Is the atom hsr? + """ + return True + + def is_incr(self, idx) -> bool: + """Is the composition non-decreasing in argument idx? + """ + return False + + def is_decr(self, idx) -> bool: + """Is the composition non-increasing in argument idx? + """ + return False + + def _domain(self) -> List[Constraint]: + """Returns constraints describing the domain of the node. + """ + return [] + + def _grad(self, values) -> List[Constraint]: + """Returns the gradient of the node. + """ + rows = self.args[0].size + cols = self.size + grad_vals = np.cos(values[0]) + return [sin.elemwise_grad_to_diag(grad_vals, rows, cols)] + + +class cos(Elementwise): + """Elementwise :math:`\\cos x`. + """ + + def __init__(self, x) -> None: + super(cos, self).__init__(x) + + @Elementwise.numpy_numeric + def numeric(self, values): + """Returns the elementwise cosine of x. + """ + return np.cos(values[0]) + + def sign_from_args(self) -> Tuple[bool, bool]: + """Returns sign (is positive, is negative) of the expression. + """ + # Always unknown. + return (False, False) + + def is_atom_convex(self) -> bool: + """Is the atom convex? + """ + return False + + def is_atom_concave(self) -> bool: + """Is the atom concave? + """ + return False + + def is_atom_esr(self) -> bool: + """Is the atom esr? + """ + return True + + def is_atom_hsr(self) -> bool: + """Is the atom hsr? + """ + return True + + def is_incr(self, idx) -> bool: + """Is the composition non-decreasing in argument idx? + """ + return False + + def is_decr(self, idx) -> bool: + """Is the composition non-increasing in argument idx? + """ + return False + + def _domain(self) -> List[Constraint]: + """Returns constraints describing the domain of the node. + """ + return [] + + def _grad(self, values) -> List[Constraint]: + """Returns the gradient of the node. + """ + rows = self.args[0].size + cols = self.size + grad_vals = -np.sin(values[0]) + return [cos.elemwise_grad_to_diag(grad_vals, rows, cols)] + + +class tan(Elementwise): + """Elementwise :math:`\\tan x`. + """ + + def __init__(self, x) -> None: + super(tan, self).__init__(x) + + @Elementwise.numpy_numeric + def numeric(self, values): + """Returns the elementwise tangent of x. + """ + return np.tan(values[0]) + + def sign_from_args(self) -> Tuple[bool, bool]: + """Returns sign (is positive, is negative) of the expression. + """ + # Always unknown. + return (False, False) + + def is_atom_convex(self) -> bool: + """Is the atom convex? + """ + return False + + def is_atom_concave(self) -> bool: + """Is the atom concave? + """ + return False + + def is_atom_esr(self) -> bool: + """Is the atom esr? + """ + return True + + def is_atom_hsr(self) -> bool: + """Is the atom hsr? + """ + return True + + def is_incr(self, idx) -> bool: + """Is the composition non-decreasing in argument idx? + """ + return False + + def is_decr(self, idx) -> bool: + """Is the composition non-increasing in argument idx? + """ + return False + + def _domain(self) -> List[Constraint]: + """Returns constraints describing the domain of the node. + """ + return [-np.pi/2 < self.args[0], self.args[0] < np.pi/2] + + def _grad(self, values) -> List[Constraint]: + """Returns the gradient of the node. + """ + rows = self.args[0].size + cols = self.size + grad_vals = 1/np.cos(values[0])**2 + return [tan.elemwise_grad_to_diag(grad_vals, rows, cols)] diff --git a/cvxpy/atoms/elementwise/xexp.py b/cvxpy/atoms/elementwise/xexp.py index 0515ba897f..744c2babf5 100644 --- a/cvxpy/atoms/elementwise/xexp.py +++ b/cvxpy/atoms/elementwise/xexp.py @@ -59,6 +59,16 @@ def is_atom_log_log_concave(self) -> bool: """Is the atom log-log concave? """ return False + + def is_atom_esr(self) -> bool: + """Is the atom esr? + """ + return True + + def is_atom_hsr(self) -> bool: + """Is the atom hsr? + """ + return True def is_incr(self, idx) -> bool: """Is the composition non-decreasing in argument idx? @@ -92,4 +102,4 @@ def _grad(self, values): def _domain(self) -> List[Constraint]: """Returns constraints describing the domain of the node. """ - return [self.args[0] >= 0] + return [self.args[0] >= 0] \ No newline at end of file diff --git a/cvxpy/atoms/geo_mean.py b/cvxpy/atoms/geo_mean.py index 4c54db5b37..1befdee402 100644 --- a/cvxpy/atoms/geo_mean.py +++ b/cvxpy/atoms/geo_mean.py @@ -311,6 +311,14 @@ def is_atom_concave(self) -> bool: """ return True + def is_atom_esr(self) -> bool: + """Is the atom esr?""" + return True + + def is_atom_hsr(self) -> bool: + """Is the atom hsr?""" + return True + def is_atom_log_log_convex(self) -> bool: """Is the atom log-log convex? """ diff --git a/cvxpy/atoms/log_sum_exp.py b/cvxpy/atoms/log_sum_exp.py index 2cef2d74af..f23eea3be8 100644 --- a/cvxpy/atoms/log_sum_exp.py +++ b/cvxpy/atoms/log_sum_exp.py @@ -81,6 +81,16 @@ def is_atom_concave(self) -> bool: """Is the atom concave? """ return False + + def is_atom_esr(self) -> bool: + """Is the atom esr? + """ + return True + + def is_atom_hsr(self) -> bool: + """Is the atom hsr? + """ + return True def is_incr(self, idx) -> bool: """Is the composition non-decreasing in argument idx? diff --git a/cvxpy/atoms/max.py b/cvxpy/atoms/max.py index a305dc5444..2cece906d2 100644 --- a/cvxpy/atoms/max.py +++ b/cvxpy/atoms/max.py @@ -99,6 +99,16 @@ def is_atom_concave(self) -> bool: """Is the atom concave? """ return False + + def is_atom_esr(self) -> bool: + """Is the atom esr? + """ + return True + + def is_atom_hsr(self) -> bool: + """Is the atom hsr? + """ + return False def is_atom_log_log_convex(self) -> bool: """Is the atom log-log convex? @@ -110,6 +120,10 @@ def is_atom_log_log_concave(self) -> bool: """ return False + def is_smooth(self): + """max is not a smooth function of its args""" + return False + def is_incr(self, idx) -> bool: """Is the composition non-decreasing in argument idx? """ diff --git a/cvxpy/atoms/min.py b/cvxpy/atoms/min.py index df32d919db..6f71cb0441 100644 --- a/cvxpy/atoms/min.py +++ b/cvxpy/atoms/min.py @@ -99,6 +99,16 @@ def is_atom_concave(self) -> bool: """Is the atom concave? """ return True + + def is_atom_esr(self) -> bool: + """Is the atom esr? + """ + return False + + def is_atom_hsr(self) -> bool: + """Is the atom hsr? + """ + return True def is_atom_log_log_convex(self) -> bool: """Is the atom log-log convex? diff --git a/cvxpy/atoms/norm1.py b/cvxpy/atoms/norm1.py index d88823c180..e1fd8e2de8 100644 --- a/cvxpy/atoms/norm1.py +++ b/cvxpy/atoms/norm1.py @@ -55,6 +55,16 @@ def is_atom_concave(self) -> bool: """Is the atom concave? """ return False + + def is_atom_esr(self) -> bool: + """Is the atom esr? + """ + return True + + def is_atom_hsr(self) -> bool: + """Is the atom hsr? + """ + return False def is_incr(self, idx) -> bool: """Is the composition non-decreasing in argument idx? diff --git a/cvxpy/atoms/norm_inf.py b/cvxpy/atoms/norm_inf.py index c8f8221d3e..d5eb9a0609 100644 --- a/cvxpy/atoms/norm_inf.py +++ b/cvxpy/atoms/norm_inf.py @@ -58,6 +58,16 @@ def is_atom_concave(self) -> bool: """Is the atom concave? """ return False + + def is_atom_esr(self) -> bool: + """Is the atom esr? + """ + return True + + def is_atom_hsr(self) -> bool: + """Is the atom hsr? + """ + return False def is_atom_log_log_convex(self) -> bool: """Is the atom log-log convex? diff --git a/cvxpy/atoms/pnorm.py b/cvxpy/atoms/pnorm.py index 6dac190947..3eeb34078a 100644 --- a/cvxpy/atoms/pnorm.py +++ b/cvxpy/atoms/pnorm.py @@ -176,6 +176,16 @@ def is_atom_concave(self) -> bool: """ return self.p < 1 + def is_atom_esr(self) -> bool: + """Is the atom esr? + """ + return self.p > 1 + + def is_atom_hsr(self) -> bool: + """Is the atom hsr? + """ + return self.p < 1 + def is_atom_log_log_convex(self) -> bool: """Is the atom log-log convex? """ diff --git a/cvxpy/atoms/prod.py b/cvxpy/atoms/prod.py index 3e8324d8a0..7d9e715e73 100644 --- a/cvxpy/atoms/prod.py +++ b/cvxpy/atoms/prod.py @@ -70,6 +70,16 @@ def is_atom_log_log_concave(self) -> bool: """ return True + def is_atom_esr(self) -> bool: + """Is the atom ESR (epigraph smooth representable)? + """ + return True + + def is_atom_hsr(self) -> bool: + """Is the atom HSR (hypograph smooth representable)? + """ + return True + def is_incr(self, idx) -> bool: """Is the composition non-decreasing in argument idx? """ diff --git a/cvxpy/atoms/quad_form.py b/cvxpy/atoms/quad_form.py index 9b610e7be6..c73cdae105 100644 --- a/cvxpy/atoms/quad_form.py +++ b/cvxpy/atoms/quad_form.py @@ -73,6 +73,16 @@ def is_atom_concave(self) -> bool: P = self.args[1] return P.is_constant() and P.is_nsd() + def is_atom_esr(self) -> bool: + """Is the atom esr? + """ + return True + + def is_atom_hsr(self) -> bool: + """Is the atom hsr? + """ + return True + def is_atom_log_log_convex(self) -> bool: """Is the atom log-log convex? """ @@ -182,7 +192,6 @@ def sign_from_args(self) -> Tuple[bool, bool]: def is_quadratic(self) -> bool: return True - def decomp_quad(P, cond=None, rcond=None, lower=True, check_finite: bool = True): """ Compute a matrix decomposition. diff --git a/cvxpy/atoms/quad_over_lin.py b/cvxpy/atoms/quad_over_lin.py index cb2fe75d33..c40794db49 100644 --- a/cvxpy/atoms/quad_over_lin.py +++ b/cvxpy/atoms/quad_over_lin.py @@ -121,6 +121,16 @@ def is_atom_concave(self) -> bool: """ return False + def is_atom_esr(self) -> bool: + """Is the atom esr? + """ + return True + + def is_atom_hsr(self) -> bool: + """Is the atom hsr? + """ + return True + def is_atom_log_log_convex(self) -> bool: """Is the atom log-log convex? """ diff --git a/cvxpy/atoms/sum_largest.py b/cvxpy/atoms/sum_largest.py index 23ae364b79..6b0ddc224c 100644 --- a/cvxpy/atoms/sum_largest.py +++ b/cvxpy/atoms/sum_largest.py @@ -118,6 +118,16 @@ def is_atom_concave(self) -> bool: """Is the atom concave? """ return False + + def is_atom_esr(self) -> bool: + """Is the atom esr? + """ + return True + + def is_atom_hsr(self) -> bool: + """Is the atom hsr? + """ + return False def is_incr(self, idx) -> bool: """Is the composition non-decreasing in argument idx? diff --git a/cvxpy/constraints/nonpos.py b/cvxpy/constraints/nonpos.py index 9be31910fd..c3a842eaba 100644 --- a/cvxpy/constraints/nonpos.py +++ b/cvxpy/constraints/nonpos.py @@ -68,6 +68,13 @@ def is_dcp(self, dpp: bool = False) -> bool: return self.args[0].is_convex() return self.args[0].is_convex() + def is_dnlp(self) -> bool: + """ + A NonPos constraint is DNLP if its + argument is epigraph smooth representable. + """ + return self.args[0].is_esr() + def is_dgp(self, dpp: bool = False) -> bool: return False @@ -127,6 +134,13 @@ def is_dcp(self, dpp: bool = False) -> bool: return self.args[0].is_concave() return self.args[0].is_concave() + def is_dnlp(self) -> bool: + """ + A non-negative constraint is DNLP if + its argument is hypograph smooth representable. + """ + return self.args[0].is_hsr() + def is_dgp(self, dpp: bool = False) -> bool: return False @@ -210,6 +224,13 @@ def is_dcp(self, dpp: bool = False) -> bool: return self.expr.is_convex() return self.expr.is_convex() + def is_dnlp(self) -> bool: + """ + An Inequality constraint is DNLP if its + argument is epigraph smooth representable. + """ + return self.expr.is_esr() + def is_dgp(self, dpp: bool = False) -> bool: if dpp: with scopes.dpp_scope(): diff --git a/cvxpy/constraints/zero.py b/cvxpy/constraints/zero.py index 013a4fd34e..7bcc5e8715 100644 --- a/cvxpy/constraints/zero.py +++ b/cvxpy/constraints/zero.py @@ -58,6 +58,10 @@ def is_dcp(self, dpp: bool = False) -> bool: return self.args[0].is_affine() return self.args[0].is_affine() + def is_dnlp(self) -> bool: + """A zero constraint is DNLP if its argument is smooth representable.""" + return self.args[0].is_smooth() + def is_dgp(self, dpp: bool = False) -> bool: return False @@ -133,6 +137,10 @@ def is_dcp(self, dpp: bool = False) -> bool: return self.expr.is_affine() return self.expr.is_affine() + def is_dnlp(self) -> bool: + """A zero constraint is DNLP if its argument is smooth representable.""" + return self.expr.is_smooth() + def is_dgp(self, dpp: bool = False) -> bool: if dpp: with scopes.dpp_scope(): diff --git a/cvxpy/error.py b/cvxpy/error.py index 6dceba4b9c..44c2018ea1 100644 --- a/cvxpy/error.py +++ b/cvxpy/error.py @@ -42,6 +42,11 @@ class DCPError(Exception): """ +class DNLPError(Exception): + """Error thrown for DNLP violations. + """ + + class DPPError(Exception): """Error thrown for DPP violations. """ diff --git a/cvxpy/expressions/expression.py b/cvxpy/expressions/expression.py index dee06e5a24..5908ff5956 100644 --- a/cvxpy/expressions/expression.py +++ b/cvxpy/expressions/expression.py @@ -327,6 +327,12 @@ def is_affine(self) -> bool: """Is the expression affine? """ return self.is_constant() or (self.is_convex() and self.is_concave()) + + @perf.compute_once + def is_smooth(self) -> bool: + """Is the expression smooth? + """ + return self.is_constant() or (self.is_esr() and self.is_hsr()) @abc.abstractmethod def is_convex(self) -> bool: @@ -339,6 +345,18 @@ def is_concave(self) -> bool: """Is the expression concave? """ raise NotImplementedError() + + #@abc.abstractmethod + def is_esr(self) -> bool: + """Is the expression esr? + """ + raise NotImplementedError() + + #@abc.abstractmethod + def is_hsr(self) -> bool: + """Is the expression hsr? + """ + raise NotImplementedError() @perf.compute_once def is_dcp(self, dpp: bool = False) -> bool: @@ -360,6 +378,12 @@ def is_dcp(self, dpp: bool = False) -> bool: return self.is_convex() or self.is_concave() return self.is_convex() or self.is_concave() + def is_dnlp(self) -> bool: + """ + The expression is smooth representable. + """ + return self.is_esr() or self.is_hsr() + def is_log_log_constant(self) -> bool: """Is the expression log-log constant, ie, elementwise positive? """ diff --git a/cvxpy/expressions/leaf.py b/cvxpy/expressions/leaf.py index 9b58b8405a..42d8a72017 100644 --- a/cvxpy/expressions/leaf.py +++ b/cvxpy/expressions/leaf.py @@ -263,6 +263,14 @@ def is_concave(self) -> bool: """Is the expression concave?""" return True + def is_esr(self) -> bool: + """Is the expression esr?""" + return True + + def is_hsr(self) -> bool: + """Is the expression hsr?""" + return True + def is_log_log_convex(self) -> bool: """Is the expression log-log convex?""" return self.is_pos() @@ -663,9 +671,18 @@ def _validate_value(self, val, sparse_path=False): attr_str = 'in bounds' else: attr_str = ([k for (k, v) in self.attributes.items() if v] + ['real'])[0] - raise ValueError( - "%s value must be %s." % (self.__class__.__name__, attr_str) - ) + if np.isnan(val).any() and self.variables(): + # necessary for NLP package extension and computing the structural jacobian + # Only allow NaN for Variables, not Parameters + return val + elif np.isnan(val).any(): + raise ValueError( + "%s value must be real." % self.__class__.__name__ + ) + else: + raise ValueError( + "%s value must be %s." % (self.__class__.__name__, attr_str) + ) return val def is_psd(self) -> bool: diff --git a/cvxpy/expressions/variable.py b/cvxpy/expressions/variable.py index d7cfd43d87..e034305feb 100644 --- a/cvxpy/expressions/variable.py +++ b/cvxpy/expressions/variable.py @@ -52,6 +52,8 @@ def __init__( self._value = None self.delta = None self.gradient = None + # bounds for sampling initial points in DNLP problems + self.sample_bounds = None super(Variable, self).__init__(shape, **kwargs) def name(self) -> str: diff --git a/cvxpy/problems/objective.py b/cvxpy/problems/objective.py index 867e2d2d60..e6930ea42d 100644 --- a/cvxpy/problems/objective.py +++ b/cvxpy/problems/objective.py @@ -156,6 +156,12 @@ def is_dcp(self, dpp: bool = False) -> bool: return self.args[0].is_convex() return self.args[0].is_convex() + def is_dnlp(self) -> bool: + """ + The objective must be epigraph smooth representable. + """ + return self.args[0].is_esr() + def is_dgp(self, dpp: bool = False) -> bool: """The objective must be log-log convex. """ @@ -227,6 +233,12 @@ def is_dcp(self, dpp: bool = False) -> bool: return self.args[0].is_concave() return self.args[0].is_concave() + def is_dnlp(self) -> bool: + """ + The objective must be hypograph smooth representable. + """ + return self.args[0].is_hsr() + def is_dgp(self, dpp: bool = False) -> bool: """The objective must be log-log concave. """ diff --git a/cvxpy/problems/param_prob.py b/cvxpy/problems/param_prob.py index fae25466ab..2784dcb79b 100644 --- a/cvxpy/problems/param_prob.py +++ b/cvxpy/problems/param_prob.py @@ -15,6 +15,8 @@ """ import abc +import numpy as np + class ParamProb(metaclass=abc.ABCMeta): """An abstract base class for parameterized problems. @@ -41,3 +43,25 @@ def apply_parameters(self, id_to_param_value=None, zero_offset: bool = False, parameters are affected """ raise NotImplementedError() + + def split_solution(self, sltn, active_vars=None): + """Splits the solution into individual variables.""" + # Import here to avoid circular imports. + from cvxpy.reductions import cvx_attr2constr + if active_vars is None: + active_vars = [v.id for v in self.variables] + sltn_dict = {} + for var_id, col in self.var_id_to_col.items(): + if var_id in active_vars: + var = self.id_to_var[var_id] + value = sltn[col:var.size + col] + if var.attributes_were_lowered(): + orig_var = var.variable_of_provenance() + value = cvx_attr2constr.recover_value_for_variable( + orig_var, value, project=False) + sltn_dict[orig_var.id] = np.reshape( + value, orig_var.shape, order='F') + else: + sltn_dict[var_id] = np.reshape( + value, var.shape, order='F') + return sltn_dict diff --git a/cvxpy/problems/problem.py b/cvxpy/problems/problem.py index a2c40559fc..08c0912f7b 100644 --- a/cvxpy/problems/problem.py +++ b/cvxpy/problems/problem.py @@ -39,12 +39,18 @@ from cvxpy.problems.objective import Maximize, Minimize from cvxpy.reductions import InverseData from cvxpy.reductions.chain import Chain +from cvxpy.reductions.cvx_attr2constr import CvxAttr2Constr +from cvxpy.reductions.dnlp2smooth.dnlp2smooth import Dnlp2Smooth from cvxpy.reductions.dqcp2dcp import dqcp2dcp from cvxpy.reductions.eval_params import EvalParams from cvxpy.reductions.flip_objective import FlipObjective from cvxpy.reductions.solution import INF_OR_UNB_MESSAGE from cvxpy.reductions.solvers import bisection from cvxpy.reductions.solvers import defines as slv_def +from cvxpy.reductions.solvers.nlp_solvers.copt_nlpif import COPT as COPT_nlp +from cvxpy.reductions.solvers.nlp_solvers.ipopt_nlpif import IPOPT as IPOPT_nlp +from cvxpy.reductions.solvers.nlp_solvers.knitro_nlpif import KNITRO as KNITRO_nlp +from cvxpy.reductions.solvers.nlp_solvers.uno_nlpif import UNO as UNO_nlp from cvxpy.reductions.solvers.solver_inverse_data import SolverInverseData from cvxpy.reductions.solvers.solving_chain import ( SolvingChain, @@ -290,6 +296,13 @@ def is_dcp(self, dpp: bool = False) -> bool: """ return all( expr.is_dcp(dpp) for expr in self.constraints + [self.objective]) + + @perf.compute_once + def is_dnlp(self) -> bool: + """ + Does the problem satisfy disciplined nonlinear programming (DNLP) rules? + """ + return all(expr.is_dnlp() for expr in self.constraints + [self.objective]) @perf.compute_once def _max_ndim(self) -> int: @@ -672,7 +685,7 @@ def get_problem_data( ignore_dpp: bool = False, verbose: bool = False, canon_backend: str | None = None, - solver_opts: Optional[dict] = None + solver_opts: Optional[dict] = None, ): """Returns the problem data used in the call to the solver. @@ -929,6 +942,7 @@ def _solve(self, enforce_dpp: bool = False, ignore_dpp: bool = False, canon_backend: str | None = None, + nlp: bool = False, **kwargs): """Solves a DCP compliant optimization problem. @@ -1065,6 +1079,93 @@ def _solve(self, self.unpack(chain.retrieve(soln)) return self.value + if nlp and self.is_dnlp(): + if type(self.objective) == Maximize: + reductions = [FlipObjective()] + else: + reductions = [] + reductions = reductions + [CvxAttr2Constr(reduce_bounds=False), Dnlp2Smooth()] + # instantiate based on user provided solver + # (default to Ipopt) + if solver is s.IPOPT or solver is None: + nlp_reductions = reductions + [IPOPT_nlp()] + elif "knitro" in solver.lower(): + if solver == "knitro_ipm": + kwargs["algorithm"] = 1 + elif solver == "knitro_sqp": + kwargs["algorithm"] = 4 + elif solver == "knitro_alm": + kwargs["algorithm"] = 6 + nlp_reductions = reductions + [KNITRO_nlp()] + elif solver is s.COPT: + nlp_reductions = reductions + [COPT_nlp()] + elif "uno" in solver.lower(): + if solver.lower() == "uno_ipm": + # Interior-point method (requires MUMPS linear solver) + kwargs["preset"] = "ipopt" + kwargs["linear_solver"] = "MUMPS" + elif solver.lower() == "uno_sqp": + # SQP method (default) + kwargs["preset"] = "filtersqp" + nlp_reductions = reductions + [UNO_nlp()] + else: + raise error.SolverError( + "Solver %s is not supported for NLP problems." % solver + ) + # canonicalize disciplined nlp problems to smooth form + nlp_chain = SolvingChain(reductions=nlp_reductions) + best_of = kwargs.pop("best_of", 1) + + # standard solve + if best_of == 1: + self.set_NLP_initial_point() + canon_problem, inverse_data = nlp_chain.apply(problem=self) + solution = nlp_chain.solver.solve_via_data(canon_problem, warm_start, + verbose, solver_opts=kwargs) + self.unpack_results(solution, nlp_chain, inverse_data) + return self.value + # best-of-N solve + else: + if (not isinstance(best_of, int)) or best_of < 1: + raise ValueError("best_of must be a positive integer.") + + best_obj, best_solution = float("inf"), None + all_objs = np.zeros(shape=(best_of,)) + + for run in range(best_of): + print("Starting NLP solve %d of %d" % (run + 1, best_of)) + self.set_random_NLP_initial_point(run) + canon_problem, inverse_data = nlp_chain.apply(problem=self) + solution = nlp_chain.solver.solve_via_data(canon_problem, warm_start, + verbose, solver_opts=kwargs) + + # This gives the objective value of the C problem + # which can be slightly different from the original NLP + # so we use the below approach with unpacking. Preferably + # we would have a way to do this without unpacking. + #obj_value = canon_problem['objective'](solution['x']) + + # set cvxpy variable + self.unpack_results(solution, nlp_chain, inverse_data) + obj_value = self.objective.value + + all_objs[run] = obj_value + if obj_value < best_obj: + best_obj = obj_value + print("best_obj: ", best_obj) + best_solution = solution + + # unpack best solution + if type(self.objective) == Maximize: + all_objs = -all_objs + + # propagate all objective values to the user + best_solution['all_objs_from_best_of'] = all_objs + self.unpack_results(best_solution, nlp_chain, inverse_data) + return self.value + elif nlp and not self.is_dnlp(): + raise error.DNLPError("The problem you specified is not DNLP.") + data, solving_chain, inverse_data = self.get_problem_data( solver, gp, enforce_dpp, ignore_dpp, verbose, canon_backend, kwargs ) @@ -1410,6 +1511,103 @@ def unpack_results(self, solution, chain: SolvingChain, inverse_data) -> None: self._solver_stats = SolverStats.from_dict(self._solution.attr, chain.solver.name()) + + def set_NLP_initial_point(self) -> dict: + """ Constructs an initial point for the optimization problem. If no + initial value is specified, look at the bounds. If both lb and ub are + specified, we initialize the variables to be their midpoints. If only + one of them is specified, we initialize the variable one unit from + the bound. If none of them is specified, we initialize it to zero. + """ + for var in self.variables(): + if var.value is not None: + continue + + bounds = var.bounds + is_nonneg = var.is_nonneg() + is_nonpos = var.is_nonpos() + + if bounds is None: + if is_nonneg: + x0 = np.ones(var.shape) + elif is_nonpos: + x0 = -np.ones(var.shape) + else: + x0 = np.zeros(var.shape) + + var.save_value(x0) + else: + lb, ub = bounds + + if is_nonneg: + lb = np.maximum(lb, 0) + elif is_nonpos: + ub = np.maximum(ub, 0) + + lb_finite = np.isfinite(lb) + ub_finite = np.isfinite(ub) + # Replace infs with zero for arithmetic + lb0 = np.where(lb_finite, lb, 0.0) + ub0 = np.where(ub_finite, ub, 0.0) + # Midpoint if both finite, one from bound if only one finite, zero if none + init = (lb_finite * ub_finite * 0.5 * (lb0 + ub0) + + lb_finite * (~ub_finite) * (lb0 + 1.0) + + (~lb_finite) * ub_finite * (ub0 - 1.0)) + # Broadcast to variable shape (handles scalar bounds) + init = np.broadcast_to(init, var.shape).copy() + var.save_value(init) + + + def set_random_NLP_initial_point(self, run) -> dict: + """ Generates a random initial point for DNLP problems. + A variable is initialized randomly in the following cases: + 1. 'sample_bounds' is set for that variable. + 2. the initial value specified by the user is None, + 'sample_bounds' is not set for that variable, but the + variable has both finite lower and upper bounds. + """ + + # store user-specified initial values for variables that do + # not have sample bounds assigned + if run == 0: + self._user_initials = {} + for var in self.variables(): + if var.sample_bounds is not None: + self._user_initials[var.id] = None + else: + self._user_initials[var.id] = var.value + + for var in self.variables(): + + # skip variables with user-specified initial value + # (note that any variable with sample bounds set will have + # _user_initials[var.id] == None) + if self._user_initials[var.id] is not None: + # reset to user-specified initial value from last solve + var.value = self._user_initials[var.id] + continue + else: + # reset to None from last solve + var.value = None + + # set sample_bounds to variable bounds if sample_bounds is None + # and variable bounds (possibly infinite) are set + if var.sample_bounds is None and var.bounds is not None: + var.sample_bounds = var.bounds + + # sample initial value if sample_bounds is set + if var.sample_bounds is not None: + low, high = var.sample_bounds + if not np.all(np.isfinite(low)) or not np.all(np.isfinite(high)): + raise ValueError( + "Variable %s has non-finite sample_bounds %s. Cannot generate" + " random initial point. Either add sample bounds or set the value. " + % (var.name(), var.sample_bounds) + ) + + initial_val = np.random.uniform(low=low, high=high, size=var.shape) + var.save_value(initial_val) + def __str__(self) -> str: if len(self.constraints) == 0: return str(self.objective) diff --git a/cvxpy/reductions/cvx_attr2constr.py b/cvxpy/reductions/cvx_attr2constr.py index ba34d1ff3b..d0d567212f 100644 --- a/cvxpy/reductions/cvx_attr2constr.py +++ b/cvxpy/reductions/cvx_attr2constr.py @@ -178,6 +178,8 @@ def apply(self, problem): new_attr['bounds'] = transformed_bounds sparse_var = Variable(n, var_id=var.id, **new_attr) + if var.value_sparse is not None: + sparse_var.value = var.value_sparse.data sparse_var.set_variable_of_provenance(var) id2new_var[var.id] = sparse_var row_idx = np.ravel_multi_index(var.sparse_idx, var.shape, order='F') @@ -188,6 +190,10 @@ def apply(self, problem): obj = reshape(coeff_matrix @ sparse_var, var.shape, order='F') elif var.attributes['diag']: diag_var = Variable(var.shape[0], var_id=var.id, **new_attr) + if var.value is not None and sp.issparse(var.value): + diag_var.value = var.value.diagonal() + elif var.value is not None: + diag_var.value = np.diag(var.value) diag_var.set_variable_of_provenance(var) id2new_var[var.id] = diag_var obj = diag(diag_var) diff --git a/cvxpy/reductions/dcp2cone/cone_matrix_stuffing.py b/cvxpy/reductions/dcp2cone/cone_matrix_stuffing.py index 55436876f1..f3500881fb 100644 --- a/cvxpy/reductions/dcp2cone/cone_matrix_stuffing.py +++ b/cvxpy/reductions/dcp2cone/cone_matrix_stuffing.py @@ -301,28 +301,6 @@ def apply_param_jac(self, delc, delA, delb, active_params=None): delta, param.shape, order='F') return param_id_to_delta_param - def split_solution(self, sltn, active_vars=None): - """Splits the solution into individual variables. - """ - if active_vars is None: - active_vars = [v.id for v in self.variables] - # var id to solution. - sltn_dict = {} - for var_id, col in self.var_id_to_col.items(): - if var_id in active_vars: - var = self.id_to_var[var_id] - value = sltn[col:var.size+col] - if var.attributes_were_lowered(): - orig_var = var.variable_of_provenance() - value = cvx_attr2constr.recover_value_for_variable( - orig_var, value, project=False) - sltn_dict[orig_var.id] = np.reshape( - value, orig_var.shape, order='F') - else: - sltn_dict[var_id] = np.reshape( - value, var.shape, order='F') - return sltn_dict - def split_adjoint(self, del_vars=None): """Adjoint of split_solution. """ @@ -413,6 +391,11 @@ def apply(self, problem): con = ExpCone(x.flatten(order='F'), y.flatten(order='F'), z.flatten(order='F'), constr_id=con.constr_id) cons.append(con) + + # Branch: diff engine path builds C expression trees instead of tensors. + if self.canon_backend == s.DIFF_ENGINE_CANON_BACKEND: + return self._apply_diff_engine(problem, cons, inverse_data) + # Need to check that intended canonicalization backend still works. lowered_con_problem = problem.copy([problem.objective, cons]) canon_backend = get_canon_backend(lowered_con_problem, self.canon_backend) @@ -466,6 +449,64 @@ def apply(self, problem): ) return new_prob, inverse_data + def _apply_diff_engine(self, problem, cons, inverse_data): + """Build a DiffEngineParamConeProg instead of a tensor-based ParamConeProg.""" + from cvxpy.reductions.dcp2cone.diff_engine_param_cone_prog import ( + DiffEngineParamConeProg, + ) + + # Reorder constraints identically to the tensor path. + constr_map = group_constraints(cons) + ordered_cons = ( + constr_map[Zero] + constr_map[NonNeg] + + constr_map[SOC] + constr_map[PSD] + constr_map[ExpCone] + + constr_map[PowCone3D] + constr_map[PowConeND] + ) + inverse_data.cons_id_map = {con.id: con.id for con in ordered_cons} + inverse_data.constraints = ordered_cons + inverse_data.minimize = type(problem.objective) == Minimize + + # Build flattened variable (same logic as stuffed_objective). + boolean, integer = extract_mip_idx(problem.variables()) + flattened_variable = Variable( + inverse_data.x_length, boolean=boolean, integer=integer + ) + + # Bounds. + variables = problem.variables() + if _has_parametric_bounds(variables): + # Use SCIPY backend for bounds tensors (small, parameter-only). + lb_tensor = extract_bounds_tensor( + variables, flattened_variable.size, + inverse_data.param_to_size, inverse_data.param_id_map, + s.SCIPY_CANON_BACKEND, which='lower') + ub_tensor = extract_bounds_tensor( + variables, flattened_variable.size, + inverse_data.param_to_size, inverse_data.param_id_map, + s.SCIPY_CANON_BACKEND, which='upper') + lower_bounds = None + upper_bounds = None + else: + lb_tensor = ub_tensor = None + lower_bounds = extract_lower_bounds(variables, flattened_variable.size) + upper_bounds = extract_upper_bounds(variables, flattened_variable.size) + + new_prob = DiffEngineParamConeProg( + x=flattened_variable, + variables=variables, + var_id_to_col=inverse_data.var_offsets, + ordered_cons=ordered_cons, + parameters=problem.parameters(), + param_id_to_col=inverse_data.param_id_map, + has_quad_obj=self.quad_obj, + objective_expr=problem.objective.expr, + lower_bounds=lower_bounds, + upper_bounds=upper_bounds, + lb_tensor=lb_tensor, + ub_tensor=ub_tensor, + ) + return new_prob, inverse_data + def invert(self, solution, inverse_data): """Retrieves a solution to the original problem""" var_map = inverse_data.var_offsets diff --git a/cvxpy/reductions/dcp2cone/diff_engine_param_cone_prog.py b/cvxpy/reductions/dcp2cone/diff_engine_param_cone_prog.py new file mode 100644 index 0000000000..142ccd7374 --- /dev/null +++ b/cvxpy/reductions/dcp2cone/diff_engine_param_cone_prog.py @@ -0,0 +1,256 @@ +"""DiffEngineParamConeProg: parameter application via the C diff engine. + +Instead of building sparse tensors and multiplying by the parameter vector, +this class builds a C expression tree once and evaluates derivatives directly. + +Key insight (evaluate-at-zero trick): After DCP2Cone, all constraint +expressions are affine in x and the objective is linear or quadratic. So: +- Jacobian(constraints) = A (constant, since affine) +- constraint_forward(0) = b +- gradient(objective) at x=0 = q +- objective_forward(0) = d +- Hessian(objective) = P (for QP; constant since quadratic) + +Copyright 2025, the CVXPY developers + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +from __future__ import annotations + +import numpy as np +from scipy import sparse + +try: + from sparsediffpy import _sparsediffengine as _diffengine +except ImportError as e: + raise ImportError( + "Diff engine backend requires sparsediffpy. Install with: pip install sparsediffpy" + ) from e + +from cvxpy.problems.param_prob import ParamProb +from cvxpy.reductions.dcp2cone.cone_matrix_stuffing import ConeDims +from cvxpy.reductions.solvers.nlp_solvers.diff_engine.converters import ( + build_parameter_dict, + build_variable_dict, + convert_expr, +) +from cvxpy.reductions.utilities import group_constraints + + +class DiffEngineParamConeProg(ParamProb): + """Parameterized cone program backed by the C diff engine. + + Duck-types ParamConeProg for the interface consumed by + ConicSolver._prepare_data_and_inv_data() and ConicSolver.apply(). + """ + + def __init__( + self, + x, + variables, + var_id_to_col, + ordered_cons, + parameters, + param_id_to_col, + has_quad_obj, + objective_expr, + lower_bounds=None, + upper_bounds=None, + lb_tensor=None, + ub_tensor=None, + ): + # Variable info. + self.x = x + self.variables = variables + self.var_id_to_col = var_id_to_col + self.id_to_var = {v.id: v for v in self.variables} + + # Constraint info. + self.constraints = ordered_cons + self.constr_size = sum(c.size for c in ordered_cons) + self.constr_map = group_constraints(ordered_cons) + self.cone_dims = ConeDims(self.constr_map) + + # Parameter info. + self.parameters = parameters + self.param_id_to_col = param_id_to_col + self.id_to_param = {p.id: p for p in self.parameters} + self.param_id_to_size = {p.id: p.size for p in self.parameters} + self.total_param_size = sum(p.size for p in self.parameters) + + # Bounds. + self.lower_bounds = lower_bounds + self.upper_bounds = upper_bounds + self.lb_tensor = lb_tensor + self.ub_tensor = ub_tensor + + # For compatibility: P is set to a truthy sentinel when quad objective + # is present, since ConicSolver.apply checks `problem.P is None`. + self.P = True if has_quad_obj else None + + # Not yet formatted (restructured) for a specific solver. + self.formatted = False + + # Restructuring matrix set by format_constraints. + self._restruct_mat = None + + # ---- Build C expression tree ---- + self._has_quad_obj = has_quad_obj + + var_dict, n_vars = build_variable_dict(self.variables) + + all_params = list({p.id: p for p in self.parameters}.values()) + if all_params: + param_dict, param_capsules = build_parameter_dict( + all_params, n_vars + ) + else: + param_dict, param_capsules = None, [] + + self._all_params = all_params + + # Convert objective. + c_objective = convert_expr(objective_expr, var_dict, n_vars, param_dict) + + # Convert constraints: decompose into individual arg expressions. + c_constraints = [] + for constr in ordered_cons: + for arg in constr.args: + c_expr = convert_expr(arg, var_dict, n_vars, param_dict) + c_constraints.append(c_expr) + + # Create C problem. + self._capsule = _diffengine.make_problem(c_objective, c_constraints, False) + + # Register parameters. + if param_capsules: + _diffengine.problem_register_params( + self._capsule, param_capsules + ) + + # Initialize derivative structures. + _diffengine.problem_init_jacobian(self._capsule) + if has_quad_obj: + _diffengine.problem_init_hessian(self._capsule) + + # Push initial parameter values. + self._update_params() + + # Precompute zero vector for evaluate-at-zero. + self._zeros = np.zeros(n_vars) + self._n_vars = n_vars + + # Compute total raw constraint size (sum of all arg sizes). + self._raw_constr_size = sum(arg.size for c in ordered_cons for arg in c.args) + + def is_mixed_integer(self): + return self.x.attributes['boolean'] or self.x.attributes['integer'] + + def _update_params(self): + """Push current parameter values to the C expression tree.""" + if not self._all_params: + return + theta = np.empty(sum(p.size for p in self._all_params)) + offset = 0 + for param in self._all_params: + val = np.asarray(param.value, dtype=np.float64).flatten(order='F') + theta[offset:offset + param.size] = val + offset += param.size + _diffengine.problem_update_params(self._capsule, theta) + + def apply_parameters(self, id_to_param_value=None, zero_offset=False, + keep_zeros=False, quad_obj=False): + """Evaluate A, b, q, d (and optionally P) via the diff engine. + + Uses the evaluate-at-zero trick: since all expressions are affine in x + after DCP canonicalization, evaluating at x=0 gives the constant + offsets, and the Jacobian/gradient gives the linear coefficients. + """ + # Update parameter values if provided. + if id_to_param_value is not None: + for param in self._all_params: + if param.id in id_to_param_value: + param.value = id_to_param_value[param.id] + self._update_params() + + # Evaluate objective at zero. + d = _diffengine.problem_objective_forward(self._capsule, self._zeros) + q = _diffengine.problem_gradient(self._capsule) + + # Evaluate constraints at zero. + b_raw = _diffengine.problem_constraint_forward(self._capsule, self._zeros) + # Get Jacobian (CSR). + jac_data, jac_indices, jac_indptr, jac_shape = _diffengine.problem_jacobian( + self._capsule + ) + A_raw = sparse.csr_matrix((jac_data, jac_indices, jac_indptr), shape=jac_shape) + + # Apply restructuring matrix if format_constraints was called. + if self._restruct_mat is not None: + # Restructuring matrix operates on the "stacked args" layout. + A = self._restruct_mat(A_raw) + b = np.asarray(self._restruct_mat( + sparse.csr_matrix(b_raw.reshape(-1, 1)) + ).todense()).flatten() + else: + A = A_raw + b = b_raw + + # Convert A to CSC format (expected by downstream solvers). + if sparse.issparse(A): + A = sparse.csc_array(A) + + # Apply parametric bounds tensors if present. + if self.lb_tensor is not None: + param_vec = self._build_param_vec(zero_offset) + if param_vec is None: + self.lower_bounds = self.lb_tensor.toarray().flatten() + else: + self.lower_bounds = np.asarray( + self.lb_tensor @ param_vec).flatten() + if self.ub_tensor is not None: + param_vec = self._build_param_vec(zero_offset) + if param_vec is None: + self.upper_bounds = self.ub_tensor.toarray().flatten() + else: + self.upper_bounds = np.asarray( + self.ub_tensor @ param_vec).flatten() + + if quad_obj: + # Hessian of the Lagrangian with unit objective weight and zero duals. + n_raw_constrs = self._raw_constr_size + zero_duals = np.zeros(n_raw_constrs) + hess_data, hess_indices, hess_indptr, hess_shape = ( + _diffengine.problem_hessian(self._capsule, 1.0, zero_duals) + ) + P_mat = sparse.csr_matrix( + (hess_data, hess_indices, hess_indptr), shape=hess_shape + ) + return P_mat, q, d, A, np.atleast_1d(b) + else: + return q, d, A, np.atleast_1d(b) + + def _build_param_vec(self, zero_offset=False): + """Build the parameter vector for bounds tensor application.""" + if self.total_param_size == 0: + return None + from cvxpy.cvxcore.python import canonInterface + param_vec = canonInterface.get_parameter_vector( + self.total_param_size, + self.param_id_to_col, + self.param_id_to_size, + lambda idx: np.array(self.id_to_param[idx].value), + zero_offset=zero_offset, + ) + return param_vec + diff --git a/cvxpy/reductions/dnlp2smooth/__init__.py b/cvxpy/reductions/dnlp2smooth/__init__.py new file mode 100644 index 0000000000..d1267fd95b --- /dev/null +++ b/cvxpy/reductions/dnlp2smooth/__init__.py @@ -0,0 +1,15 @@ +""" +Copyright 2025 CVXPY developers + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" diff --git a/cvxpy/reductions/dnlp2smooth/canonicalizers/__init__.py b/cvxpy/reductions/dnlp2smooth/canonicalizers/__init__.py new file mode 100644 index 0000000000..fbb310b9da --- /dev/null +++ b/cvxpy/reductions/dnlp2smooth/canonicalizers/__init__.py @@ -0,0 +1,107 @@ +""" +Copyright 2025 CVXPY developers + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +from cvxpy.atoms.geo_mean import GeoMean, GeoMeanApprox +from cvxpy.atoms.prod import Prod +from cvxpy.atoms.quad_over_lin import quad_over_lin +from cvxpy.atoms.elementwise.exp import exp +from cvxpy.atoms.elementwise.logistic import logistic +from cvxpy.atoms.elementwise.log import log +from cvxpy.atoms.elementwise.entr import entr +from cvxpy.atoms.elementwise.rel_entr import rel_entr +from cvxpy.atoms.elementwise.kl_div import kl_div +from cvxpy.atoms.elementwise.minimum import minimum +from cvxpy.atoms.elementwise.maximum import maximum +from cvxpy.atoms.elementwise.power import Power, PowerApprox +from cvxpy.atoms.elementwise.trig import cos, sin, tan +from cvxpy.atoms.elementwise.hyperbolic import sinh, asinh, tanh, atanh +from cvxpy.atoms.elementwise.huber import huber +from cvxpy.atoms.norm1 import norm1 +from cvxpy.atoms.norm_inf import norm_inf +from cvxpy.atoms.pnorm import Pnorm, PnormApprox +from cvxpy.atoms.sum_largest import sum_largest +from cvxpy.atoms.elementwise.abs import abs +from cvxpy.atoms.max import max +from cvxpy.atoms.min import min +from cvxpy.atoms.log_sum_exp import log_sum_exp +from cvxpy.atoms.affine.binary_operators import DivExpression, MulExpression, multiply +from cvxpy.reductions.dnlp2smooth.canonicalizers.geo_mean_canon import geo_mean_canon +from cvxpy.reductions.dnlp2smooth.canonicalizers.prod_canon import prod_canon +from cvxpy.reductions.dnlp2smooth.canonicalizers.quad_over_lin_canon import quad_over_lin_canon +from cvxpy.reductions.dnlp2smooth.canonicalizers.div_canon import div_canon +from cvxpy.reductions.dnlp2smooth.canonicalizers.log_canon import log_canon +from cvxpy.reductions.dnlp2smooth.canonicalizers.exp_canon import exp_canon +from cvxpy.reductions.dnlp2smooth.canonicalizers.logistic_canon import logistic_canon +from cvxpy.reductions.dnlp2smooth.canonicalizers.multiply_canon import matmul_canon, multiply_canon +from cvxpy.reductions.dnlp2smooth.canonicalizers.pnorm_canon import pnorm_canon +from cvxpy.reductions.dnlp2smooth.canonicalizers.power_canon import power_canon +from cvxpy.reductions.dnlp2smooth.canonicalizers.entr_canon import entr_canon +from cvxpy.reductions.dnlp2smooth.canonicalizers.rel_entr_canon import rel_entr_canon +from cvxpy.reductions.dnlp2smooth.canonicalizers.kl_div_canon import kl_div_canon +from cvxpy.reductions.dnlp2smooth.canonicalizers.trig_canon import cos_canon, sin_canon, tan_canon +from cvxpy.reductions.dnlp2smooth.canonicalizers.hyperbolic_canon import (sinh_canon, asinh_canon, + tanh_canon, atanh_canon) +from cvxpy.reductions.dnlp2smooth.canonicalizers.log_sum_exp_canon import log_sum_exp_canon +from cvxpy.reductions.dnlp2smooth.canonicalizers.huber_canon import huber_canon +from cvxpy.reductions.eliminate_pwl.canonicalizers.norm1_canon import norm1_canon +from cvxpy.reductions.eliminate_pwl.canonicalizers.norm_inf_canon import norm_inf_canon +from cvxpy.reductions.eliminate_pwl.canonicalizers.max_canon import max_canon +from cvxpy.reductions.eliminate_pwl.canonicalizers.min_canon import min_canon +from cvxpy.reductions.eliminate_pwl.canonicalizers.maximum_canon import maximum_canon +from cvxpy.reductions.eliminate_pwl.canonicalizers.minimum_canon import minimum_canon +from cvxpy.reductions.eliminate_pwl.canonicalizers.abs_canon import abs_canon +from cvxpy.reductions.eliminate_pwl.canonicalizers.sum_largest_canon import sum_largest_canon + + +SMOOTH_CANON_METHODS = { + log: log_canon, + exp: exp_canon, + logistic: logistic_canon, + sin: sin_canon, + cos: cos_canon, + tan: tan_canon, + sinh: sinh_canon, + asinh: asinh_canon, + tanh: tanh_canon, + atanh: atanh_canon, + quad_over_lin: quad_over_lin_canon, + Power: power_canon, + PowerApprox: power_canon, + Pnorm: pnorm_canon, + PnormApprox: pnorm_canon, + DivExpression: div_canon, + entr: entr_canon, + rel_entr: rel_entr_canon, + kl_div: kl_div_canon, + multiply: multiply_canon, + MulExpression: matmul_canon, + GeoMean: geo_mean_canon, + GeoMeanApprox: geo_mean_canon, + log_sum_exp: log_sum_exp_canon, + Prod: prod_canon, + + # ESR atoms + abs: abs_canon, + maximum: maximum_canon, + max: max_canon, + norm1: norm1_canon, + norm_inf: norm_inf_canon, + huber: huber_canon, + sum_largest: sum_largest_canon, + + # HSR atoms + minimum: minimum_canon, + min: min_canon, +} diff --git a/cvxpy/reductions/dnlp2smooth/canonicalizers/div_canon.py b/cvxpy/reductions/dnlp2smooth/canonicalizers/div_canon.py new file mode 100644 index 0000000000..b2b42f52b3 --- /dev/null +++ b/cvxpy/reductions/dnlp2smooth/canonicalizers/div_canon.py @@ -0,0 +1,55 @@ +""" +Copyright 2025 CVXPY developers + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import numpy as np + +from cvxpy.atoms.affine.binary_operators import multiply +from cvxpy.expressions.variable import Variable + +MIN_INIT = 1e-4 + +# We canonicalize div(f(x), g(x)) as z * y = f(x), y = g(x), y >= 0. +# In other words, it assumes that the denominator is nonnegative. +def div_canon(expr, args): + dim = args[0].shape + sgn_z = args[0].sign + + if sgn_z == 'NONNEGATIVE': + z = Variable(dim, bounds=[0, None]) + elif sgn_z == 'NONPOSITIVE': + z = Variable(dim, bounds=[None, 0]) + else: + z = Variable(dim) + + y = Variable(args[1].shape, bounds=[0, None]) + + if args[1].value is not None: + y.value = np.maximum(args[1].value, MIN_INIT) + else: + y.value = expr.point_in_domain() + + if args[0].value is not None: + val = args[0].value / y.value + else: + val = expr.point_in_domain() + + # dimension hack + if dim == () and val.shape == (1,): + z.value = val[0] + else: + z.value = val + + return z, [multiply(z, y) == args[0], y == args[1]] \ No newline at end of file diff --git a/cvxpy/reductions/dnlp2smooth/canonicalizers/entr_canon.py b/cvxpy/reductions/dnlp2smooth/canonicalizers/entr_canon.py new file mode 100644 index 0000000000..014c2c2065 --- /dev/null +++ b/cvxpy/reductions/dnlp2smooth/canonicalizers/entr_canon.py @@ -0,0 +1,31 @@ +""" +Copyright 2025 CVXPY developers + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import numpy as np + +from cvxpy.expressions.variable import Variable + +MIN_INIT = 1e-4 + +def entr_canon(expr, args): + t = Variable(args[0].shape, bounds=[0, None]) + if args[0].value is not None: + t.value = np.maximum(args[0].value, MIN_INIT) + else: + t.value = expr.point_in_domain() + + return expr.copy([t]), [t == args[0]] + diff --git a/cvxpy/reductions/dnlp2smooth/canonicalizers/exp_canon.py b/cvxpy/reductions/dnlp2smooth/canonicalizers/exp_canon.py new file mode 100644 index 0000000000..1d2b12dbc4 --- /dev/null +++ b/cvxpy/reductions/dnlp2smooth/canonicalizers/exp_canon.py @@ -0,0 +1,26 @@ +""" +Copyright 2025 CVXPY developers + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +from cvxpy.expressions.variable import Variable + + +def exp_canon(expr, args): + if isinstance(args[0], Variable): + return expr.copy([args[0]]), [] + else: + t = Variable(args[0].shape) + if args[0].value is not None: + t.value = args[0].value + return expr.copy([t]), [t == args[0]] diff --git a/cvxpy/reductions/dnlp2smooth/canonicalizers/geo_mean_canon.py b/cvxpy/reductions/dnlp2smooth/canonicalizers/geo_mean_canon.py new file mode 100644 index 0000000000..e66ef591db --- /dev/null +++ b/cvxpy/reductions/dnlp2smooth/canonicalizers/geo_mean_canon.py @@ -0,0 +1,41 @@ +""" +Copyright 2025 CVXPY developers + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import numpy as np + +from cvxpy.atoms.affine.binary_operators import multiply +from cvxpy.atoms.affine.sum import sum +from cvxpy.atoms.elementwise.log import log +from cvxpy.expressions.variable import Variable +from cvxpy.reductions.dnlp2smooth.canonicalizers.log_canon import log_canon + +MIN_INIT = 1e-4 + +def geo_mean_canon(expr, args): + """ + Canonicalization for the geometric mean function. + """ + t = Variable(expr.shape, nonneg=True) + + if args[0].value is not None: + t.value = np.max((expr.numeric(args[0].value), MIN_INIT)) + else: + t.value = np.ones(expr.shape) + + weights = np.array([float(w) for w in expr.w]) + log_expr = log(args[0]) + var, constr = log_canon(log_expr, expr.args) + return t, [log(t) == sum(multiply(weights, var))] + constr diff --git a/cvxpy/reductions/dnlp2smooth/canonicalizers/huber_canon.py b/cvxpy/reductions/dnlp2smooth/canonicalizers/huber_canon.py new file mode 100644 index 0000000000..9291aa13ef --- /dev/null +++ b/cvxpy/reductions/dnlp2smooth/canonicalizers/huber_canon.py @@ -0,0 +1,55 @@ +""" +Copyright 2025 CVXPY developers + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +# This file is identical to the dcp2cone version, except that it uses the +# dnlp2smooth canonicalizer for the power atom. This is necessary. + +import numpy as np + +from cvxpy.atoms.elementwise.abs import abs +from cvxpy.atoms.elementwise.power import power +from cvxpy.expressions.variable import Variable +from cvxpy.reductions.dnlp2smooth.canonicalizers.power_canon import power_canon +from cvxpy.reductions.eliminate_pwl.canonicalizers.abs_canon import abs_canon + + +def huber_canon(expr, args): + M = expr.M + x = args[0] + shape = expr.shape + n = Variable(shape) + s = Variable(shape) + + if x.value is None: + x.value = np.zeros(x.shape) + + # this choice of initial value follows from how the smooth epigraph + # form of the huber function is constructed + n.value = np.minimum(np.abs(x.value), M.value) * np.sign(x.value) + s.value = x.value - n.value + + # n**2 + 2*M*|s| + power_expr = power(n, 2) + n2, constr_sq = power_canon(power_expr, power_expr.args) + abs_expr = abs(s) + abs_s, constr_abs = abs_canon(abs_expr, abs_expr.args) + obj = n2 + 2 * M * abs_s + + # x == s + n + constraints = constr_sq + constr_abs + constraints.append(x == s + n) + + return obj, constraints diff --git a/cvxpy/reductions/dnlp2smooth/canonicalizers/hyperbolic_canon.py b/cvxpy/reductions/dnlp2smooth/canonicalizers/hyperbolic_canon.py new file mode 100644 index 0000000000..b6b2b48d79 --- /dev/null +++ b/cvxpy/reductions/dnlp2smooth/canonicalizers/hyperbolic_canon.py @@ -0,0 +1,50 @@ +""" +Copyright 2025 CVXPY developers + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +from cvxpy.expressions.variable import Variable + + +def sinh_canon(expr, args): + if isinstance(args[0], Variable): + return expr, [] + else: + t = Variable(args[0].shape) + if args[0].value is not None: + t.value = args[0].value + return expr.copy([t]), [t == args[0]] + +def tanh_canon(expr, args): + if isinstance(args[0], Variable): + return expr, [] + else: + t = Variable(args[0].shape) + if args[0].value is not None: + t.value = args[0].value + return expr.copy([t]), [t == args[0]] + +def asinh_canon(expr, args): + if isinstance(args[0], Variable): + return expr, [] + else: + t = Variable(args[0].shape) + if args[0].value is not None: + t.value = args[0].value + return expr.copy([t]), [t == args[0]] + +def atanh_canon(expr, args): + t = Variable(args[0].shape, bounds=[-1, 1]) + if args[0].value is not None: + t.value = args[0].value + return expr.copy([t]), [t == args[0]] diff --git a/cvxpy/reductions/dnlp2smooth/canonicalizers/kl_div_canon.py b/cvxpy/reductions/dnlp2smooth/canonicalizers/kl_div_canon.py new file mode 100644 index 0000000000..b0888d64e5 --- /dev/null +++ b/cvxpy/reductions/dnlp2smooth/canonicalizers/kl_div_canon.py @@ -0,0 +1,25 @@ +""" +Copyright 2025 CVXPY developers + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +from cvxpy.atoms.elementwise.rel_entr import rel_entr +from cvxpy.reductions.dnlp2smooth.canonicalizers.rel_entr_canon import rel_entr_canon + + +def kl_div_canon(expr, args): + _rel_entr = rel_entr(args[0], args[1]) + rel_entr_expr, constr = rel_entr_canon(_rel_entr, _rel_entr.args) + return rel_entr_expr - args[0] + args[1] , constr + diff --git a/cvxpy/reductions/dnlp2smooth/canonicalizers/log_canon.py b/cvxpy/reductions/dnlp2smooth/canonicalizers/log_canon.py new file mode 100644 index 0000000000..67f8a29074 --- /dev/null +++ b/cvxpy/reductions/dnlp2smooth/canonicalizers/log_canon.py @@ -0,0 +1,31 @@ +""" +Copyright 2025 CVXPY developers + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import numpy as np + +from cvxpy.expressions.variable import Variable + +MIN_INIT = 1e-4 + +def log_canon(expr, args): + t = Variable(args[0].shape, bounds=[0, None]) + + if args[0].value is not None: + t.value = np.maximum(args[0].value, MIN_INIT) + else: + t.value = expr.point_in_domain() + + return expr.copy([t]), [t == args[0]] diff --git a/cvxpy/reductions/dnlp2smooth/canonicalizers/log_sum_exp_canon.py b/cvxpy/reductions/dnlp2smooth/canonicalizers/log_sum_exp_canon.py new file mode 100644 index 0000000000..10c7fc89ee --- /dev/null +++ b/cvxpy/reductions/dnlp2smooth/canonicalizers/log_sum_exp_canon.py @@ -0,0 +1,40 @@ +""" +Copyright 2025 CVXPY developers + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import numpy as np + +from cvxpy.atoms.affine.sum import sum +from cvxpy.atoms.elementwise.exp import exp +from cvxpy.expressions.variable import Variable + + +# t = log_sum_exp(x) is equivalent exp(t) = sum(exp(x)), which is +# equivalent to sum(exp(x - t)) = 1. Now we introduce v = x - t, +# which must be nonpositive. +def log_sum_exp_canon(expr, args): + x = args[0] + t = Variable(expr.shape) + v = Variable(x.shape, nonpos=True) + + if x.value is not None: + t.value = expr.numeric(x.value) + v.value = np.minimum(x.value - t.value, -1) + else: + t.value = np.ones(expr.shape) + v.value = -np.ones(x.shape) + + constraints = [sum(exp(v)) == 1, v == x - t] + return t, constraints diff --git a/cvxpy/reductions/dnlp2smooth/canonicalizers/logistic_canon.py b/cvxpy/reductions/dnlp2smooth/canonicalizers/logistic_canon.py new file mode 100644 index 0000000000..e884c1d161 --- /dev/null +++ b/cvxpy/reductions/dnlp2smooth/canonicalizers/logistic_canon.py @@ -0,0 +1,26 @@ +""" +Copyright 2025 CVXPY developers + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +from cvxpy.expressions.variable import Variable + + +def logistic_canon(expr, args): + if isinstance(args[0], Variable): + return expr.copy([args[0]]), [] + else: + t = Variable(args[0].shape) + if args[0].value is not None: + t.value = args[0].value + return expr.copy([t]), [t == args[0]] diff --git a/cvxpy/reductions/dnlp2smooth/canonicalizers/multiply_canon.py b/cvxpy/reductions/dnlp2smooth/canonicalizers/multiply_canon.py new file mode 100644 index 0000000000..d302d55e73 --- /dev/null +++ b/cvxpy/reductions/dnlp2smooth/canonicalizers/multiply_canon.py @@ -0,0 +1,63 @@ +""" +Copyright 2025 CVXPY developers + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + + +from cvxpy.expressions.variable import Variable + + +# If a user insert x * x where x is a variable it gets canonicalized to +# square(x) before this function is called. +def multiply_canon(expr, args): + t1 = args[0] + t2 = args[1] + constraints = [] + + # if either is constant, no canonicalization needed + if t1.is_constant() or t2.is_constant(): + return expr.copy([t1, t2]), [] + + if not isinstance(t1, Variable): + t1 = Variable(t1.shape) + constraints += [t1 == args[0]] + t1.value = args[0].value + + if not isinstance(t2, Variable): + t2 = Variable(t2.shape) + constraints += [t2 == args[1]] + t2.value = args[1].value + + return expr.copy([t1, t2]), constraints + +def matmul_canon(expr, args): + t1 = args[0] + t2 = args[1] + constraints = [] + + # if either is constant, no canonicalization needed + if t1.is_constant() or t2.is_constant(): + return expr.copy([t1, t2]), [] + + if not isinstance(t1, Variable): + t1 = Variable(t1.shape) + constraints += [t1 == args[0]] + t1.value = args[0].value + + if not isinstance(t2, Variable): + t2 = Variable(t2.shape) + constraints += [t2 == args[1]] + t2.value = args[1].value + + return expr.copy([t1, t2]), constraints diff --git a/cvxpy/reductions/dnlp2smooth/canonicalizers/pnorm_canon.py b/cvxpy/reductions/dnlp2smooth/canonicalizers/pnorm_canon.py new file mode 100644 index 0000000000..7283b3367d --- /dev/null +++ b/cvxpy/reductions/dnlp2smooth/canonicalizers/pnorm_canon.py @@ -0,0 +1,38 @@ +""" +Copyright 2025 CVXPY developers + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +from cvxpy.atoms.quad_over_lin import quad_over_lin +from cvxpy.expressions.variable import Variable +from cvxpy.reductions.dnlp2smooth.canonicalizers.quad_over_lin_canon import quad_over_lin_canon + + +def pnorm_canon(expr, args): + x = args[0] + p = expr.p + shape = expr.shape + t = Variable(shape, nonneg=True) + + # expression will always have a value here in DNLP + t.value = expr.value + + # we canonicalize 2-norm as follows: + # ||x||_2 <= t <=> quad_over_lin(x, t) <= t + if p == 2: + expr = quad_over_lin(x, t) + new_expr, constr = quad_over_lin_canon(expr, expr.args) + return t, constr + [new_expr <= t] + else: + raise ValueError("Only p=2 is supported as Pnorm.") diff --git a/cvxpy/reductions/dnlp2smooth/canonicalizers/power_canon.py b/cvxpy/reductions/dnlp2smooth/canonicalizers/power_canon.py new file mode 100644 index 0000000000..6d2e70c63f --- /dev/null +++ b/cvxpy/reductions/dnlp2smooth/canonicalizers/power_canon.py @@ -0,0 +1,55 @@ +""" +Copyright 2025 CVXPY developers + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + + +import numpy as np + +from cvxpy.expressions.constants import Constant +from cvxpy.expressions.variable import Variable + +MIN_INIT = 1e-4 + +def power_canon(expr, args): + x = args[0] + p = expr.p_used + shape = expr.shape + ones = Constant(np.ones(shape)) + if p == 0: + return ones, [] + elif p == 1: + return x, [] + elif isinstance(p, int) and p > 1: + if isinstance(x, Variable): + return expr.copy(args), [] + + t = Variable(shape) + if x.value is not None: + t.value = x.value + else: + t.value = expr.point_in_domain() + + return expr.copy([t]), [t == x] + elif p > 0: + t = Variable(shape, nonneg=True) + + if x.value is not None: + t.value = np.maximum(x.value, MIN_INIT) + else: + t.value = expr.point_in_domain() + + return expr.copy([t]), [t == x] + else: + raise NotImplementedError(f'The power {p} is not yet supported.') diff --git a/cvxpy/reductions/dnlp2smooth/canonicalizers/prod_canon.py b/cvxpy/reductions/dnlp2smooth/canonicalizers/prod_canon.py new file mode 100644 index 0000000000..fd7ec1b47c --- /dev/null +++ b/cvxpy/reductions/dnlp2smooth/canonicalizers/prod_canon.py @@ -0,0 +1,39 @@ +""" +Copyright 2025 CVXPY developers + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import numpy as np + +from cvxpy.expressions.variable import Variable + + +def prod_canon(expr, args): + """ + Canonicalization for the product function. + + Since prod is a smooth function with implemented gradients, + we simply ensure the argument is a Variable. + """ + if isinstance(args[0], Variable): + return expr.copy([args[0]]), [] + + t = Variable(args[0].shape) + + if args[0].value is not None: + t.value = args[0].value + else: + t.value = np.ones(args[0].shape) + + return expr.copy([t]), [t == args[0]] diff --git a/cvxpy/reductions/dnlp2smooth/canonicalizers/quad_over_lin_canon.py b/cvxpy/reductions/dnlp2smooth/canonicalizers/quad_over_lin_canon.py new file mode 100644 index 0000000000..2bd57dede4 --- /dev/null +++ b/cvxpy/reductions/dnlp2smooth/canonicalizers/quad_over_lin_canon.py @@ -0,0 +1,54 @@ +""" +Copyright 2025 CVXPY developers + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import numpy as np + +from cvxpy.atoms.affine.sum import Sum +from cvxpy.atoms.elementwise.power import power +from cvxpy.expressions.variable import Variable +from cvxpy.reductions.dnlp2smooth.canonicalizers.power_canon import power_canon + +MIN_INIT = 1e-4 + +def quad_over_lin_canon(expr, args): + """ + Canonicalize a quadratic over linear expression. + If the denominator is constant, we can use the power canonicalizer. + Otherwise, we introduce new variables for the numerator and denominator. + """ + if args[1].is_constant(): + expr = power(args[0], 2) + var, constr = power_canon(expr, expr.args) + summation = Sum(var) + return 1/args[1].value * summation, constr + else: + t1 = args[0] + t2 = args[1] + constraints = [] + if not isinstance(t1, Variable): + t1 = Variable(t1.shape) + constraints += [t1 == args[0]] + t1.value = args[0].value + # always introduce a new variable for the denominator + # so that we can initialize it to 1 (point in domain) + t2 = Variable(t2.shape, nonneg=True) + constraints += [t2 == args[1]] + + if args[1].value is not None and args[1].value > MIN_INIT: + t2.value = args[1].value + else: + t2.value = np.ones(t2.shape) + + return expr.copy([t1, t2]), constraints diff --git a/cvxpy/reductions/dnlp2smooth/canonicalizers/rel_entr_canon.py b/cvxpy/reductions/dnlp2smooth/canonicalizers/rel_entr_canon.py new file mode 100644 index 0000000000..f660ccb084 --- /dev/null +++ b/cvxpy/reductions/dnlp2smooth/canonicalizers/rel_entr_canon.py @@ -0,0 +1,61 @@ +""" +Copyright 2025 CVXPY developers + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import numpy as np + +from cvxpy.atoms.affine.binary_operators import multiply +from cvxpy.atoms.elementwise.entr import entr +from cvxpy.atoms.elementwise.log import log +from cvxpy.expressions.variable import Variable +from cvxpy.reductions.dnlp2smooth.canonicalizers.entr_canon import entr_canon +from cvxpy.reductions.dnlp2smooth.canonicalizers.log_canon import log_canon +from cvxpy.reductions.dnlp2smooth.canonicalizers.multiply_canon import multiply_canon + +MIN_INIT = 1e-4 + +def rel_entr_canon(expr, args): + + # if the first argument is constant we canonicalize using log + if args[0].is_constant(): + _log = log(args[1]) + log_expr, constr_log = log_canon(_log, _log.args) + x = args[0].value + return x * np.log(x) - multiply(x, log_expr), constr_log + + # if the second argument is constant we canonicalize using entropy + if args[1].is_constant(): + _entr = entr(args[0]) + entr_expr, constr_entr = entr_canon(_entr, _entr.args) + _mult = multiply(args[0], np.log(args[1].value)) + mult_expr, constr_mult = multiply_canon(_mult, _mult.args) + return -entr_expr - mult_expr, constr_entr + constr_mult + + # here we know that neither argument is constant + t1 = Variable(args[0].shape, bounds=[0, None]) + t2 = Variable(args[1].shape, bounds=[0, None]) + constraints = [t1 == args[0], t2 == args[1]] + + if args[0].value is not None: + t1.value = np.maximum(args[0].value, MIN_INIT) + else: + t1.value = expr.point_in_domain(argument=0) + + if args[1].value is not None: + t2.value = np.maximum(args[1].value, MIN_INIT) + else: + t2.value = expr.point_in_domain(argument=1) + + return expr.copy([t1, t2]), constraints diff --git a/cvxpy/reductions/dnlp2smooth/canonicalizers/trig_canon.py b/cvxpy/reductions/dnlp2smooth/canonicalizers/trig_canon.py new file mode 100644 index 0000000000..e049b86ebc --- /dev/null +++ b/cvxpy/reductions/dnlp2smooth/canonicalizers/trig_canon.py @@ -0,0 +1,41 @@ +""" +Copyright 2025 CVXPY developers + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +from cvxpy.expressions.variable import Variable + + +def sin_canon(expr, args): + if isinstance(args[0], Variable): + return expr, [] + else: + t = Variable(args[0].shape) + if args[0].value is not None: + t.value = args[0].value + return expr.copy([t]), [t == args[0]] + +def cos_canon(expr, args): + if isinstance(args[0], Variable): + return expr, [] + else: + t = Variable(args[0].shape) + if args[0].value is not None: + t.value = args[0].value + return expr.copy([t]), [t == args[0]] + +def tan_canon(expr, args): + t = Variable(args[0].shape, bounds=[-3.14159/2, 3.14159/2]) + if args[0].value is not None: + t.value = args[0].value + return expr.copy([t]), [t == args[0]] diff --git a/cvxpy/reductions/dnlp2smooth/dnlp2smooth.py b/cvxpy/reductions/dnlp2smooth/dnlp2smooth.py new file mode 100644 index 0000000000..3deb851e7e --- /dev/null +++ b/cvxpy/reductions/dnlp2smooth/dnlp2smooth.py @@ -0,0 +1,111 @@ +""" +Copyright 2025 CVXPY developers + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +from typing import Tuple + +from cvxpy import problems +from cvxpy.expressions.expression import Expression +from cvxpy.problems.objective import Minimize +from cvxpy.reductions.canonicalization import Canonicalization +from cvxpy.reductions.dnlp2smooth.canonicalizers import SMOOTH_CANON_METHODS as smooth_canon_methods +from cvxpy.reductions.inverse_data import InverseData + + +class Dnlp2Smooth(Canonicalization): + """ + Reduce a disciplined nonlinear program to an equivalent smooth program + + This reduction takes as input (minimization) expressions and converts + them into smooth expressions. + """ + def __init__(self, problem=None) -> None: + super(Canonicalization, self).__init__(problem=problem) + self.smooth_canon_methods = smooth_canon_methods + + def accepts(self, problem): + """A problem is always accepted""" + return True + + def apply(self, problem): + """Converts an expr to a smooth program""" + inverse_data = InverseData(problem) + + inverse_data.minimize = type(problem.objective) == Minimize + + # smoothen objective function + canon_objective, canon_constraints = self.canonicalize_tree( + problem.objective, True) + + # smoothen constraints + for constraint in problem.constraints: + # canon_constr is the constraint re-expressed in terms of + # its canonicalized arguments, and aux_constr are the constraints + # generated while canonicalizing the arguments of the original + # constraint + canon_constr, aux_constr = self.canonicalize_tree( + constraint, False) + canon_constraints += aux_constr + [canon_constr] + inverse_data.cons_id_map.update({constraint.id: canon_constr.id}) + + new_problem = problems.problem.Problem(canon_objective, + canon_constraints) + return new_problem, inverse_data + + def canonicalize_tree(self, expr, affine_above: bool) -> Tuple[Expression, list]: + """Recursively canonicalize an Expression. + + Parameters + ---------- + expr : The expression tree to canonicalize. + affine_above : The path up to the root node is all affine atoms. + + Returns + ------- + A tuple of the canonicalized expression and generated constraints. + """ + affine_atom = type(expr) not in self.smooth_canon_methods + canon_args = [] + constrs = [] + for arg in expr.args: + canon_arg, c = self.canonicalize_tree(arg, affine_atom and affine_above) + canon_args += [canon_arg] + constrs += c + canon_expr, c = self.canonicalize_expr(expr, canon_args, affine_above) + constrs += c + return canon_expr, constrs + + def canonicalize_expr(self, expr, args, affine_above: bool) -> Tuple[Expression, list]: + """Canonicalize an expression, w.r.t. canonicalized arguments. + + Parameters + ---------- + expr : The expression tree to canonicalize. + args : The canonicalized arguments of expr. + affine_above : The path up to the root node is all affine atoms. + + Returns + ------- + A tuple of the canonicalized expression and generated constraints. + """ + # Constant trees are collapsed, but parameter trees are preserved. + if isinstance(expr, Expression) and ( + expr.is_constant() and not expr.parameters()): + return expr, [] + + if type(expr) in self.smooth_canon_methods: + return self.smooth_canon_methods[type(expr)](expr, args) + + return expr.copy(args), [] diff --git a/cvxpy/reductions/eliminate_pwl/canonicalizers/abs_canon.py b/cvxpy/reductions/eliminate_pwl/canonicalizers/abs_canon.py index 19091f1f39..86f2797b06 100644 --- a/cvxpy/reductions/eliminate_pwl/canonicalizers/abs_canon.py +++ b/cvxpy/reductions/eliminate_pwl/canonicalizers/abs_canon.py @@ -14,6 +14,7 @@ limitations under the License. """ + from cvxpy.expressions.variable import Variable from cvxpy.utilities.bounds import get_expr_bounds_if_supported from cvxpy.utilities.solver_context import SolverInfo @@ -24,4 +25,10 @@ def abs_canon(expr, args, solver_context: SolverInfo | None = None): bounds = get_expr_bounds_if_supported(expr, solver_context) t = Variable(expr.shape, bounds=bounds) constraints = [t >= x, t >= -x] + + # for DNLP we must initialize the new variable (DNLP guarantees that + # x.value will be set when this function is called) + if expr.value is not None: + t.value = expr.value + return t, constraints diff --git a/cvxpy/reductions/eliminate_pwl/canonicalizers/max_canon.py b/cvxpy/reductions/eliminate_pwl/canonicalizers/max_canon.py index 43899b8456..a203d097a8 100644 --- a/cvxpy/reductions/eliminate_pwl/canonicalizers/max_canon.py +++ b/cvxpy/reductions/eliminate_pwl/canonicalizers/max_canon.py @@ -38,4 +38,10 @@ def max_canon(expr, args, solver_context: SolverInfo | None = None): promoted_t = reshape(t, (x.shape[0], 1), order='F') @ Constant(np.ones((1, x.shape[1]))) constraints = [x <= promoted_t] + + # for DNLP we must initialize the new variable (DNLP guarantees that + # x.value will be set when this function is called) + if expr.value is not None: + t.value = expr.value + return t, constraints diff --git a/cvxpy/reductions/eliminate_pwl/canonicalizers/maximum_canon.py b/cvxpy/reductions/eliminate_pwl/canonicalizers/maximum_canon.py index 73079afd28..c9ed3f7f14 100644 --- a/cvxpy/reductions/eliminate_pwl/canonicalizers/maximum_canon.py +++ b/cvxpy/reductions/eliminate_pwl/canonicalizers/maximum_canon.py @@ -24,11 +24,17 @@ def maximum_canon(expr, args, solver_context: SolverInfo | None = None): shape = expr.shape bounds = get_expr_bounds_if_supported(expr, solver_context) t = Variable(shape, bounds=bounds) - + + # for DNLP we must initialize the new variable (DNLP guarantees that + # x.value will be set when this function is called) + if expr.value is not None: + t.value = expr.value + if expr.is_nonneg(): t = nonneg_wrap(t) if expr.is_nonpos(): t = nonpos_wrap(t) constraints = [t >= elem for elem in args] + return t, constraints diff --git a/cvxpy/reductions/eliminate_pwl/canonicalizers/norm_inf_canon.py b/cvxpy/reductions/eliminate_pwl/canonicalizers/norm_inf_canon.py index 1c56359f21..3b8fee9e1f 100644 --- a/cvxpy/reductions/eliminate_pwl/canonicalizers/norm_inf_canon.py +++ b/cvxpy/reductions/eliminate_pwl/canonicalizers/norm_inf_canon.py @@ -37,4 +37,9 @@ def norm_inf_canon(expr, args, solver_context: SolverInfo | None = None): else: # shape = (m, 1) promoted_t = reshape(t, (x.shape[0], 1), order='F') @ Constant(np.ones((1, x.shape[1]))) + # for DNLP we must initialize the new variable (DNLP guarantees that + # x.value will be set when this function is called) + if expr.value is not None: + t.value = expr.value + return t, [x <= promoted_t, x + promoted_t >= 0] diff --git a/cvxpy/reductions/eliminate_pwl/canonicalizers/sum_largest_canon.py b/cvxpy/reductions/eliminate_pwl/canonicalizers/sum_largest_canon.py index 617ba588cb..8f82dba037 100644 --- a/cvxpy/reductions/eliminate_pwl/canonicalizers/sum_largest_canon.py +++ b/cvxpy/reductions/eliminate_pwl/canonicalizers/sum_largest_canon.py @@ -14,6 +14,8 @@ limitations under the License. """ +import numpy as np + from cvxpy.atoms.affine.sum import sum from cvxpy.expressions.variable import Variable from cvxpy.utilities.solver_context import SolverInfo @@ -30,4 +32,17 @@ def sum_largest_canon(expr, args, solver_context: SolverInfo | None = None): q = Variable() obj = sum(t) + k*q constraints = [x <= t + q, t >= 0] + + # for DNLP we must initialize the new variable (DNLP guarantees that + # x.value will be set when this function is called). The initialization + # below is motivated by the optimal solution of min sum(t) + kq subject + # to x <= t + q, 0 <= t + if x.value is not None: + sorted_indices = np.argsort(x.value) + idx_of_smallest = sorted_indices[:k] + idx_of_largest = sorted_indices[-k:] + q.value = np.max(x.value[idx_of_smallest]) + t.value = np.zeros_like(x.value) + t.value[idx_of_largest] = x.value[idx_of_largest] - q.value + return obj, constraints diff --git a/cvxpy/reductions/inverse_data.py b/cvxpy/reductions/inverse_data.py index d4fc351bc7..b5879f23f7 100644 --- a/cvxpy/reductions/inverse_data.py +++ b/cvxpy/reductions/inverse_data.py @@ -54,3 +54,16 @@ def get_var_offsets(variables): id_map[x.id] = (vert_offset, x.size) vert_offset += x.size return (id_map, var_offsets, vert_offset, var_shapes) + + @staticmethod + def get_param_offsets(parameters): + param_shapes = {} + param_offsets = {} + id_map = {} + offset = 0 + for p in parameters: + param_shapes[p.id] = p.shape + param_offsets[p.id] = offset + id_map[p.id] = (offset, p.size) + offset += p.size + return (id_map, param_offsets, offset, param_shapes) diff --git a/cvxpy/reductions/matrix_stuffing.py b/cvxpy/reductions/matrix_stuffing.py index cc43aeb9af..8f82e42c91 100644 --- a/cvxpy/reductions/matrix_stuffing.py +++ b/cvxpy/reductions/matrix_stuffing.py @@ -234,6 +234,7 @@ def apply(self, problem) -> None: InverseData Data for solution retrieval """ + def invert(self, solution, inverse_data): raise NotImplementedError() diff --git a/cvxpy/reductions/solvers/conic_solvers/conic_solver.py b/cvxpy/reductions/solvers/conic_solvers/conic_solver.py index b015c5a950..137368ce5b 100644 --- a/cvxpy/reductions/solvers/conic_solvers/conic_solver.py +++ b/cvxpy/reductions/solvers/conic_solvers/conic_solver.py @@ -20,6 +20,7 @@ import cvxpy.settings as s from cvxpy.constraints import PSD, SOC, ExpCone, NonNeg, PowCone3D, PowConeND, Zero +from cvxpy.problems.param_prob import ParamProb from cvxpy.reductions.cvx_attr2constr import convex_attributes from cvxpy.reductions.dcp2cone.cone_matrix_stuffing import ParamConeProg from cvxpy.reductions.solution import Solution, failure_solution @@ -117,7 +118,7 @@ class ConicSolver(Solver): EXP_CONE_ORDER = None def accepts(self, problem): - return (isinstance(problem, ParamConeProg) + return (isinstance(problem, ParamProb) and (self.MIP_CAPABLE or not problem.is_mixed_integer()) and not convex_attributes([problem.x]) and (len(problem.constraints) > 0 or not self.REQUIRES_CONSTR) @@ -191,62 +192,111 @@ def format_constraints(cls, problem, exp_cone_order): Returns: ParamConeProg with structured A. """ - # Create a matrix to reshape constraints, then replicate for each - # variable entry. - restruct_mat = [] # Form a block diagonal matrix. - for constr in problem.constraints: - total_height = sum([arg.size for arg in constr.args]) + # Lazy import to avoid circular dependency (conic_solver → diff_engine + # → converters → cvxpy). + from cvxpy.reductions.dcp2cone.diff_engine_param_cone_prog import ( + DiffEngineParamConeProg, + ) + if isinstance(problem, DiffEngineParamConeProg): + restruct_mat = cls._build_restruct_operator( + problem.constraints, exp_cone_order + ) + problem._restruct_mat = restruct_mat + problem.formatted = True + return problem + + # Build the restructuring operator (shared with DiffEngine path). + restruct_mat = cls._build_restruct_operator( + problem.constraints, exp_cone_order + ) + + # Apply the operator to the problem data tensor. + if restruct_mat is not None: + # this is equivalent to but _much_ faster than: + # restruct_mat_rep = sp.block_diag([restruct_mat]*(problem.x.size + 1)) + # restruct_A = restruct_mat_rep * problem.A + unspecified, _ = np.divmod(problem.A.shape[0] * problem.A.shape[1], + restruct_mat.shape[1], dtype=np.int64) + reshaped_A = problem.A.reshape(restruct_mat.shape[1], + unspecified, order='F').tocsr() + restructured_A = restruct_mat(reshaped_A).tocoo() + # Because of a bug in scipy versions < 1.20, `reshape` + # can overflow if indices are int32s. + restructured_A.row = restructured_A.row.astype(np.int64) + restructured_A.col = restructured_A.col.astype(np.int64) + restructured_A = restructured_A.reshape( + np.int64(restruct_mat.shape[0]) * (np.int64(problem.x.size) + 1), + problem.A.shape[1], order='F') + else: + restructured_A = problem.A + new_param_cone_prog = ParamConeProg( + problem.q, + problem.x, + restructured_A, + problem.variables, + problem.var_id_to_col, + problem.constraints, + problem.parameters, + problem.param_id_to_col, + P=problem.P, + formatted=True, + lower_bounds=problem.lower_bounds, + upper_bounds=problem.upper_bounds, + lb_tensor=problem.lb_tensor, + ub_tensor=problem.ub_tensor, + ) + return new_param_cone_prog + + @classmethod + def _build_restruct_operator(cls, constraints, exp_cone_order): + """Build a block-diagonal restructuring LinearOperator. + + Returns a LinearOperator that reorders/interleaves constraint + argument rows into the layout expected by solvers. Used by both + format_constraints (applied to the tensor) and DiffEngineParamConeProg + (applied to the Jacobian/offset at solve time). + """ + blocks = [] + for constr in constraints: + total_height = sum(arg.size for arg in constr.args) if type(constr) == Zero: - restruct_mat.append(NegativeIdentityOperator(constr.size)) + blocks.append(NegativeIdentityOperator(constr.size)) elif type(constr) == NonNeg: - restruct_mat.append(IdentityOperator(constr.size)) + blocks.append(IdentityOperator(constr.size)) elif type(constr) == SOC: - # Group each t row with appropriate X rows. assert constr.axis == 0, 'SOC must be lowered to axis == 0' - - # Interleave the rows of coeffs[0] and coeffs[1]: - # coeffs[0][0, :] - # coeffs[1][0:gap-1, :] - # coeffs[0][1, :] - # coeffs[1][gap-1:2*(gap-1), :] - # Handle scalar X (shape is empty tuple) x_dim = constr.args[1].shape[0] if constr.args[1].shape else 1 - t_spacer = ConicSolver.get_spacing_matrix( + t_spacer = cls.get_spacing_matrix( shape=(total_height, constr.args[0].size), - spacing=x_dim, - streak=1, - num_blocks=constr.args[0].size, - offset=0, + spacing=x_dim, streak=1, + num_blocks=constr.args[0].size, offset=0, ) - X_spacer = ConicSolver.get_spacing_matrix( + X_spacer = cls.get_spacing_matrix( shape=(total_height, constr.args[1].size), - spacing=1, - streak=x_dim, - num_blocks=constr.args[0].size, - offset=1, + spacing=1, streak=x_dim, + num_blocks=constr.args[0].size, offset=1, ) - restruct_mat.append(sp.hstack([t_spacer, X_spacer])) + blocks.append(sp.hstack([t_spacer, X_spacer])) elif type(constr) == ExpCone: arg_mats = [] for i, arg in enumerate(constr.args): - space_mat = ConicSolver.get_spacing_matrix( + space_mat = cls.get_spacing_matrix( shape=(total_height, arg.size), spacing=len(exp_cone_order) - 1, - streak=1, - num_blocks=arg.size, + streak=1, num_blocks=arg.size, offset=exp_cone_order[i], ) arg_mats.append(space_mat) - restruct_mat.append(sp.hstack(arg_mats)) + blocks.append(sp.hstack(arg_mats)) elif type(constr) == PowCone3D: arg_mats = [] for i, arg in enumerate(constr.args): - space_mat = ConicSolver.get_spacing_matrix( + space_mat = cls.get_spacing_matrix( shape=(total_height, arg.size), spacing=2, streak=1, num_blocks=arg.size, offset=i, ) arg_mats.append(space_mat) - restruct_mat.append(sp.hstack(arg_mats)) + blocks.append(sp.hstack(arg_mats)) elif type(constr) == PowConeND: arg_mats = [] if constr.args[0].ndim == 1: @@ -255,66 +305,27 @@ def format_constraints(cls, problem, exp_cone_order): else: m, n = constr.args[0].shape for j in range(n): - space_mat = ConicSolver.get_spacing_matrix( + space_mat = cls.get_spacing_matrix( shape=(total_height, m), spacing=0, - streak=1, num_blocks=m, offset=(m+1)*j, + streak=1, num_blocks=m, offset=(m + 1) * j, ) arg_mats.append(space_mat) - - # Hypo columns arg = constr.args[1] assert arg.size == n - space_mat = ConicSolver.get_spacing_matrix( + space_mat = cls.get_spacing_matrix( shape=(total_height, n), spacing=m, streak=1, num_blocks=n, offset=m, ) arg_mats.append(space_mat) - restruct_mat.append(sp.hstack(arg_mats)) - + blocks.append(sp.hstack(arg_mats)) elif type(constr) == PSD: - restruct_mat.append(cls.psd_format_mat(constr)) + blocks.append(cls.psd_format_mat(constr)) else: raise ValueError("Unsupported constraint type.") - # Form new ParamConeProg - if restruct_mat: - # TODO(akshayka): profile to see whether using linear operators - # or bmat is faster - restruct_mat = as_block_diag_linear_operator(restruct_mat) - # this is equivalent to but _much_ faster than: - # restruct_mat_rep = sp.block_diag([restruct_mat]*(problem.x.size + 1)) - # restruct_A = restruct_mat_rep * problem.A - unspecified, _ = np.divmod(problem.A.shape[0] * problem.A.shape[1], - restruct_mat.shape[1], dtype=np.int64) - reshaped_A = problem.A.reshape(restruct_mat.shape[1], - unspecified, order='F').tocsr() - restructured_A = restruct_mat(reshaped_A).tocoo() - # Because of a bug in scipy versions < 1.20, `reshape` - # can overflow if indices are int32s. - restructured_A.row = restructured_A.row.astype(np.int64) - restructured_A.col = restructured_A.col.astype(np.int64) - restructured_A = restructured_A.reshape( - np.int64(restruct_mat.shape[0]) * (np.int64(problem.x.size) + 1), - problem.A.shape[1], order='F') - else: - restructured_A = problem.A - new_param_cone_prog = ParamConeProg( - problem.q, - problem.x, - restructured_A, - problem.variables, - problem.var_id_to_col, - problem.constraints, - problem.parameters, - problem.param_id_to_col, - P=problem.P, - formatted=True, - lower_bounds=problem.lower_bounds, - upper_bounds=problem.upper_bounds, - lb_tensor=problem.lb_tensor, - ub_tensor=problem.ub_tensor, - ) - return new_param_cone_prog + if blocks: + return as_block_diag_linear_operator(blocks) + return None def invert(self, solution, inverse_data): """Returns the solution to the original problem given the inverse_data. diff --git a/cvxpy/reductions/solvers/defines.py b/cvxpy/reductions/solvers/defines.py index cb9780372b..edfd1c7aa2 100644 --- a/cvxpy/reductions/solvers/defines.py +++ b/cvxpy/reductions/solvers/defines.py @@ -46,6 +46,11 @@ from cvxpy.reductions.solvers.conic_solvers.sdpa_conif import SDPA as SDPA_con from cvxpy.reductions.solvers.conic_solvers.xpress_conif import XPRESS as XPRESS_con +# NLP interfaces +from cvxpy.reductions.solvers.nlp_solvers.copt_nlpif import COPT as COPT_nlp +from cvxpy.reductions.solvers.nlp_solvers.ipopt_nlpif import IPOPT as IPOPT_nlp +from cvxpy.reductions.solvers.nlp_solvers.uno_nlpif import UNO as UNO_nlp + # QP interfaces from cvxpy.reductions.solvers.qp_solvers.copt_qpif import COPT as COPT_qp from cvxpy.reductions.solvers.qp_solvers.cplex_qpif import CPLEX as CPLEX_qp @@ -77,9 +82,14 @@ MPAX_qp(), KNITRO_qp(), ]} +SOLVER_MAP_NLP = {inst.name(): inst for inst in [ + IPOPT_nlp(), UNO_nlp(), COPT_nlp(), +]} + # Preference-ordered solver name lists, derived from the maps above. CONIC_SOLVERS = list(SOLVER_MAP_CONIC) QP_SOLVERS = list(SOLVER_MAP_QP) +NLP_SOLVERS = list(SOLVER_MAP_NLP) # Mixed-integer solver lists, derived from solver class attributes. MI_SOLVERS = [ @@ -100,7 +110,7 @@ def installed_solvers(): """List the installed solvers.""" return list(dict.fromkeys( - name for name, slv in {**SOLVER_MAP_CONIC, **SOLVER_MAP_QP}.items() + name for name, slv in {**SOLVER_MAP_CONIC, **SOLVER_MAP_QP, **SOLVER_MAP_NLP}.items() if slv.is_installed() )) diff --git a/cvxpy/reductions/solvers/nlp_solvers/__init__.py b/cvxpy/reductions/solvers/nlp_solvers/__init__.py new file mode 100644 index 0000000000..964545a6f8 --- /dev/null +++ b/cvxpy/reductions/solvers/nlp_solvers/__init__.py @@ -0,0 +1,15 @@ +""" +Copyright 2025 The CVXPY Developers + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" diff --git a/cvxpy/reductions/solvers/nlp_solvers/copt_nlpif.py b/cvxpy/reductions/solvers/nlp_solvers/copt_nlpif.py new file mode 100644 index 0000000000..2668d06d80 --- /dev/null +++ b/cvxpy/reductions/solvers/nlp_solvers/copt_nlpif.py @@ -0,0 +1,288 @@ +""" +Copyright 2025, the CVXPY developers + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import numpy as np + +import cvxpy.settings as s +from cvxpy.reductions.solution import Solution, failure_solution +from cvxpy.reductions.solvers.nlp_solvers.nlp_solver import NLPsolver +from cvxpy.utilities.citations import CITATION_DICT + + +class COPT(NLPsolver): + """ + NLP interface for the COPT solver. + """ + # Map between COPT status and CVXPY status + STATUS_MAP = { + 1: s.OPTIMAL, # optimal + 2: s.INFEASIBLE, # infeasible + 3: s.UNBOUNDED, # unbounded + 4: s.INF_OR_UNB, # infeasible or unbounded + 5: s.SOLVER_ERROR, # numerical + 6: s.USER_LIMIT, # node limit + 7: s.OPTIMAL_INACCURATE, # imprecise + 8: s.USER_LIMIT, # time out + 9: s.SOLVER_ERROR, # unfinished + 10: s.USER_LIMIT # interrupted + } + + def name(self): + """ + The name of solver. + """ + return 'COPT' + + def import_solver(self): + """ + Imports the solver. + """ + import coptpy # noqa F401 + + def invert(self, solution, inverse_data): + """ + Returns the solution to the original problem given the inverse_data. + """ + attr = { + s.NUM_ITERS: solution.get('num_iters'), + s.SOLVE_TIME: solution.get('solve_time_real'), + } + + status = self.STATUS_MAP[solution['status']] + if status in s.SOLUTION_PRESENT: + primal_val = solution['obj_val'] + opt_val = primal_val + inverse_data.offset + primal_vars = {} + x_opt = solution['x'] + for id, offset in inverse_data.var_offsets.items(): + shape = inverse_data.var_shapes[id] + size = np.prod(shape, dtype=int) + primal_vars[id] = np.reshape(x_opt[offset:offset+size], shape, order='F') + return Solution(status, opt_val, primal_vars, {}, attr) + else: + return failure_solution(status, attr) + + def solve_via_data(self, data, warm_start: bool, verbose: bool, solver_opts, solver_cache=None): + """ + Returns the result of the call to the solver. + + Parameters + ---------- + data : dict + Data used by the solver. + This consists of: + - "oracles": An Oracles object that computes the objective and constraints + - "x0": Initial guess for the primal variables + - "lb": Lower bounds on the primal variables + - "ub": Upper bounds on the primal variables + - "cl": Lower bounds on the constraints + - "cu": Upper bounds on the constraints + - "objective": Function to compute the objective value + - "gradient": Function to compute the objective gradient + - "constraints": Function to compute the constraint values + - "jacobian": Function to compute the constraint Jacobian + - "jacobianstructure": Function to compute the structure of the Jacobian + - "hessian": Function to compute the Hessian of the Lagrangian + - "hessianstructure": Function to compute the structure of the Hessian + warm_start : bool + Not used. + verbose : bool + Should the solver print output? + solver_opts : dict + Additional arguments for the solver. + solver_cache: None + None + + Returns + ------- + tuple + (status, optimal value, primal, equality dual, inequality dual) + """ + import coptpy as copt + + from cvxpy.reductions.solvers.nlp_solvers.nlp_solver import Oracles + + # Create oracles object (deferred from apply() so we have access to verbose) + bounds = data["_bounds"] + + # COPT always uses exact Hessian (no quasi-Newton option currently) + use_hessian = True + + oracles = Oracles(bounds.new_problem, bounds.x0, len(bounds.cl), + verbose=verbose, use_hessian=use_hessian) + + class COPTNlpCallbackCVXPY(copt.NlpCallbackBase): + def __init__(self, oracles, m): + super().__init__() + self._oracles = oracles + self._m = m + + def EvalObj(self, xdata, outdata): + x = copt.NdArray(xdata) + outval = copt.NdArray(outdata) + + x_np = x.tonumpy() + outval_np = self._oracles.objective(x_np) + + outval[:] = outval_np + return 0 + + def EvalGrad(self, xdata, outdata): + x = copt.NdArray(xdata) + outval = copt.NdArray(outdata) + + x_np = x.tonumpy() + outval_np = self._oracles.gradient(x_np) + + outval[:] = np.asarray(outval_np).flatten() + return 0 + + def EvalCon(self, xdata, outdata): + if self._m > 0: + x = copt.NdArray(xdata) + outval = copt.NdArray(outdata) + + x_np = x.tonumpy() + outval_np = self._oracles.constraints(x_np) + + outval[:] = np.asarray(outval_np).flatten() + return 0 + + def EvalJac(self, xdata, outdata): + if self._m > 0: + x = copt.NdArray(xdata) + outval = copt.NdArray(outdata) + + x_np = x.tonumpy() + outval_np = self._oracles.jacobian(x_np) + + outval[:] = np.asarray(outval_np).flatten() + return 0 + + def EvalHess(self, xdata, sigma, lambdata, outdata): + x = copt.NdArray(xdata) + lagrange = copt.NdArray(lambdata) + outval = copt.NdArray(outdata) + + x_np = x.tonumpy() + lagrange_np = lagrange.tonumpy() + outval_np = self._oracles.hessian(x_np, lagrange_np, sigma) + + outval[:] = np.asarray(outval_np).flatten() + return 0 + + # Create COPT environment and model + envconfig = copt.EnvrConfig() + if not verbose: + envconfig.set('nobanner', '1') + + env = copt.Envr(envconfig) + model = env.createModel() + + # Pass through verbosity + model.setParam(copt.COPT.Param.Logging, verbose) + + # Get the NLP problem data + x0 = data['x0'] + lb, ub = data['lb'].copy(), data['ub'].copy() + cl, cu = data['cl'].copy(), data['cu'].copy() + + lb[lb == -np.inf] = -copt.COPT.INFINITY + ub[ub == +np.inf] = +copt.COPT.INFINITY + cl[cl == -np.inf] = -copt.COPT.INFINITY + cu[cu == +np.inf] = +copt.COPT.INFINITY + + n = len(lb) + m = len(cl) + + cbtype = copt.COPT.EVALTYPE_OBJVAL | copt.COPT.EVALTYPE_CONSTRVAL | \ + copt.COPT.EVALTYPE_GRADIENT | copt.COPT.EVALTYPE_JACOBIAN | \ + copt.COPT.EVALTYPE_HESSIAN + cbfunc = COPTNlpCallbackCVXPY(oracles, m) + + if m > 0: + jac_rows, jac_cols = oracles.jacobianstructure() + nnz_jac = len(jac_rows) + else: + jac_rows = None + jac_cols = None + nnz_jac = 0 + + if n > 0: + hess_rows, hess_cols = oracles.hessianstructure() + nnz_hess = len(hess_rows) + else: + hess_rows = None + hess_cols = None + nnz_hess = 0 + + # Load NLP problem data + model.loadNlData(n, # Number of variables + m, # Number of constraints + copt.COPT.MINIMIZE, # Objective sense + copt.COPT.DENSETYPE_ROWMAJOR, None, # Dense objective gradient + nnz_jac, jac_rows, jac_cols, # Sparse jacobian + nnz_hess, hess_rows, hess_cols, # Sparse hessian + lb, ub, # Variable bounds + cl, cu, # Constraint bounds + x0, # Starting point + cbtype, cbfunc # Callback function + ) + + # Set parameters + for key, value in solver_opts.items(): + model.setParam(key, value) + + # Solve problem + model.solve() + + # Get solution + nlp_status = model.status + nlp_hassol = model.haslpsol + + if nlp_hassol: + objval = model.objval + x_sol = model.getValues() + lambda_sol = model.getDuals() + else: + objval = +np.inf + x_sol = [0.0] * n + lambda_sol = [0.0] * m + + num_iters = model.barrieriter + solve_time_real = model.solvingtime + + # Return results in dictionary format expected by invert() + solution = { + 'status': nlp_status, + 'obj_val': objval, + 'x': np.array(x_sol), + 'lambda': np.array(lambda_sol), + 'num_iters': num_iters, + 'solve_time_real': solve_time_real + } + + return solution + + def cite(self, data): + """Returns bibtex citation for the solver. + + Parameters + ---------- + data : dict + Data generated via an apply call. + """ + return CITATION_DICT["COPT"] diff --git a/cvxpy/reductions/solvers/nlp_solvers/diff_engine/__init__.py b/cvxpy/reductions/solvers/nlp_solvers/diff_engine/__init__.py new file mode 100644 index 0000000000..a0f448250b --- /dev/null +++ b/cvxpy/reductions/solvers/nlp_solvers/diff_engine/__init__.py @@ -0,0 +1,38 @@ +""" +Copyright 2025, the CVXPY developers + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +""" +CVXPY integration layer for the DNLP diff engine. + +This module converts CVXPY expressions to C expression trees +for automatic differentiation. +""" + +from cvxpy.reductions.solvers.nlp_solvers.diff_engine.c_problem import C_problem +from cvxpy.reductions.solvers.nlp_solvers.diff_engine.converters import ( + ATOM_CONVERTERS, + build_variable_dict, + convert_expr, + convert_expressions, +) + +__all__ = [ + "C_problem", + "ATOM_CONVERTERS", + "build_variable_dict", + "convert_expr", + "convert_expressions", +] diff --git a/cvxpy/reductions/solvers/nlp_solvers/diff_engine/c_problem.py b/cvxpy/reductions/solvers/nlp_solvers/diff_engine/c_problem.py new file mode 100644 index 0000000000..affea7f11f --- /dev/null +++ b/cvxpy/reductions/solvers/nlp_solvers/diff_engine/c_problem.py @@ -0,0 +1,134 @@ +"""Wrapper around C problem struct for CVXPY problems. + +Copyright 2025, the CVXPY developers + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import numpy as np +from scipy import sparse + +import cvxpy as cp + +# Import the low-level C bindings +try: + from sparsediffpy import _sparsediffengine as _diffengine +except ImportError as e: + raise ImportError( + "NLP support requires sparsediffpy. Install with: pip install sparsediffpy" + ) from e + +from cvxpy.reductions.solvers.nlp_solvers.diff_engine.converters import ( + convert_expressions, +) + + +class C_problem: + """Wrapper around C problem struct for CVXPY problems.""" + + def __init__(self, cvxpy_problem: cp.Problem, verbose: bool = True): + c_obj, c_constraints, param_capsules, all_params = ( + convert_expressions(cvxpy_problem) + ) + self._capsule = _diffengine.make_problem(c_obj, c_constraints, verbose) + + # Register parameters with the C problem + if param_capsules: + _diffengine.problem_register_params( + self._capsule, param_capsules + ) + + self._all_params = all_params + self._jacobian_allocated = False + self._hessian_allocated = False + + # Push initial parameter values to C + self.update_params() + + def init_jacobian(self): + """Initialize Jacobian structures only. Must be called before jacobian().""" + _diffengine.problem_init_jacobian(self._capsule) + self._jacobian_allocated = True + + def init_hessian(self): + """Initialize Hessian structures only. Must be called before hessian().""" + _diffengine.problem_init_hessian(self._capsule) + self._hessian_allocated = True + + def objective_forward(self, u: np.ndarray) -> float: + """Evaluate objective. Returns obj_value float.""" + return _diffengine.problem_objective_forward(self._capsule, u) + + def constraint_forward(self, u: np.ndarray) -> np.ndarray: + """Evaluate constraints only. Returns constraint_values array.""" + return _diffengine.problem_constraint_forward(self._capsule, u) + + def gradient(self) -> np.ndarray: + """Compute gradient of objective. Call objective_forward first. Returns gradient array.""" + return _diffengine.problem_gradient(self._capsule) + + def jacobian(self) -> sparse.csr_matrix: + """Compute constraint Jacobian. Call constraint_forward first.""" + data, indices, indptr, shape = _diffengine.problem_jacobian(self._capsule) + return sparse.csr_matrix((data, indices, indptr), shape=shape) + + def get_jacobian(self) -> sparse.csr_matrix: + """Get constraint Jacobian. This function does not evaluate the jacobian. """ + data, indices, indptr, shape = _diffengine.get_jacobian(self._capsule) + return sparse.csr_matrix((data, indices, indptr), shape=shape) + + def hessian(self, obj_factor: float, lagrange: np.ndarray) -> sparse.csr_matrix: + """Compute Lagrangian Hessian. + + Computes: obj_factor * H_obj + sum(lagrange_i * H_constraint_i) + + Call objective_forward and constraint_forward before this. + + Args: + obj_factor: Weight for objective Hessian + lagrange: Array of Lagrange multipliers (length = total_constraint_size) + + Returns: + scipy CSR matrix of shape (n_vars, n_vars) + """ + data, indices, indptr, shape = _diffengine.problem_hessian( + self._capsule, obj_factor, lagrange + ) + return sparse.csr_matrix((data, indices, indptr), shape=shape) + + def get_hessian(self) -> sparse.csr_matrix: + """Get Lagrangian Hessian. This function does not evaluate the hessian.""" + data, indices, indptr, shape = _diffengine.get_hessian(self._capsule) + return sparse.csr_matrix((data, indices, indptr), shape=shape) + + def update_params(self): + """Read current .value from each Parameter and push to C. + + Call this after changing parameter values to update the C expression tree + without rebuilding it. After calling this, objective_forward/gradient/etc. + will use the new parameter values. + """ + if not self._all_params: + return + theta = np.empty(sum(p.size for p in self._all_params)) + offset = 0 + for param in self._all_params: + if param.sparse_idx is not None: + val = np.asarray( + param.value_sparse.toarray(), dtype=np.float64 + ).flatten(order='C') + else: + val = np.asarray(param.value, dtype=np.float64).flatten(order='C') + theta[offset:offset + param.size] = val + offset += param.size + _diffengine.problem_update_params(self._capsule, theta) diff --git a/cvxpy/reductions/solvers/nlp_solvers/diff_engine/converters.py b/cvxpy/reductions/solvers/nlp_solvers/diff_engine/converters.py new file mode 100644 index 0000000000..cb87d0f11d --- /dev/null +++ b/cvxpy/reductions/solvers/nlp_solvers/diff_engine/converters.py @@ -0,0 +1,554 @@ + +"""Converters from CVXPY expressions to C diff engine expressions. + +This module provides the mapping between CVXPY atom types and their +corresponding C diff engine constructors. + +Copyright 2025, the CVXPY developers + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import numpy as np +from scipy import sparse + +import cvxpy as cp +from cvxpy.reductions.inverse_data import InverseData + +# Import the low-level C bindings +try: + from sparsediffpy import _sparsediffengine as _diffengine +except ImportError as e: + raise ImportError( + "NLP support requires sparsediffpy. Install with: pip install sparsediffpy" + ) from e + + +def _normalize_shape(shape): + """Normalize a CVXPY shape to (d1, d2).""" + x_shape = tuple(shape) + x_shape = (1,) * (2 - len(x_shape)) + x_shape + return x_shape + + + + +def _chain_add(children): + """Chain multiple children with binary adds: a + b + c -> add(add(a, b), c).""" + result = children[0] + for child in children[1:]: + result = _diffengine.make_add(result, child) + return result + + +def _convert_matmul(expr, children): + """Convert matrix multiplication A @ f(x), f(x) @ A, or X @ Y.""" + left_arg, right_arg = expr.args + + if left_arg.is_constant(): + if isinstance(left_arg, cp.Parameter): + # Updatable parameter: dense CSR — all m*n entries present so any + # entry can change between solves via refresh_param_values. + if left_arg.sparse_idx is not None: + A_dense = np.asarray(left_arg.value_sparse.toarray(), + dtype=np.float64) + else: + A_dense = np.asarray(left_arg.value, dtype=np.float64) + if A_dense.ndim == 1: + A_dense = A_dense.reshape(1, -1) + m, n = A_dense.shape + A = sparse.csr_matrix(np.ones((m, n))) + A.data[:] = A_dense.ravel(order='C') # row-major = CSR data order + param_or_none = children[0] + else: + A = left_arg.value + if not isinstance(A, sparse.csr_matrix): + A = sparse.csr_matrix(A) + param_or_none = None + + return _diffengine.make_left_matmul( + param_or_none, + children[1], + A.data.astype(np.float64), + A.indices.astype(np.int32), + A.indptr.astype(np.int32), + A.shape[0], + A.shape[1], + ) + elif right_arg.is_constant(): + A = right_arg.value + + if not isinstance(A, sparse.csr_matrix): + A = sparse.csr_matrix(A) + + return _diffengine.make_right_matmul( + children[0], + A.data.astype(np.float64), + A.indices.astype(np.int32), + A.indptr.astype(np.int32), + A.shape[0], + A.shape[1], + ) + else: + return _diffengine.make_matmul(children[0], children[1]) + +def _convert_hstack(expr, children): + """Convert horizontal stack (hstack) of expressions.""" + return _diffengine.make_hstack(children) + +def _convert_multiply(expr, children): + """Convert multiplication based on argument types.""" + left_arg, right_arg = expr.args + + if left_arg.is_constant(): + if isinstance(left_arg, cp.Parameter): + if left_arg.size == 1: + return _diffengine.make_param_scalar_mult(children[0], children[1]) + return _diffengine.make_param_vector_mult(children[0], children[1]) + + a = left_arg.value + # we only support dense constants for elementwise multiplication + if sparse.issparse(a): + a = a.todense() + a = np.asarray(a, dtype=np.float64) + + # Scalar constant + if a.size == 1: + scalar = float(a.flat[0]) + if scalar == 1.0: + return children[1] + else: + return _diffengine.make_const_scalar_mult(children[1], scalar) + + # non-scalar constant + return _diffengine.make_const_vector_mult(children[1], a.flatten(order='F')) + + elif right_arg.is_constant(): + if isinstance(right_arg, cp.Parameter): + if right_arg.size == 1: + return _diffengine.make_param_scalar_mult(children[1], children[0]) + return _diffengine.make_param_vector_mult(children[1], children[0]) + + a = right_arg.value + # we only support dense constants for elementwise multiplication + if sparse.issparse(a): + a = a.todense() + a = np.asarray(a, dtype=np.float64) + + # Scalar constant + if a.size == 1: + scalar = float(a.flat[0]) + if scalar == 1.0: + return children[0] + else: + return _diffengine.make_const_scalar_mult(children[0], scalar) + + # non-scalar constant + return _diffengine.make_const_vector_mult(children[0], a.flatten(order='F')) + + # Neither is constant, use general multiply + return _diffengine.make_multiply(children[0], children[1]) + + +def _extract_flat_indices_from_index(expr): + """Extract flattened indices from CVXPY index expression.""" + parent_shape = expr.args[0].shape + indices_per_dim = [np.arange(s.start, s.stop, s.step) for s in expr.key] + + if len(indices_per_dim) == 1: + return indices_per_dim[0].astype(np.int32) + elif len(indices_per_dim) == 2: + # Fortran order: idx = row + col * n_rows + return ( + np.add.outer(indices_per_dim[0], indices_per_dim[1] * parent_shape[0]) + .flatten(order="F") + .astype(np.int32) + ) + else: + raise NotImplementedError("index with >2 dimensions not supported") + + +def _extract_flat_indices_from_special_index(expr): + """Extract flattened indices from CVXPY special_index expression.""" + return np.reshape(expr._select_mat, expr._select_mat.size, order="F").astype(np.int32) + + +def _convert_rel_entr(expr, children): + """Convert rel_entr(x, y) = x * log(x/y) elementwise. + + Uses specialized functions based on argument shapes: + - Both scalar or both same size: make_rel_entr (elementwise) + - First arg vector, second scalar: make_rel_entr_vector_scalar + - First arg scalar, second vector: make_rel_entr_scalar_vector + """ + x_arg, y_arg = expr.args + x_size = x_arg.size + y_size = y_arg.size + + # Determine which variant to use based on sizes + if x_size == y_size: + return _diffengine.make_rel_entr(children[0], children[1]) + elif x_size > 1 and y_size == 1: + return _diffengine.make_rel_entr_vector_scalar(children[0], children[1]) + elif x_size == 1 and y_size > 1: + return _diffengine.make_rel_entr_scalar_vector(children[0], children[1]) + else: + raise ValueError( + f"rel_entr requires arguments to be either both scalars, both same size, " + f"or one scalar and one vector. Got sizes: x={x_size}, y={y_size}" + ) + + +def _convert_quad_form(expr, children): + """Convert quadratic form x.T @ P @ x.""" + + P = expr.args[1] + + if not isinstance(P, cp.Constant): + raise NotImplementedError("quad_form requires P to be a constant matrix") + + P = P.value + + if not isinstance(P, sparse.csr_matrix): + P = sparse.csr_matrix(P) + + return _diffengine.make_quad_form( + children[0], + P.data.astype(np.float64), + P.indices.astype(np.int32), + P.indptr.astype(np.int32), + P.shape[0], + P.shape[1], + ) + + +def _convert_reshape(expr, children): + """Convert reshape - only Fortran order is supported. + + Note: Only order='F' (Fortran/column-major) is supported. + """ + if expr.order != "F": + raise NotImplementedError( + f"reshape with order='{expr.order}' not supported. " + "Only order='F' (Fortran) is currently supported." + ) + + d1, d2 = _normalize_shape(expr.shape) + return _diffengine.make_reshape(children[0], d1, d2) + +def _convert_broadcast(expr, children): + d1, d2 = expr.broadcast_shape + d1_C, d2_C = _diffengine.get_expr_dimensions(children[0]) + if d1_C == d1 and d2_C == d2: + return children[0] + + return _diffengine.make_broadcast(children[0], d1, d2) + +def _convert_sum(expr, children): + axis = expr.axis + if axis is None: + axis = -1 + return _diffengine.make_sum(children[0], axis) + +def _convert_promote(expr, children): + d1, d2 = _normalize_shape(expr.shape) + return _diffengine.make_promote(children[0], d1, d2) + +def _convert_NegExpression(_expr, children): + return _diffengine.make_neg(children[0]) + +def _convert_quad_over_lin(_expr, children): + return _diffengine.make_quad_over_lin(children[0], children[1]) + +def _convert_index(expr, children): + idxs = _extract_flat_indices_from_index(expr) + d1, d2 = _normalize_shape(expr.shape) + + return _diffengine.make_index(children[0], d1, d2, idxs) + +def _convert_special_index(expr, children): + idxs = _extract_flat_indices_from_special_index(expr) + d1, d2 = _normalize_shape(expr.shape) + + return _diffengine.make_index(children[0], d1, d2, idxs) + +def _convert_prod(expr, children): + axis = expr.axis + if axis is None: + return _diffengine.make_prod(children[0]) + elif axis == 0: + return _diffengine.make_prod_axis_zero(children[0]) + elif axis == 1: + return _diffengine.make_prod_axis_one(children[0]) + +def _convert_transpose(expr, children): + # If the child is a vector (shape (n,) or (n,1) or (1,n)), use reshape to transpose + child_shape = _normalize_shape(expr.args[0].shape) + + if 1 in child_shape: + return _diffengine.make_reshape(children[0], child_shape[1], child_shape[0]) + else: + return _diffengine.make_transpose(children[0]) + +def _convert_trace(_expr, children): + return _diffengine.make_trace(children[0]) + +def _convert_diag_vec(expr, children): + # C implementation only supports k=0 (main diagonal) + if expr.k != 0: + raise NotImplementedError("diag_vec with k != 0 not supported in diff engine") + return _diffengine.make_diag_vec(children[0]) + +def _convert_symbolic_quad_form(expr, children): + """Convert SymbolicQuadForm. + + Scalar output: same as QuadForm (x^T P x). + Vector output: diagonal quadratic form (diag(P) * x^2 elementwise). + """ + output_shape = expr.shape + if output_shape == () or output_shape == (1,): + # Scalar case: delegate to QuadForm converter. + return _convert_quad_form(expr, children) + + # Vector case: output[i] = P[i,i] * x[i]^2. + # This is: const_vector_mult(power(x, 2), diag(P)). + P = expr.args[1] + if not isinstance(P, cp.Constant): + raise NotImplementedError("SymbolicQuadForm requires P to be a constant matrix") + P_val = P.value + if sparse.issparse(P_val): + P_diag = np.asarray(P_val.diagonal(), dtype=np.float64) + else: + P_diag = np.asarray(np.diag(P_val), dtype=np.float64) + + x_squared = _diffengine.make_power(children[0], 2.0) + return _diffengine.make_const_vector_mult(x_squared, P_diag) + + +def _convert_divide(expr, children): + """Convert division expression. After DCP canonicalization, denominator is always constant.""" + rh_arg = expr.args[1] + if not rh_arg.is_constant(): + raise NotImplementedError("DivExpression with non-constant denominator not supported") + + denom = rh_arg.value + if sparse.issparse(denom): + denom = denom.todense() + denom = np.asarray(denom, dtype=np.float64) + + if denom.size == 1: + return _diffengine.make_const_scalar_mult(children[0], 1.0 / float(denom.flat[0])) + else: + inv_denom = (1.0 / denom).flatten(order='F') + return _diffengine.make_const_vector_mult(children[0], inv_denom) + +# Mapping from CVXPY atom names to C diff engine functions +# Converters receive (expr, children) where expr is the CVXPY expression +ATOM_CONVERTERS = { + # Elementwise unary + "log": lambda _expr, children: _diffengine.make_log(children[0]), + "exp": lambda _expr, children: _diffengine.make_exp(children[0]), + # Affine unary + "NegExpression": _convert_NegExpression, + "Promote": _convert_promote, + # N-ary (handles 2+ args) + "AddExpression": lambda _expr, children: _chain_add(children), + # Reductions + "Sum": _convert_sum, + # Bivariate + "multiply": _convert_multiply, + "QuadForm": _convert_quad_form, + "SymbolicQuadForm": _convert_symbolic_quad_form, + "quad_over_lin": _convert_quad_over_lin, + "rel_entr": _convert_rel_entr, + # Matrix multiplication + "MulExpression": _convert_matmul, + # Elementwise univariate with parameter + "Power": lambda expr, children: _diffengine.make_power(children[0], float(expr.p.value)), + "PowerApprox": lambda expr, children: _diffengine.make_power(children[0], float(expr.p.value)), + # Trigonometric + "sin": lambda _expr, children: _diffengine.make_sin(children[0]), + "cos": lambda _expr, children: _diffengine.make_cos(children[0]), + "tan": lambda _expr, children: _diffengine.make_tan(children[0]), + # Hyperbolic + "sinh": lambda _expr, children: _diffengine.make_sinh(children[0]), + "tanh": lambda _expr, children: _diffengine.make_tanh(children[0]), + "asinh": lambda _expr, children: _diffengine.make_asinh(children[0]), + "atanh": lambda _expr, children: _diffengine.make_atanh(children[0]), + # Other elementwise + "entr": lambda _expr, children: _diffengine.make_entr(children[0]), + "logistic": lambda _expr, children: _diffengine.make_logistic(children[0]), + "xexp": lambda _expr, children: _diffengine.make_xexp(children[0]), + # Indexing/slicing + "index": _convert_index, + "special_index": _convert_special_index, + "reshape": _convert_reshape, + "broadcast_to": _convert_broadcast, + # Reductions returning scalar + "Prod": _convert_prod, + "transpose": _convert_transpose, + # Horizontal stack + "Hstack": _convert_hstack, + "Trace": _convert_trace, + # Diagonal + "diag_vec": _convert_diag_vec, + # Division (denominator is always constant after DCP canonicalization) + "DivExpression": _convert_divide, +} + + +def build_variable_dict(variables: list) -> tuple[dict, int]: + """ + Build dictionary mapping CVXPY variable ids to C variables. + + Args: + variables: list of CVXPY Variable objects + + Returns: + var_dict: {var.id: c_variable} mapping + n_vars: total number of scalar variables + """ + id_map, _, n_vars, var_shapes = InverseData.get_var_offsets(variables) + + var_dict = {} + for var in variables: + offset, _ = id_map[var.id] + shape = var_shapes[var.id] + if len(shape) == 2: + d1, d2 = shape[0], shape[1] + elif len(shape) == 1: + # NumPy and CVXPY broadcasting rules treat a (n, ) vector as (1, n), + # not as (n, 1) + d1, d2 = 1, shape[0] + else: # scalar + d1, d2 = 1, 1 + c_var = _diffengine.make_variable(d1, d2, offset, n_vars) + var_dict[var.id] = c_var + + return var_dict, n_vars + + +def build_parameter_dict(parameters: list, n_vars: int) -> tuple[dict, list]: + """ + Build dictionary mapping CVXPY parameter ids to C parameter nodes. + + Args: + parameters: list of unique CVXPY Parameter objects + n_vars: total number of scalar variables + + Returns: + param_dict: {param.id: c_param_node} mapping + param_capsules: list of C capsules for registration + """ + id_map, _, _, param_shapes = InverseData.get_param_offsets(parameters) + + param_dict = {} + param_capsules = [] + for param in parameters: + offset, _ = id_map[param.id] + d1, d2 = _normalize_shape(param_shapes[param.id]) + c_param = _diffengine.make_parameter(d1, d2, offset, n_vars) + param_dict[param.id] = c_param + param_capsules.append(c_param) + return param_dict, param_capsules + + +def convert_expr(expr, var_dict: dict, n_vars: int, param_dict: dict = None): + """Convert CVXPY expression using pre-built variable dictionary.""" + # Base case: variable lookup + if isinstance(expr, cp.Variable): + return var_dict[expr.id] + + # Base case: parameter lookup (before Constant since both are Leaf subclasses) + if isinstance(expr, cp.Parameter): + if param_dict is not None and expr.id in param_dict: + return param_dict[expr.id] + # Fall through to constant if no param_dict or not found + c = expr.value + if c is None: + raise ValueError( + f"Parameter '{expr.name()}' has no value set. " + "Set parameter values before converting." + ) + if sparse.issparse(c): + c = c.todense() + c = np.asarray(c, dtype=np.float64) + d1, d2 = _normalize_shape(expr.shape) + return _diffengine.make_constant(d1, d2, n_vars, c.flatten(order='F')) + + # Base case: constant + if isinstance(expr, cp.Constant): + c = expr.value + + # we only support dense constants for now + if sparse.issparse(c): + c = c.todense() + + c = np.asarray(c, dtype=np.float64) + d1, d2 = _normalize_shape(expr.shape) + return _diffengine.make_constant(d1, d2, n_vars, c.flatten(order='F')) + + # Recursive case: atoms + atom_name = type(expr).__name__ + + if atom_name in ATOM_CONVERTERS: + children = [convert_expr(arg, var_dict, n_vars, param_dict) for arg in expr.args] + C_expr = ATOM_CONVERTERS[atom_name](expr, children) + + # check that python dimension is consistent with C dimension + d1_C, d2_C = _diffengine.get_expr_dimensions(C_expr) + d1_Python, d2_Python = _normalize_shape(expr.shape) + + if d1_C != d1_Python or d2_C != d2_Python: + raise ValueError( + f"Dimension mismatch for atom '{atom_name}': " + f"C dimensions ({d1_C}, {d2_C}) vs Python dimensions ({d1_Python}, {d2_Python})" + ) + + return C_expr + + raise NotImplementedError(f"Atom '{atom_name}' not supported") + + +def convert_expressions(problem: cp.Problem) -> tuple: + """ + Convert CVXPY Problem to C expressions (low-level). + + Args: + problem: CVXPY Problem object + + Returns: + c_objective: C expression for objective + c_constraints: list of C expressions for constraints + param_capsules: list of parameter capsules (empty if no params) + all_params: list of unique CVXPY Parameter objects + """ + var_dict, n_vars = build_variable_dict(problem.variables()) + + # Collect unique parameters + all_params = list({p.id: p for p in problem.parameters()}.values()) + if all_params: + param_dict, param_capsules = build_parameter_dict(all_params, n_vars) + else: + param_dict, param_capsules = None, [] + + # Convert objective + c_objective = convert_expr(problem.objective.expr, var_dict, n_vars, param_dict) + + # Convert constraints (expression part only for now) + c_constraints = [] + for constr in problem.constraints: + c_expr = convert_expr(constr.expr, var_dict, n_vars, param_dict) + c_constraints.append(c_expr) + + return c_objective, c_constraints, param_capsules, all_params diff --git a/cvxpy/reductions/solvers/nlp_solvers/ipopt_nlpif.py b/cvxpy/reductions/solvers/nlp_solvers/ipopt_nlpif.py new file mode 100644 index 0000000000..3406a01a89 --- /dev/null +++ b/cvxpy/reductions/solvers/nlp_solvers/ipopt_nlpif.py @@ -0,0 +1,203 @@ +""" +Copyright 2025, the CVXPY developers + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import numpy as np + +import cvxpy.settings as s +from cvxpy.reductions.solution import Solution, failure_solution +from cvxpy.reductions.solvers.nlp_solvers.nlp_solver import NLPsolver +from cvxpy.utilities.citations import CITATION_DICT + + +class IPOPT(NLPsolver): + """ + NLP interface for the IPOPT solver + """ + # Map between IPOPT status and CVXPY status + # taken from https://github.com/jump-dev/Ipopt.jl/blob/master/src/C_wrapper.jl#L485-L511 + STATUS_MAP = { + # Success cases + 0: s.OPTIMAL, # Solve_Succeeded + 1: s.OPTIMAL_INACCURATE, # Solved_To_Acceptable_Level + 6: s.OPTIMAL, # Feasible_Point_Found + + # Infeasibility/Unboundedness + 2: s.INFEASIBLE, # Infeasible_Problem_Detected + 4: s.UNBOUNDED, # Diverging_Iterates + + # Numerical/Algorithm issues + 3: s.SOLVER_ERROR, # Search_Direction_Becomes_Too_Small + -2: s.SOLVER_ERROR, # Restoration_Failed + -3: s.SOLVER_ERROR, # Error_In_Step_Computation + -13: s.SOLVER_ERROR, # Invalid_Number_Detected + -100: s.SOLVER_ERROR, # Unrecoverable_Exception + -101: s.SOLVER_ERROR, # NonIpopt_Exception_Thrown + -199: s.SOLVER_ERROR, # Internal_Error + + # User/Resource limits + 5: s.USER_LIMIT, # User_Requested_Stop + -1: s.USER_LIMIT, # Maximum_Iterations_Exceeded + -4: s.USER_LIMIT, # Maximum_CpuTime_Exceeded + -5: s.USER_LIMIT, # Maximum_WallTime_Exceeded + -102: s.USER_LIMIT, # Insufficient_Memory + + # Problem definition issues + -10: s.SOLVER_ERROR, # Not_Enough_Degrees_Of_Freedom + -11: s.SOLVER_ERROR, # Invalid_Problem_Definition + -12: s.SOLVER_ERROR, # Invalid_Option + } + + def name(self): + """ + The name of solver. + """ + return 'IPOPT' + + def import_solver(self): + """ + Imports the solver. + """ + import cyipopt # noqa F401 + + def invert(self, solution, inverse_data): + """ + Returns the solution to the original problem given the inverse_data. + """ + attr = {} + status = self.STATUS_MAP[solution['status']] + # the info object does not contain all the attributes we want + # see https://github.com/mechmotum/cyipopt/issues/17 + # attr[s.SOLVE_TIME] = solution.solve_time + attr[s.NUM_ITERS] = solution['iterations'] + # more detailed statistics here when available + # attr[s.EXTRA_STATS] = solution.extra.FOO + if 'all_objs_from_best_of' in solution: + attr[s.EXTRA_STATS] = {'all_objs_from_best_of': + solution['all_objs_from_best_of']} + + if status in s.SOLUTION_PRESENT: + primal_val = solution['obj_val'] + opt_val = primal_val + inverse_data.offset + primal_vars = {} + x_opt = solution['x'] + for id, offset in inverse_data.var_offsets.items(): + shape = inverse_data.var_shapes[id] + size = np.prod(shape, dtype=int) + primal_vars[id] = np.reshape(x_opt[offset:offset+size], shape, order='F') + return Solution(status, opt_val, primal_vars, {}, attr) + else: + return failure_solution(status, attr) + + def solve_via_data(self, data, warm_start: bool, verbose: bool, solver_opts, solver_cache=None): + """ + Returns the result of the call to the solver. + + Parameters + ---------- + data : dict + Data used by the solver. + This consists of: + - "oracles": An Oracles object that computes the objective and constraints + - "x0": Initial guess for the primal variables + - "lb": Lower bounds on the primal variables + - "ub": Upper bounds on the primal variables + - "cl": Lower bounds on the constraints + - "cu": Upper bounds on the constraints + - "objective": Function to compute the objective value + - "gradient": Function to compute the objective gradient + - "constraints": Function to compute the constraint values + - "jacobian": Function to compute the constraint Jacobian + - "jacobianstructure": Function to compute the structure of the Jacobian + - "hessian": Function to compute the Hessian of the Lagrangian + - "hessianstructure": Function to compute the structure of the Hessian + warm_start : bool + Not used. + verbose : bool + Should the solver print output? + solver_opts : dict + Additional arguments for the solver. + solver_cache: None + None + + Returns + ------- + tuple + (status, optimal value, primal, equality dual, inequality dual) + """ + import cyipopt + + from cvxpy.reductions.solvers.nlp_solvers.nlp_solver import Oracles + + # Create oracles object (deferred from apply() so we have access to verbose) + bounds = data["_bounds"] + + # Detect quasi-Newton mode (L-BFGS) - skip Hessian initialization if not needed + hessian_approx = 'exact' + if solver_opts: + hessian_approx = solver_opts.get('hessian_approximation', 'exact') + use_hessian = (hessian_approx == 'exact') + + oracles = Oracles(bounds.new_problem, bounds.x0, len(bounds.cl), + verbose=verbose, use_hessian=use_hessian) + + nlp = cyipopt.Problem( + n=len(data["x0"]), + m=len(data["cl"]), + problem_obj=oracles, + lb=data["lb"], + ub=data["ub"], + cl=data["cl"], + cu=data["cu"], + ) + # Set default IPOPT options, but use solver_opts if provided + default_options = { + 'mu_strategy': 'adaptive', + 'tol': 1e-7, + 'bound_relax_factor': 0.0, + 'hessian_approximation': 'exact', + 'derivative_test': 'none', + 'least_square_init_duals': 'yes' + } + # Update defaults with user-provided options + if solver_opts: + default_options.update(solver_opts) + if not verbose and 'print_level' not in default_options: + default_options['print_level'] = 3 + # Apply all options to the nlp object + for option_name, option_value in default_options.items(): + nlp.add_option(option_name, option_value) + + _, info = nlp.solve(data["x0"]) + + if oracles.iterations == 0 and info['status'] == s.OPTIMAL: + print("Warning: IPOPT returned after 0 iterations. This may indicate that\n" + "the initial point passed to Ipopt is a stationary point, and it is\n" + "quite unlikely that the initial point is also a local minimizer. \n" + "Perturb the initial point and try again.") + + # add number of iterations to info dict from oracles + info['iterations'] = oracles.iterations + return info + + def cite(self, data): + """Returns bibtex citation for the solver. + + Parameters + ---------- + data : dict + Data generated via an apply call. + """ + return CITATION_DICT["IPOPT"] diff --git a/cvxpy/reductions/solvers/nlp_solvers/knitro_nlpif.py b/cvxpy/reductions/solvers/nlp_solvers/knitro_nlpif.py new file mode 100644 index 0000000000..d10a1e762c --- /dev/null +++ b/cvxpy/reductions/solvers/nlp_solvers/knitro_nlpif.py @@ -0,0 +1,393 @@ +""" +Copyright 2025, the CVXPY developers + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import numpy as np + +import cvxpy.settings as s +from cvxpy.reductions.solution import Solution, failure_solution +from cvxpy.reductions.solvers.nlp_solvers.nlp_solver import NLPsolver +from cvxpy.utilities.citations import CITATION_DICT + + +class KNITRO(NLPsolver): + """ + NLP interface for the KNITRO solver + """ + + BOUNDED_VARIABLES = True + # Keys: + CONTEXT_KEY = "context" + X_INIT_KEY = "x_init" + Y_INIT_KEY = "y_init" + + # Keyword arguments for the CVXPY interface. + INTERFACE_ARGS = [X_INIT_KEY, Y_INIT_KEY] + + # Map of Knitro status to CVXPY status. + # This is based on the Knitro documentation: + # https://www.artelys.com/app/docs/knitro/3_referenceManual/returnCodes.html + STATUS_MAP = { + 0: s.OPTIMAL, + -100: s.OPTIMAL_INACCURATE, + -101: s.USER_LIMIT, + -102: s.USER_LIMIT, + -103: s.USER_LIMIT, + -200: s.INFEASIBLE, + -201: s.INFEASIBLE, + -202: s.INFEASIBLE, + -203: s.INFEASIBLE, + -204: s.INFEASIBLE, + -205: s.INFEASIBLE, + -300: s.UNBOUNDED, + -301: s.UNBOUNDED, + -400: s.USER_LIMIT, + -401: s.USER_LIMIT, + -402: s.USER_LIMIT, + -403: s.USER_LIMIT, + -404: s.USER_LIMIT, + -405: s.USER_LIMIT, + -406: s.USER_LIMIT, + -410: s.USER_LIMIT, + -411: s.USER_LIMIT, + -412: s.USER_LIMIT, + -413: s.USER_LIMIT, + -415: s.USER_LIMIT, + -416: s.USER_LIMIT, + -500: s.SOLVER_ERROR, + -501: s.SOLVER_ERROR, + -502: s.SOLVER_ERROR, + -503: s.SOLVER_ERROR, + -504: s.SOLVER_ERROR, + -505: s.SOLVER_ERROR, + -506: s.SOLVER_ERROR, + -507: s.SOLVER_ERROR, + -508: s.SOLVER_ERROR, + -509: s.SOLVER_ERROR, + -510: s.SOLVER_ERROR, + -511: s.SOLVER_ERROR, + -512: s.SOLVER_ERROR, + -513: s.SOLVER_ERROR, + -514: s.SOLVER_ERROR, + -515: s.SOLVER_ERROR, + -516: s.SOLVER_ERROR, + -517: s.SOLVER_ERROR, + -518: s.SOLVER_ERROR, + -519: s.SOLVER_ERROR, + -520: s.SOLVER_ERROR, + -521: s.SOLVER_ERROR, + -522: s.SOLVER_ERROR, + -523: s.SOLVER_ERROR, + -524: s.SOLVER_ERROR, + -525: s.SOLVER_ERROR, + -526: s.SOLVER_ERROR, + -527: s.SOLVER_ERROR, + -528: s.SOLVER_ERROR, + -529: s.SOLVER_ERROR, + -530: s.SOLVER_ERROR, + -531: s.SOLVER_ERROR, + -532: s.SOLVER_ERROR, + -600: s.SOLVER_ERROR, + } + + def name(self): + """ + The name of solver. + """ + return 'KNITRO' + + def import_solver(self): + """ + Imports the solver. + """ + import knitro # noqa F401 + + def invert(self, solution, inverse_data): + """ + Returns the solution to the original problem given the inverse_data. + """ + attr = { + s.NUM_ITERS: solution.get('num_iters'), + s.SOLVE_TIME: solution.get('solve_time_real'), + } + status = self.STATUS_MAP[solution['status']] + if status in s.SOLUTION_PRESENT: + primal_val = solution['obj_val'] + opt_val = primal_val + inverse_data.offset + primal_vars = {} + x_opt = solution['x'] + for id, offset in inverse_data.var_offsets.items(): + shape = inverse_data.var_shapes[id] + size = np.prod(shape, dtype=int) + primal_vars[id] = np.reshape(x_opt[offset:offset+size], shape, order='F') + return Solution(status, opt_val, primal_vars, {}, attr) + else: + return failure_solution(status, attr) + + def solve_via_data(self, data, warm_start: bool, verbose: bool, solver_opts, solver_cache=None): + """ + Returns the result of the call to the solver. + + Parameters + ---------- + data : dict + Data used by the solver. + This consists of: + - "oracles": An Oracles object that computes the objective and constraints + - "x0": Initial guess for the primal variables + - "lb": Lower bounds on the primal variables + - "ub": Upper bounds on the primal variables + - "cl": Lower bounds on the constraints + - "cu": Upper bounds on the constraints + warm_start : bool + Not used. + verbose : bool + Should the solver print output? + solver_opts : dict + Additional arguments for the solver. + solver_cache: None + None + + Returns + ------- + tuple + (status, optimal value, primal, equality dual, inequality dual) + """ + import knitro + + from cvxpy.reductions.solvers.nlp_solvers.nlp_solver import Oracles + + # Create oracles object (deferred from apply() so we have access to verbose) + bounds = data["_bounds"] + + # Detect quasi-Newton mode: hessopt 1=exact, 2=BFGS, 3=SR1, 6=L-BFGS + # Only use exact Hessian when hessopt is 1 (default) + hessopt = solver_opts.get('hessopt', 1) if solver_opts else 1 + use_hessian = (hessopt == 1) + + oracles = Oracles(bounds.new_problem, bounds.x0, len(bounds.cl), + verbose=verbose, use_hessian=use_hessian) + + # Extract data from the data dictionary + x0 = data["x0"] + lb, ub = data["lb"].copy(), data["ub"].copy() + cl, cu = data["cl"].copy(), data["cu"].copy() + + lb[lb == -np.inf] = -knitro.KN_INFINITY + ub[ub == np.inf] = knitro.KN_INFINITY + cl[cl == -np.inf] = -knitro.KN_INFINITY + cu[cu == np.inf] = knitro.KN_INFINITY + + n = len(x0) # number of variables + m = len(cl) # number of constraints + + # Create a new Knitro solver instance + kc = knitro.KN_new() + + try: + # Add variables + knitro.KN_add_vars(kc, n) + # Set variable bounds + knitro.KN_set_var_lobnds(kc, xLoBnds=lb) + knitro.KN_set_var_upbnds(kc, xUpBnds=ub) + + # Set initial values for variables + knitro.KN_set_var_primal_init_values(kc, xInitVals=x0) + + # Add constraints (if any) + if m > 0: + knitro.KN_add_cons(kc, m) + knitro.KN_set_con_lobnds(kc, cLoBnds=cl) + knitro.KN_set_con_upbnds(kc, cUpBnds=cu) + + # Set objective goal to minimize + knitro.KN_set_obj_goal(kc, knitro.KN_OBJGOAL_MINIMIZE) + # Set verbosity + if not verbose: + knitro.KN_set_int_param(kc, knitro.KN_PARAM_OUTLEV, 0) + + # Define the callback for evaluating objective and constraints (EVALFC) + def callbackEvalFC(kc, cb, evalRequest, evalResult, userParams): + if evalRequest.type != knitro.KN_RC_EVALFC: + return -1 # Error: wrong evaluation type + + # Convert x from list to numpy array + x = np.array(evalRequest.x) + + # Evaluate objective + evalResult.obj = oracles.objective(x) + + # Evaluate constraints (if any) + if m > 0: + c_vals = oracles.constraints(x) + evalResult.c = c_vals + return 0 # Success + + # Register the evaluation callback + cb = knitro.KN_add_eval_callback( + kc, + evalObj=True, + indexCons=(list(range(m)) if m > 0 else None), + funcCallback=callbackEvalFC, + ) + + # Get the Jacobian sparsity structure + if m > 0: + jac_rows, jac_cols = oracles.jacobianstructure() + else: + jac_rows = None + jac_cols = None + + # Define the callback for evaluating gradients (EVALGA) + def callbackEvalGA(kc, cb, evalRequest, evalResult, userParams): + if evalRequest.type != knitro.KN_RC_EVALGA: + return -1 # Error: wrong evaluation type + + try: + x = np.array(evalRequest.x) + # Evaluate objective gradient + grad = oracles.gradient(x) + evalResult.objGrad = np.asarray(grad).flatten() + + # Evaluate constraint Jacobian (if any) + if m > 0: + jac_vals = oracles.jacobian(x) + evalResult.jac = np.asarray(jac_vals).flatten() + + return 0 # Success + except Exception as e: + print(f"Error in callbackEvalGA: {e}") + return -1 + + # Register the gradient callback with sparsity structure + knitro.KN_set_cb_grad( + kc, + cb, + objGradIndexVars=list(range(n)), + jacIndexCons=jac_rows, + jacIndexVars=jac_cols, + gradCallback=callbackEvalGA, + ) + # oracles.hessianstructure() returns lower triangular (rows >= cols) + # KNITRO expects upper triangular, so we swap rows and cols + hess_cols, hess_rows = oracles.hessianstructure() + + # Define the callback for evaluating Hessian (EVALH) + def callbackEvalH(kc, cb, evalRequest, evalResult, userParams): + if evalRequest.type not in (knitro.KN_RC_EVALH, knitro.KN_RC_EVALH_NO_F): + return -1 # Error: wrong evaluation type + + try: + x = np.array(evalRequest.x) + # Get sigma (objective factor) and lambda (constraint multipliers) + sigma = evalRequest.sigma + lambda_ = np.array(evalRequest.lambda_) + + # For KN_RC_EVALH_NO_F, the objective component should not be included + if evalRequest.type == knitro.KN_RC_EVALH_NO_F: + sigma = 0.0 + + # Evaluate Hessian of the Lagrangian + hess_vals = oracles.hessian(x, lambda_, sigma) + hess_vals = np.asarray(hess_vals).flatten() + + evalResult.hess = hess_vals + + return 0 # Success + except Exception as e: + print(f"Error in callbackEvalH: {e}") + return -1 + + # Register the Hessian callback with sparsity structure + knitro.KN_set_cb_hess( + kc, + cb, + hessIndexVars1=hess_rows, + hessIndexVars2=hess_cols, + hessCallback=callbackEvalH, + ) + + # default options (can be overridden by solver_opts) + knitro.KN_set_int_param(kc, knitro.KN_PARAM_HESSOPT, knitro.KN_HESSOPT_EXACT) + knitro.KN_set_int_param(kc, knitro.KN_PARAM_HONORBNDS, 1) + + # Apply solver options from solver_opts + # Map common string option names to KNITRO parameter constants + OPTION_MAP = { + 'algorithm': knitro.KN_PARAM_ALGORITHM, + 'maxit': knitro.KN_PARAM_MAXIT, + 'outlev': knitro.KN_PARAM_OUTLEV, + 'hessopt': knitro.KN_PARAM_HESSOPT, + 'gradopt': knitro.KN_PARAM_GRADOPT, + 'feastol': knitro.KN_PARAM_FEASTOL, + 'opttol': knitro.KN_PARAM_OPTTOL, + 'honorbnds': knitro.KN_PARAM_HONORBNDS, + } + + if solver_opts: + for option_name, option_value in solver_opts.items(): + # Map string names to KNITRO param IDs + if isinstance(option_name, str): + option_name_lower = option_name.lower() + if option_name_lower in OPTION_MAP: + param_id = OPTION_MAP[option_name_lower] + else: + raise ValueError(f"Unknown KNITRO option: {option_name}") + else: + # Assume it's already a KNITRO param ID + param_id = option_name + + # Set the parameter based on value type + if isinstance(option_value, int): + knitro.KN_set_int_param(kc, param_id, option_value) + elif isinstance(option_value, float): + knitro.KN_set_double_param(kc, param_id, option_value) + + # Solve the problem + nStatus = knitro.KN_solve(kc) + + # Retrieve the solution + nStatus, objSol, x_sol, lambda_sol = knitro.KN_get_solution(kc) + + # Retrieve solve statistics + num_iters = knitro.KN_get_number_iters(kc) + solve_time_cpu = knitro.KN_get_solve_time_cpu(kc) + solve_time_real = knitro.KN_get_solve_time_real(kc) + + # Return results in dictionary format expected by invert() + solution = { + 'status': nStatus, + 'obj_val': objSol, + 'x': np.array(x_sol), + 'lambda': np.array(lambda_sol), + 'num_iters': num_iters, + 'solve_time_cpu': solve_time_cpu, + 'solve_time_real': solve_time_real, + } + return solution + + finally: + # Always free the Knitro context + knitro.KN_free(kc) + + def cite(self, data): + """Returns bibtex citation for the solver. + + Parameters + ---------- + data : dict + Data generated via an apply call. + """ + return CITATION_DICT["KNITRO"] diff --git a/cvxpy/reductions/solvers/nlp_solvers/nlp_solver.py b/cvxpy/reductions/solvers/nlp_solvers/nlp_solver.py new file mode 100644 index 0000000000..1211e6f374 --- /dev/null +++ b/cvxpy/reductions/solvers/nlp_solvers/nlp_solver.py @@ -0,0 +1,495 @@ +""" +Copyright 2025, the CVXPY developers + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import numpy as np + +from cvxpy.constraints import ( + Equality, + Inequality, + NonPos, +) +from cvxpy.reductions.inverse_data import InverseData +from cvxpy.reductions.solvers.solver import Solver +from cvxpy.reductions.utilities import ( + lower_equality, + lower_ineq_to_nonneg, + nonpos2nonneg, +) + + +class NLPsolver(Solver): + """ + A non-linear programming (NLP) solver. + """ + REQUIRES_CONSTR = False + MIP_CAPABLE = False + + def accepts(self, problem): + """ + Only accepts disciplined nonlinear programs. + """ + return problem.is_dnlp() + + def apply(self, problem): + """ + Construct NLP problem data stored in a dictionary. + The NLP has the following form + + minimize f(x) + subject to g^l <= g(x) <= g^u + x^l <= x <= x^u + where f and g are non-linear (and possibly non-convex) functions + """ + problem, data, inv_data = self._prepare_data_and_inv_data(problem) + + return data, inv_data + + def _prepare_data_and_inv_data(self, problem): + data = dict() + bounds = Bounds(problem) + inverse_data = InverseData(bounds.new_problem) + inverse_data.offset = 0.0 + data["problem"] = bounds.new_problem + data["cl"], data["cu"] = bounds.cl, bounds.cu + data["lb"], data["ub"] = bounds.lb, bounds.ub + data["x0"] = bounds.x0 + data["_bounds"] = bounds # Store for deferred Oracles creation in solve_via_data + return problem, data, inverse_data + +class Bounds(): + def __init__(self, problem): + self.problem = problem + self.main_var = problem.variables() + self.get_constraint_bounds() + self.get_variable_bounds() + self.construct_initial_point() + + def get_constraint_bounds(self): + """ + Get constraint bounds for all constraints. + Also converts inequalities to nonneg form, + as well as equalities to zero constraints and forms + a new problem from the canonicalized problem. + """ + lower, upper = [], [] + new_constr = [] + for constraint in self.problem.constraints: + if isinstance(constraint, Equality): + lower.extend([0.0] * constraint.size) + upper.extend([0.0] * constraint.size) + new_constr.append(lower_equality(constraint)) + elif isinstance(constraint, Inequality): + lower.extend([0.0] * constraint.size) + upper.extend([np.inf] * constraint.size) + new_constr.append(lower_ineq_to_nonneg(constraint)) + elif isinstance(constraint, NonPos): + lower.extend([0.0] * constraint.size) + upper.extend([np.inf] * constraint.size) + new_constr.append(nonpos2nonneg(constraint)) + canonicalized_prob = self.problem.copy([self.problem.objective, new_constr]) + self.new_problem = canonicalized_prob + self.cl = np.array(lower) + self.cu = np.array(upper) + + def get_variable_bounds(self): + """ + Get variable bounds for all variables. + Uses the variable's get_bounds() method which handles bounds attributes, + nonneg/nonpos attributes, and properly broadcasts scalar bounds. + """ + var_lower, var_upper = [], [] + for var in self.main_var: + # get_bounds() returns arrays broadcastable to var.shape + # and handles all edge cases (scalar bounds, sign attributes, etc.) + lb, ub = var.get_bounds() + # Flatten in column-major (Fortran) order and convert to contiguous array + # (broadcast_to creates read-only views that need to be copied) + lb_flat = np.asarray(lb).flatten(order='F') + ub_flat = np.asarray(ub).flatten(order='F') + var_lower.extend(lb_flat) + var_upper.extend(ub_flat) + self.lb = np.array(var_lower) + self.ub = np.array(var_upper) + + + def construct_initial_point(self): + """ Loop through all variables and collect the intial point.""" + x0 = [] + for var in self.main_var: + if var.value is None: + raise ValueError("Variable %s has no value. This is a bug and should be reported." + % var.name()) + + x0.append(np.atleast_1d(var.value).flatten(order='F')) + self.x0 = np.concatenate(x0, axis=0) + +class Oracles(): + """ + Oracle interface for NLP solvers using the C-based diff engine. + + Provides function and derivative oracles (objective, gradient, constraints, + jacobian, hessian) by wrapping the C_problem class from dnlp_diff_engine. + """ + + def __init__(self, problem, initial_point, num_constraints, + verbose: bool = True, use_hessian: bool = True): + # Import from cvxpy's diff_engine integration layer + from cvxpy.reductions.solvers.nlp_solvers.diff_engine import C_problem + + self.c_problem = C_problem(problem, verbose=verbose) + self.use_hessian = use_hessian + + # Always initialize Jacobian + self.c_problem.init_jacobian() + + # Only initialize Hessian if needed (not for quasi-Newton methods) + if use_hessian: + self.c_problem.init_hessian() + + self.initial_point = initial_point + self.num_constraints = num_constraints + self.iterations = 0 + + # Cached sparsity structures + self._jac_structure = None + self._hess_structure = None + self.constraints_forward_passed = False + self.objective_forward_passed = False + + def objective(self, x): + """Returns the scalar value of the objective given x.""" + self.objective_forward_passed = True + return self.c_problem.objective_forward(x) + + def gradient(self, x): + """Returns the gradient of the objective with respect to x.""" + + if not self.objective_forward_passed: + self.objective(x) + + return self.c_problem.gradient() + + def constraints(self, x): + """Returns the constraint values.""" + self.constraints_forward_passed = True + return self.c_problem.constraint_forward(x) + + def jacobian(self, x): + """Returns the Jacobian values in COO format at the sparsity structure. """ + + if not self.constraints_forward_passed: + self.constraints(x) + + jac_csr = self.c_problem.jacobian() + jac_coo = jac_csr.tocoo() + return jac_coo.data.copy() + + def jacobianstructure(self): + """Returns the sparsity structure of the Jacobian.""" + if self._jac_structure is not None: + return self._jac_structure + + jac_csr = self.c_problem.get_jacobian() + jac_coo = jac_csr.tocoo() + + self._jac_structure = (jac_coo.row.astype(np.int32), + jac_coo.col.astype(np.int32)) + return self._jac_structure + + def hessian(self, x, duals, obj_factor): + """Returns the lower triangular Hessian values in COO format. """ + if not self.use_hessian: + # Shouldn't be called when using quasi-Newton, but return empty array + return np.array([]) + + if not self.objective_forward_passed: + self.objective(x) + if not self.constraints_forward_passed: + self.constraints(x) + + hess_csr = self.c_problem.hessian(obj_factor, duals) + hess_coo = hess_csr.tocoo() + + # Extract lower triangular values + mask = hess_coo.row >= hess_coo.col + + return hess_coo.data[mask] + + def hessianstructure(self): + """Returns the sparsity structure of the lower triangular Hessian.""" + if not self.use_hessian: + # Return empty structure when using quasi-Newton approximation + return (np.array([], dtype=np.int32), np.array([], dtype=np.int32)) + + if self._hess_structure is not None: + return self._hess_structure + + hess_csr = self.c_problem.get_hessian() + hess_coo = hess_csr.tocoo() + + # Keep only lower triangular + mask = hess_coo.row >= hess_coo.col + self._hess_structure = ( + hess_coo.row[mask].astype(np.int32), + hess_coo.col[mask].astype(np.int32) + ) + return self._hess_structure + + def intermediate(self, alg_mod, iter_count, obj_value, inf_pr, inf_du, mu, + d_norm, regularization_size, alpha_du, alpha_pr, + ls_trials): + """Prints information at every Ipopt iteration.""" + self.iterations = iter_count + self.objective_forward_passed = False + self.constraints_forward_passed = False + + +# TODO: maybe add a cchecker like this to the diff-engine? Or rather do a checker that +# uses cvxpy expressions to evaluate values. It will be slower, but will better test +# consistency with cvxpy. +class DerivativeChecker: + """ + A utility class to verify derivative computations by comparing + C-based diff engine results against Python-based evaluations. + """ + + def __init__(self, problem): + """ + Initialize the derivative checker with a CVXPY problem. + + Parameters + ---------- + problem : cvxpy.Problem + The CVXPY problem to check derivatives for. + """ + from cvxpy.reductions.dnlp2smooth.dnlp2smooth import Dnlp2Smooth + from cvxpy.reductions.solvers.nlp_solvers.diff_engine import C_problem + + self.original_problem = problem + + # Apply Dnlp2Smooth to get canonicalized problem + canon = Dnlp2Smooth().apply(problem) + self.canonicalized_problem = canon[0] + + # Construct the C version + print("Constructing C diff engine problem for derivative checking...") + self.c_problem = C_problem(self.canonicalized_problem) + print("Done constructing C diff engine problem.") + + # Construct initial point using Bounds functionality + self.bounds = Bounds(self.canonicalized_problem) + self.x0 = self.bounds.x0 + + # Initialize constraint bounds for checking + self.cl = self.bounds.cl + self.cu = self.bounds.cu + + def check_constraint_values(self, x=None): + if x is None: + x = self.x0 + + # Evaluate constraints using C implementation + c_values = self.c_problem.constraint_forward(x) + + # Evaluate constraints using Python implementation + # First, set variable values + x_offset = 0 + for var in self.canonicalized_problem.variables(): + var_size = var.size + var.value = x[x_offset:x_offset + var_size].reshape(var.shape, order='F') + x_offset += var_size + + # Now evaluate each constraint + python_values = [] + for constr in self.canonicalized_problem.constraints: + constr_val = constr.expr.value.flatten(order='F') + python_values.append(constr_val) + + python_values = np.hstack(python_values) if python_values else np.array([]) + + match = np.allclose(c_values, python_values, rtol=1e-10, atol=1e-10) + return match + + def check_jacobian(self, x=None, epsilon=1e-8): + if x is None: + x = self.x0 + + # Get Jacobian from C implementation + self.c_problem.init_jacobian() + self.c_problem.init_hessian() + self.c_problem.constraint_forward(x) + c_jac_csr = self.c_problem.jacobian() + c_jac_dense = c_jac_csr.toarray() + + # Compute numerical Jacobian using central differences + n_vars = len(x) + n_constraints = len(self.cl) + numerical_jac = np.zeros((n_constraints, n_vars)) + + # Define constraint function for finite differences + def constraint_func(x_eval): + return self.c_problem.constraint_forward(x_eval) + + # Compute each column using central differences + for j in range(n_vars): + x_plus = x.copy() + x_minus = x.copy() + x_plus[j] += epsilon + x_minus[j] -= epsilon + + c_plus = constraint_func(x_plus) + c_minus = constraint_func(x_minus) + + numerical_jac[:, j] = (c_plus - c_minus) / (2 * epsilon) + + match = np.allclose(c_jac_dense, numerical_jac, rtol=1e-4, atol=1e-5) + return match + + def check_hessian(self, x=None, duals=None, obj_factor=1.0, epsilon=1e-8): + if x is None: + x = self.x0 + + if duals is None: + duals = np.random.rand(len(self.cl)) + + # Get Hessian from C implementation + self.c_problem.objective_forward(x) + self.c_problem.constraint_forward(x) + #jac = self.c_problem.jacobian() + + # must run gradient because for logistic it fills some values + self.c_problem.gradient() + c_hess_csr = self.c_problem.hessian(obj_factor, duals) + + # Convert to full dense matrix (C returns lower triangular) + c_hess_coo = c_hess_csr.tocoo() + n_vars = len(x) + c_hess_dense = np.zeros((n_vars, n_vars)) + + # Fill in the full symmetric matrix from lower triangular + for i, j, v in zip(c_hess_coo.row, c_hess_coo.col, c_hess_coo.data): + c_hess_dense[i, j] = v + if i != j: + c_hess_dense[j, i] = v + + # Compute numerical Hessian using finite differences of the Lagrangian gradient + # Lagrangian gradient: ∇L = obj_factor * ∇f + J^T * duals + def lagrangian_gradient(x_eval): + self.c_problem.objective_forward(x_eval) + grad_f = self.c_problem.gradient() + + self.c_problem.constraint_forward(x_eval) + jac = self.c_problem.jacobian() + + # Lagrangian gradient = obj_factor * grad_f + J^T * duals + return obj_factor * grad_f + jac.T @ duals + + # Compute Hessian via central differences of gradient + numerical_hess = np.zeros((n_vars, n_vars)) + for j in range(n_vars): + x_plus = x.copy() + x_minus = x.copy() + x_plus[j] += epsilon + x_minus[j] -= epsilon + + grad_plus = lagrangian_gradient(x_plus) + grad_minus = lagrangian_gradient(x_minus) + + numerical_hess[:, j] = (grad_plus - grad_minus) / (2 * epsilon) + + # Symmetrize the numerical Hessian (average with transpose to reduce numerical errors) + numerical_hess = (numerical_hess + numerical_hess.T) / 2 + + match = np.allclose(c_hess_dense, numerical_hess, rtol=1e-4, atol=1e-6) + return match + + def check_objective_value(self, x=None): + """ Compare objective value from C implementation with Python implementation. """ + if x is None: + x = self.x0 + + # Evaluate objective using C implementation + c_obj_value = self.c_problem.objective_forward(x) + + # Evaluate objective using Python implementation + x_offset = 0 + for var in self.canonicalized_problem.variables(): + var_size = var.size + var.value = x[x_offset:x_offset + var_size].reshape(var.shape, order='F') + x_offset += var_size + + python_obj_value = self.canonicalized_problem.objective.expr.value + + # Compare results + match = np.allclose(c_obj_value, python_obj_value, rtol=1e-10, atol=1e-10) + + return match + + def check_gradient(self, x=None, epsilon=1e-8): + """ Compare C-based gradient with numerical approximation using finite differences. """ + if x is None: + x = self.x0 + # Get gradient from C implementation + self.c_problem.objective_forward(x) + c_grad = self.c_problem.gradient() + + # Compute numerical gradient using central differences + n_vars = len(x) + numerical_grad = np.zeros(n_vars) + + def objective_func(x_eval): + return self.c_problem.objective_forward(x_eval) + + # Compute each component using central differences + for j in range(n_vars): + x_plus = x.copy() + x_minus = x.copy() + x_plus[j] += epsilon + x_minus[j] -= epsilon + + f_plus = objective_func(x_plus) + f_minus = objective_func(x_minus) + + numerical_grad[j] = (f_plus - f_minus) / (2 * epsilon) + + match = np.allclose(c_grad, numerical_grad, rtol= 5 * 1e-3, atol=1e-5) + assert(match) + return match + + def run(self, x=None): + """ Run all derivative checks (constraints, Jacobian, and Hessian). """ + + self.c_problem.init_jacobian() + self.c_problem.init_hessian() + objective_result = self.check_objective_value(x) + gradient_result = self.check_gradient(x) + constraints_result = self.check_constraint_values() + jacobian_result = self.check_jacobian(x) + hessian_result = self.check_hessian(x) + + result = {'objective': objective_result, + 'gradient': gradient_result, + 'constraints': constraints_result, + 'jacobian': jacobian_result, + 'hessian': hessian_result} + + return result + + def run_and_assert(self, x=None): + """ Run all derivative checks and assert correctness. """ + results = self.run(x) + for key, passed in results.items(): + assert passed, f"Derivative check failed for {key}." \ No newline at end of file diff --git a/cvxpy/reductions/solvers/nlp_solvers/uno_nlpif.py b/cvxpy/reductions/solvers/nlp_solvers/uno_nlpif.py new file mode 100644 index 0000000000..1d53eb0e53 --- /dev/null +++ b/cvxpy/reductions/solvers/nlp_solvers/uno_nlpif.py @@ -0,0 +1,342 @@ +""" +Copyright 2025, the CVXPY developers + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import numpy as np + +import cvxpy.settings as s +from cvxpy.reductions.solution import Solution, failure_solution +from cvxpy.reductions.solvers.nlp_solvers.nlp_solver import NLPsolver +from cvxpy.utilities.citations import CITATION_DICT + + +class UNO(NLPsolver): + """ + NLP interface for the Uno solver. + + Uno is a modern nonlinear optimization solver that unifies Lagrange-Newton + methods by decomposing them into modular building blocks for constraint + relaxation, descent directions, and globalization strategies. + + For more information, see: https://github.com/cvanaret/Uno + """ + + STATUS_MAP = { + # Success cases (optimization_status) + "SUCCESS": s.OPTIMAL, + + # Limit cases + "ITERATION_LIMIT": s.USER_LIMIT, + "TIME_LIMIT": s.USER_LIMIT, + + # Error cases + "EVALUATION_ERROR": s.SOLVER_ERROR, + "ALGORITHMIC_ERROR": s.SOLVER_ERROR, + + # Solution status cases + "FEASIBLE_KKT_POINT": s.OPTIMAL, + "FEASIBLE_FJ_POINT": s.OPTIMAL_INACCURATE, + "FEASIBLE_SMALL_STEP": s.OPTIMAL_INACCURATE, + "INFEASIBLE_STATIONARY_POINT": s.INFEASIBLE, + "INFEASIBLE_SMALL_STEP": s.INFEASIBLE, + "UNBOUNDED": s.UNBOUNDED, + "NOT_OPTIMAL": s.SOLVER_ERROR, + } + + def name(self): + """ + The name of solver. + """ + return 'UNO' + + def import_solver(self): + """ + Imports the solver. + """ + import unopy # noqa F401 + + def invert(self, solution, inverse_data): + """ + Returns the solution to the original problem given the inverse_data. + """ + attr = {} + + # Get status from the solution - try optimization_status first, + # then solution_status + status_key = solution.get('optimization_status', solution.get('solution_status')) + status = self.STATUS_MAP.get(str(status_key), s.SOLVER_ERROR) + + attr[s.NUM_ITERS] = solution.get('iterations', 0) + if 'cpu_time' in solution: + attr[s.SOLVE_TIME] = solution['cpu_time'] + + if status in s.SOLUTION_PRESENT: + primal_val = solution['obj_val'] + opt_val = primal_val + inverse_data.offset + primal_vars = {} + x_opt = solution['x'] + for id, offset in inverse_data.var_offsets.items(): + shape = inverse_data.var_shapes[id] + size = np.prod(shape, dtype=int) + primal_vars[id] = np.reshape(x_opt[offset:offset+size], shape, order='F') + return Solution(status, opt_val, primal_vars, {}, attr) + else: + return failure_solution(status, attr) + + def solve_via_data(self, data, warm_start: bool, verbose: bool, solver_opts, solver_cache=None): + """ + Returns the result of the call to the solver. + + Parameters + ---------- + data : dict + Data used by the solver. This consists of: + - "oracles": An Oracles object that computes the objective and constraints + - "x0": Initial guess for the primal variables + - "lb": Lower bounds on the primal variables + - "ub": Upper bounds on the primal variables + - "cl": Lower bounds on the constraints + - "cu": Upper bounds on the constraints + warm_start : bool + Not used. + verbose : bool + Should the solver print output? + solver_opts : dict + Additional arguments for the solver. Common options include: + - "preset": Solver preset ("filtersqp" or "ipopt") + - Any other Uno option name-value pairs + solver_cache: None + Not used. + + Returns + ------- + dict + Solution dictionary with status, objective value, and primal solution. + """ + import unopy + + from cvxpy.reductions.solvers.nlp_solvers.nlp_solver import Oracles + + # Create oracles object (deferred from apply() so we have access to verbose) + bounds = data["_bounds"] + + # UNO always uses exact Hessian (no quasi-Newton option currently) + use_hessian = True + + oracles = Oracles(bounds.new_problem, bounds.x0, len(bounds.cl), + verbose=verbose, use_hessian=use_hessian) + + # Extract data from the data dictionary + x0 = data["x0"] + lb = data["lb"].copy() + ub = data["ub"].copy() + cl = data["cl"].copy() + cu = data["cu"].copy() + + n = len(x0) # number of variables + m = len(cl) # number of constraints + + # Create the Uno model using unopy constants + # Model(problem_type, number_variables, variables_lower_bounds, + # variables_upper_bounds, base_indexing) + model = unopy.Model( + unopy.PROBLEM_NONLINEAR, + n, + lb.tolist(), + ub.tolist(), + unopy.ZERO_BASED_INDEXING + ) + + # Helper to convert unopy.Vector to numpy array + # unopy.Vector doesn't support len() but supports indexing + def to_numpy(vec, size): + return np.array([vec[i] for i in range(size)]) + + # Define objective function callback + # Signature: objective(number_variables, x, objective_value, user_data) -> int + # Must write result to objective_value[0] and return 0 on success + def objective_callback(number_variables, x, objective_value, user_data): + try: + x_arr = to_numpy(x, number_variables) + objective_value[0] = oracles.objective(x_arr) + return 0 + except Exception: + return 1 + + # Define objective gradient callback + # Signature: gradient(number_variables, x, gradient, user_data) -> int + # Must write result to gradient array and return 0 on success + def gradient_callback(number_variables, x, gradient, user_data): + try: + x_arr = to_numpy(x, number_variables) + grad = oracles.gradient(x_arr) + for i in range(n): + gradient[i] = grad[i] + return 0 + except Exception: + return 1 + + # Set objective (minimization) + model.set_objective(unopy.MINIMIZE, objective_callback, gradient_callback) + + # Set constraints if there are any + if m > 0: + # Define constraints callback + # Signature: constraints(n, m, x, constraint_values, user_data) -> int + def constraints_callback(number_variables, number_constraints, x, + constraint_values, user_data): + try: + x_arr = to_numpy(x, number_variables) + cons = oracles.constraints(x_arr) + for i in range(m): + constraint_values[i] = cons[i] + return 0 + except Exception: + return 1 + + # Get Jacobian sparsity structure + jac_rows, jac_cols = oracles.jacobianstructure() + nnz_jacobian = len(jac_rows) + + # Define Jacobian callback + # Signature: jacobian(n, nnz, x, jacobian_values, user_data) -> int + def jacobian_callback(number_variables, number_jacobian_nonzeros, x, + jacobian_values, user_data): + try: + x_arr = to_numpy(x, number_variables) + jac_vals = oracles.jacobian(x_arr) + # Flatten in case it's returned as a 2D memoryview + jac_vals_arr = np.asarray(jac_vals).flatten() + for i in range(nnz_jacobian): + jacobian_values[i] = float(jac_vals_arr[i]) + return 0 + except Exception: + return 1 + + # set_constraints(number_constraints, constraint_functions, + # constraints_lower_bounds, constraints_upper_bounds, + # number_jacobian_nonzeros, jacobian_row_indices, + # jacobian_column_indices, constraint_jacobian) + model.set_constraints( + m, + constraints_callback, + cl.tolist(), + cu.tolist(), + nnz_jacobian, + jac_rows.tolist(), + jac_cols.tolist(), + jacobian_callback + ) + + # Get Hessian sparsity structure + # oracles.hessianstructure() returns lower triangular (rows >= cols) + hess_rows, hess_cols = oracles.hessianstructure() + nnz_hessian = len(hess_rows) + + # Define Lagrangian Hessian callback + # Signature: hessian(n, m, nnz, x, obj_factor, multipliers, hessian_values, user_data) + # Uno's MULTIPLIER_POSITIVE convention: L = sigma*f + sum_i lambda_i * g_i + # This matches our oracles.hessian convention + def hessian_callback(number_variables, number_constraints, number_hessian_nonzeros, + x, objective_multiplier, multipliers, hessian_values, user_data): + try: + x_arr = to_numpy(x, number_variables) + mult_arr = to_numpy(multipliers, number_constraints) if m > 0 else np.array([]) + hess_vals = oracles.hessian(x_arr, mult_arr, objective_multiplier) + # Flatten in case it's returned as a 2D array + hess_vals_arr = np.asarray(hess_vals).flatten() + for i in range(nnz_hessian): + hessian_values[i] = float(hess_vals_arr[i]) + return 0 + except Exception: + return 1 + + # set_lagrangian_hessian(number_hessian_nonzeros, hessian_triangular_part, + # hessian_row_indices, hessian_column_indices, lagrangian_hessian, + # lagrangian_sign_convention) + # hessian_triangular_part: LOWER_TRIANGLE since we store lower triangular + # lagrangian_sign_convention: MULTIPLIER_POSITIVE means L = sigma*f + lambda*g + model.set_lagrangian_hessian( + nnz_hessian, + unopy.LOWER_TRIANGLE, + hess_rows.tolist(), + hess_cols.tolist(), + hessian_callback, + unopy.MULTIPLIER_POSITIVE + ) + + # Set initial primal iterate + model.set_initial_primal_iterate(x0.tolist()) + + # Create solver and configure + uno_solver = unopy.UnoSolver() + + # Make a copy of solver_opts to avoid modifying the original + opts = dict(solver_opts) if solver_opts else {} + + # Set default preset (can be overridden by solver_opts) + default_preset = opts.pop("preset", "filtersqp") + uno_solver.set_preset(default_preset) + + # Set verbosity + if not verbose: + uno_solver.set_option("print_solution", False) + uno_solver.set_option("statistics_print_header_frequency", 0) + + # Apply user-provided solver options + for option_name, option_value in opts.items(): + uno_solver.set_option(option_name, option_value) + + # Solve the problem + result = uno_solver.optimize(model) + + # Extract solution information + # Convert enum to string for status mapping + opt_status = str(result.optimization_status).split('.')[-1] + sol_status = str(result.solution_status).split('.')[-1] + + # Convert unopy.Vector to numpy array via list() + # (np.array(unopy.Vector) returns a 0-d object array, not what we want) + solution = { + 'optimization_status': opt_status, + 'solution_status': sol_status, + 'obj_val': result.solution_objective, + 'x': np.array(list(result.primal_solution)), + 'iterations': result.number_iterations, + 'cpu_time': result.cpu_time, + 'primal_feasibility': result.solution_primal_feasibility, + 'stationarity': result.solution_stationarity, + 'complementarity': result.solution_complementarity, + } + + # Include dual solutions if available + if hasattr(result, 'constraint_dual_solution'): + solution['constraint_dual'] = np.array(list(result.constraint_dual_solution)) + if hasattr(result, 'lower_bound_dual_solution'): + solution['lower_bound_dual'] = np.array(list(result.lower_bound_dual_solution)) + if hasattr(result, 'upper_bound_dual_solution'): + solution['upper_bound_dual'] = np.array(list(result.upper_bound_dual_solution)) + + return solution + + def cite(self, data): + """Returns bibtex citation for the solver. + + Parameters + ---------- + data : dict + Data generated via an apply call. + """ + return CITATION_DICT["UNO"] diff --git a/cvxpy/reductions/solvers/qp_solvers/qp_solver.py b/cvxpy/reductions/solvers/qp_solvers/qp_solver.py index ca20d0b732..9030ae83f3 100644 --- a/cvxpy/reductions/solvers/qp_solvers/qp_solver.py +++ b/cvxpy/reductions/solvers/qp_solvers/qp_solver.py @@ -20,8 +20,8 @@ import cvxpy.settings as s from cvxpy.constraints import NonNeg, Zero from cvxpy.error import SolverError +from cvxpy.problems.param_prob import ParamProb from cvxpy.reductions.cvx_attr2constr import convex_attributes -from cvxpy.reductions.dcp2cone.cone_matrix_stuffing import ParamConeProg from cvxpy.reductions.solvers.solver import Solver @@ -73,7 +73,7 @@ def supports_quad_obj(self) -> bool: return True def accepts(self, problem): - return (isinstance(problem, ParamConeProg) + return (isinstance(problem, ParamProb) and (self.MIP_CAPABLE or not problem.is_mixed_integer()) and not convex_attributes([problem.x]) and (len(problem.constraints) > 0 or not self.REQUIRES_CONSTR) diff --git a/cvxpy/reductions/solvers/solving_chain_utils.py b/cvxpy/reductions/solvers/solving_chain_utils.py index d7b0ada124..de05ebf9aa 100644 --- a/cvxpy/reductions/solvers/solving_chain_utils.py +++ b/cvxpy/reductions/solvers/solving_chain_utils.py @@ -1,6 +1,7 @@ from cvxpy.settings import ( COO_CANON_BACKEND, CPP_CANON_BACKEND, + DIFF_ENGINE_CANON_BACKEND, SCIPY_CANON_BACKEND, ) from cvxpy.utilities.warn import warn @@ -27,6 +28,9 @@ def get_canon_backend(problem, canon_backend: str) -> str: canon_backend : str The canonicalization backend to use. """ + # Diff engine handles everything internally, no validation needed. + if canon_backend == DIFF_ENGINE_CANON_BACKEND: + return canon_backend if not problem._supports_cpp(): if canon_backend is None: diff --git a/cvxpy/settings.py b/cvxpy/settings.py index 40b4236fc6..a22df134cb 100644 --- a/cvxpy/settings.py +++ b/cvxpy/settings.py @@ -100,13 +100,15 @@ DAQP = "DAQP" HIGHS = "HIGHS" MPAX = "MPAX" +IPOPT = "IPOPT" KNITRO = "KNITRO" +UNO = "UNO" COSMO = "COSMO" SOLVERS = [CLARABEL, ECOS, CVXOPT, GLOP, GLPK, GLPK_MI, SCS, SDPA, GUROBI, OSQP, CPLEX, MOSEK, MOREAU, CBC, COPT, XPRESS, PIQP, PROXQP, QOCO, QPALM, NAG, PDLP, SCIP, SCIPY, DAQP, HIGHS, MPAX, - CUCLARABEL, CUOPT, KNITRO, COSMO, PDCS] + CUCLARABEL, CUOPT, KNITRO, COSMO, PDCS, IPOPT, UNO] # Xpress-specific items XPRESS_IIS = "XPRESS_IIS" @@ -192,6 +194,7 @@ RUST_CANON_BACKEND = "RUST" CPP_CANON_BACKEND = "CPP" COO_CANON_BACKEND = "COO" # 3D COO sparse tensor backend: O(nnz) operations for large parameters +DIFF_ENGINE_CANON_BACKEND = "DIFF_ENGINE" # C diff engine backend for parameter application # Default canonicalization backend, pyodide uses SciPy DEFAULT_CANON_BACKEND = CPP_CANON_BACKEND if sys.platform != "emscripten" else SCIPY_CANON_BACKEND @@ -203,7 +206,7 @@ # Numerical tolerances EIGVAL_TOL = 1e-10 PSD_NSD_PROJECTION_TOL = 1e-8 -GENERAL_PROJECTION_TOL = 1e-10 +GENERAL_PROJECTION_TOL = 1e-6 SPARSE_PROJECTION_TOL = 1e-10 ATOM_EVAL_TOL = 1e-4 CHOL_SYM_TOL = 1e-14 diff --git a/cvxpy/tests/nlp_tests/stress_tests_diff_engine/test_affine_matrix_atoms.py b/cvxpy/tests/nlp_tests/stress_tests_diff_engine/test_affine_matrix_atoms.py new file mode 100644 index 0000000000..731c288db0 --- /dev/null +++ b/cvxpy/tests/nlp_tests/stress_tests_diff_engine/test_affine_matrix_atoms.py @@ -0,0 +1,148 @@ + +import numpy as np +import pytest + +import cvxpy as cp +from cvxpy.reductions.solvers.defines import INSTALLED_SOLVERS +from cvxpy.reductions.solvers.nlp_solvers.nlp_solver import DerivativeChecker + + +@pytest.mark.skipif('IPOPT' not in INSTALLED_SOLVERS, reason='IPOPT is not installed.') +class TestAffineMatrixAtomsDiffEngine: + # Stress tests for affine matrix atoms in the diff engine. + + def test_one_trace(self): + np.random.seed(0) + X = cp.Variable((10, 10)) + A = np.random.rand(10, 10) + obj = cp.Minimize(cp.Trace(cp.log(A@ X))) + constr = [X >= 0.5, X <= 1] + prob = cp.Problem(obj, constr) + prob.solve(solver=cp.IPOPT, nlp=True, verbose=False) + checker = DerivativeChecker(prob) + checker.run_and_assert() + + def test_two_trace(self): + np.random.seed(0) + Y = cp.Variable((15, 5), bounds=[0.5, 1]) + A = np.random.rand(5, 15) + obj = cp.Minimize(cp.Trace(A @ Y)) + constr =[] + prob = cp.Problem(obj, constr) + prob.solve(solver=cp.IPOPT, nlp=True, verbose=False) + checker = DerivativeChecker(prob) + checker.run_and_assert() + + def test_three_trace(self): + np.random.seed(0) + X = cp.Variable((20, 20), bounds=[0.5, 1]) + Y = cp.Variable((20, 20), bounds=[0, 1]) + A = np.random.rand(20, 20) + obj = cp.Minimize(cp.Trace(cp.log(A @ X) + X @ Y)) + prob = cp.Problem(obj) + prob.solve(solver=cp.IPOPT, nlp=True, verbose=False) + checker = DerivativeChecker(prob) + checker.run_and_assert() + + def test_one_transpose(self): + np.random.seed(0) + n = 10 + k = 3 + A = np.random.rand(n, k) + X = cp.Variable((n, k), bounds = [1, 5]) + obj = cp.sum(A @ cp.transpose(cp.log(X))) + prob = cp.Problem(cp.Minimize(obj)) + prob.solve(solver=cp.IPOPT, nlp=True, verbose=False) + checker = DerivativeChecker(prob) + checker.run_and_assert() + + def test_two_transpose(self): + np.random.seed(0) + n = 10 + A = np.random.rand(n, n) + X = cp.Variable((n, n), bounds = [0.5, 5]) + obj = cp.sum(A @ (cp.log(X).T + cp.exp(X))) + constraints = [cp.sum((A @ X).T) == np.sum(A @ np.ones((n, n)))] + prob = cp.Problem(cp.Minimize(obj), constraints) + prob.solve(solver=cp.IPOPT, nlp=True, verbose=False) + checker = DerivativeChecker(prob) + checker.run_and_assert() + + def test_three_transpose(self): + np.random.seed(0) + n = 10 + A = np.random.rand(n, n) + X = cp.Variable((n, n), bounds = [0.5, 5]) + obj = cp.sum(A @ (cp.log(X).T + cp.exp(X).T)) + constraints = [cp.sum((A @ X).T.T) == np.sum(A @ np.ones((n, n)))] + prob = cp.Problem(cp.Minimize(obj), constraints) + prob.solve(solver=cp.IPOPT, nlp=True, verbose=False) + checker = DerivativeChecker(prob) + checker.run_and_assert() + + def test_four_transpose(self): + np.random.seed(0) + n, k = 10, 5 + A = np.random.randn(n, n) + A = A + A.T + V = cp.Variable((n, k)) + constraints = [V.T @ V == np.eye(k)] + + # Get eigenvectors for proper initialization (eigh returns sorted ascending) + eigvals, eigvecs = np.linalg.eigh(A) + + # find k smallest eigenvalues - initialize with k smallest eigenvectors + obj = cp.Minimize(cp.Trace(V.T @ A @ V)) + prob = cp.Problem(obj, constraints) + V.value = eigvecs[:, :k] # smallest k eigenvectors + prob.solve(solver=cp.IPOPT, nlp=True, least_square_init_duals='no') + checker = DerivativeChecker(prob) + checker.run_and_assert() + assert np.allclose(prob.value, np.sum(eigvals[:k])) + + # find k largest eigenvalues - initialize with k largest eigenvectors + obj = cp.Maximize(cp.Trace(V.T @ A @ V)) + prob = cp.Problem(obj, constraints) + V.value = eigvecs[:, -k:] # largest k eigenvectors + prob.solve(solver=cp.IPOPT, nlp=True, least_square_init_duals='no') + checker = DerivativeChecker(prob) + checker.run_and_assert() + assert np.allclose(prob.value, np.sum(eigvals[-k:])) + + def test_one_diag_vec(self): + np.random.seed(0) + n = 5 + x = cp.Variable(n, bounds=[0.5, 2]) + A = np.random.rand(n, n) + # diag(x) creates diagonal matrix from vector x + obj = cp.Minimize(cp.sum(A @ cp.diag(cp.log(x)))) + prob = cp.Problem(obj) + prob.solve(solver=cp.IPOPT, nlp=True, verbose=False) + checker = DerivativeChecker(prob) + checker.run_and_assert() + + def test_two_diag_vec(self): + np.random.seed(0) + n = 8 + x = cp.Variable(n, bounds=[1, 3]) + A = np.random.rand(n, n) + B = np.random.rand(n, n) + # Trace of product with diagonal matrix + obj = cp.Minimize(cp.Trace(A @ cp.diag(cp.exp(x)) @ B)) + prob = cp.Problem(obj) + prob.solve(solver=cp.IPOPT, nlp=True, verbose=False) + checker = DerivativeChecker(prob) + checker.run_and_assert() + + def test_three_diag_vec(self): + np.random.seed(0) + n = 6 + x = cp.Variable(n, bounds=[0.5, 2]) + y = cp.Variable(n, bounds=[0.5, 2]) + A = np.random.rand(n, n) + # Two diagonal matrices in expression + obj = cp.Minimize(cp.sum(cp.diag(x) @ A @ cp.diag(y))) + prob = cp.Problem(obj) + prob.solve(solver=cp.IPOPT, nlp=True, verbose=False) + checker = DerivativeChecker(prob) + checker.run_and_assert() diff --git a/cvxpy/tests/nlp_tests/stress_tests_diff_engine/test_affine_vector_atoms.py b/cvxpy/tests/nlp_tests/stress_tests_diff_engine/test_affine_vector_atoms.py new file mode 100644 index 0000000000..64794de75b --- /dev/null +++ b/cvxpy/tests/nlp_tests/stress_tests_diff_engine/test_affine_vector_atoms.py @@ -0,0 +1,231 @@ +import numpy as np +import pytest + +import cvxpy as cp +from cvxpy.reductions.solvers.defines import INSTALLED_SOLVERS +from cvxpy.reductions.solvers.nlp_solvers.nlp_solver import DerivativeChecker + + +@pytest.mark.skipif('IPOPT' not in INSTALLED_SOLVERS, reason='IPOPT is not installed.') +class TestAffineDiffEngine: + # Stress tests for affine vector atoms in the diff engine. + def test_row_broadcast(self): + # x is 1 x n, Y is m x n + np.random.seed(0) + m, n = 3, 4 + x = cp.Variable((1, n), bounds=[-2, 2]) + Y = cp.Variable((m, n), bounds=[-1, 1]) + obj = cp.Minimize(cp.sum(x + Y)) + prob = cp.Problem(obj) + x.value = np.random.rand(1, n) + Y.value = np.random.rand(m, n) + checker = DerivativeChecker(prob) + checker.run_and_assert() + prob.solve(solver=cp.IPOPT, nlp=True) + # Solution: x = -2, Y = -1 + assert prob.status == cp.OPTIMAL + assert np.allclose(x.value, -2, atol=1e-4) + assert np.allclose(Y.value, -1, atol=1e-4) + + def test_col_broadcast(self): + # x is m x 1, Y is m x n + np.random.seed(0) + m, n = 3, 4 + x = cp.Variable((m, 1), bounds=[-2, 2]) + Y = cp.Variable((m, n), bounds=[-1, 1]) + obj = cp.Minimize(cp.sum(x + Y)) + prob = cp.Problem(obj) + x.value = np.random.rand(m, 1) + Y.value = np.random.rand(m, n) + checker = DerivativeChecker(prob) + checker.run_and_assert() + prob.solve(solver=cp.IPOPT, nlp=True) + # Solution: x = -2, Y = -1 + assert prob.status == cp.OPTIMAL + assert np.allclose(x.value, -2, atol=1e-4) + assert np.allclose(Y.value, -1, atol=1e-4) + + def test_index_stress(self): + np.random.seed(0) + m, n = 3, 4 + X = cp.Variable((m, n), bounds=[-2, 2]) + expr = (cp.sum(X[0, :]) + cp.sum(X[0, :]) + + cp.sum(X[1, :]) + cp.sum(X[:, 2]) + X[0, 1] + X[2, 2]) + obj = cp.Minimize(expr) + prob = cp.Problem(obj) + X.value = np.random.rand(m, n) + checker = DerivativeChecker(prob) + checker.run_and_assert() + prob.solve(solver=cp.IPOPT, nlp=True) + # Solution: all X at lower bound + assert prob.status == cp.OPTIMAL + assert np.allclose(prob.value, -34.0) + + def test_duplicate_indices(self): + np.random.seed(0) + m, n = 3, 3 + X = cp.Variable((m, n), bounds=[-2, 2]) + # Use duplicate indices: X[[0,0],[1,1]] = [X[0,1], X[0,1]] + expr = cp.sum(X[[0, 0], [1, 1]]) - 2 * X[0, 1] + cp.sum(X) + obj = cp.Minimize(expr) + prob = cp.Problem(obj) + X.value = np.random.rand(m, n) + checker = DerivativeChecker(prob) + checker.run_and_assert() + prob.solve(solver=cp.IPOPT, nlp=True) + assert prob.status == cp.OPTIMAL + assert np.allclose(X.value, -2, atol=1e-4) + + def test_promote_row(self): + # Promote scalar to row vector + np.random.seed(0) + n = 4 + x = cp.Variable(bounds=[-3, 3]) + Y = cp.Variable((1, n), bounds=[-2, 2]) + obj = cp.Minimize(cp.sum(x + Y)) + prob = cp.Problem(obj) + x.value = 2.0 + Y.value = np.random.rand(1, n) + checker = DerivativeChecker(prob) + checker.run_and_assert() + prob.solve(solver=cp.IPOPT, nlp=True) + # Solution: x = -3, Y = -2 + assert prob.status == cp.OPTIMAL + assert np.allclose(x.value, -3, atol=1e-4) + assert np.allclose(Y.value, -2, atol=1e-4) + + def test_promote_col(self): + # Promote scalar to column vector + np.random.seed(0) + m = 4 + x = cp.Variable(bounds=[-3, 3]) + Y = cp.Variable((m, 1), bounds=[-2, 2]) + obj = cp.Minimize(cp.sum(x + Y)) + prob = cp.Problem(obj) + x.value = 2.0 + Y.value = np.random.rand(m, 1) + checker = DerivativeChecker(prob) + checker.run_and_assert() + prob.solve(solver=cp.IPOPT, nlp=True) + # Solution: x = -3, Y = -2 + assert prob.status == cp.OPTIMAL + assert np.allclose(x.value, -3, atol=1e-4) + assert np.allclose(Y.value, -2, atol=1e-4) + + def test_promote_add(self): + # Scalar x, matrix Y, with bounds set via the bounds attribute + np.random.seed(0) + x = cp.Variable(bounds=[-1, 1]) + Y = cp.Variable((2, 2), bounds=[0, 2]) + obj = cp.Minimize(cp.sum(x + Y)) + prob = cp.Problem(obj) + x.value = 0.0 + Y.value = np.random.rand(2, 2) + checker = DerivativeChecker(prob) + checker.run_and_assert() + prob.solve(solver=cp.IPOPT, nlp=True) + # Solution: x = -1, Y = 0 + assert prob.status == cp.OPTIMAL + assert np.allclose(x.value, -1, atol=1e-4) + assert np.allclose(Y.value, 0, atol=1e-4) + + def test_reshape(self): + x = cp.Variable(8, bounds=[-5, 5]) + A = np.random.rand(4, 2) + obj = cp.Minimize(cp.sum_squares(cp.reshape(x, (4, 2), order='F') - A)) + prob = cp.Problem(obj) + x.value = np.linspace(-2, 2, 8) + checker = DerivativeChecker(prob) + checker.run_and_assert() + prob.solve(solver=cp.IPOPT, nlp=True) + assert prob.status == cp.OPTIMAL + assert np.allclose(x.value, A.flatten(order='F'), atol=1e-4) + + def test_broadcast(self): + np.random.seed(0) + x = cp.Variable(8, bounds=[-5, 5]) + A = np.random.rand(8, 1) + obj = cp.Minimize(cp.sum_squares(x - A)) + prob = cp.Problem(obj) + x.value = np.linspace(-2, 2, 8) + checker = DerivativeChecker(prob) + checker.run_and_assert() + prob.solve(solver=cp.IPOPT, nlp=True, verbose=False) + assert prob.status == cp.OPTIMAL + assert np.allclose(x.value, np.mean(A), atol=1e-4) + + def test_hstack(self): + np.random.seed(0) + m = 5 + n = 3 + x = cp.Variable((n, 1), bounds=[-3, 3]) + y = cp.Variable((n, 1), bounds=[-2, 2]) + A1 = np.random.rand(m, n) + A2 = np.random.rand(m, n) + b1 = np.random.rand(m, 1) + b2 = np.random.rand(m, 1) + obj = cp.Minimize(cp.sum_squares(cp.hstack([A1 @ x + A2 @ y - b1, + A1 @ y + A2 @ x - b2, + A2 @ x - A1 @ y]))) + + prob = cp.Problem(obj) + + # check derivatives + x.value = np.random.rand(n, 1) + y.value = np.random.rand(n, 1) + checker = DerivativeChecker(prob) + checker.run_and_assert() + + # solve as an NLP + prob.solve(solver=cp.IPOPT, nlp=True) + nlp_sol_x = x.value + nlp_sol_y = y.value + nlp_sol_val = prob.value + + # solve as DCP + prob.solve(solver=cp.CLARABEL) + dcp_sol_x = x.value + dcp_sol_y = y.value + dcp_sol_val = prob.value + + assert np.allclose(nlp_sol_x, dcp_sol_x, atol=1e-4) + assert np.allclose(nlp_sol_y, dcp_sol_y, atol=1e-4) + assert np.allclose(nlp_sol_val, dcp_sol_val, atol=1e-4) + + def test_hstack_matrices(self): + np.random.seed(0) + m = 5 + n = 3 + X = cp.Variable((n, m), bounds=[-3, 3]) + Y = cp.Variable((n, m), bounds=[-2, 2]) + A1 = np.random.rand(m, n) + A2 = np.random.rand(m, n) + b1 = np.random.rand(m, m) + b2 = np.random.rand(m, m) + obj = cp.Minimize(cp.sum_squares(cp.hstack([A1 @ X + A2 @ Y - b1, + A1 @ Y + A2 @ X - b2, + A2 @ X - A1 @ Y]))) + + prob = cp.Problem(obj) + + # check derivatives + X.value = np.random.rand(n, m) + Y.value = np.random.rand(n, m) + checker = DerivativeChecker(prob) + checker.run_and_assert() + + # solve as an NLP + prob.solve(solver=cp.IPOPT, nlp=True) + nlp_sol_x = X.value + nlp_sol_y = Y.value + nlp_sol_val = prob.value + + # solve as DCP + prob.solve(solver=cp.CLARABEL) + dcp_sol_x = X.value + dcp_sol_y = Y.value + dcp_sol_val = prob.value + + assert np.allclose(nlp_sol_x, dcp_sol_x, atol=1e-4) + assert np.allclose(nlp_sol_y, dcp_sol_y, atol=1e-4) + assert np.allclose(nlp_sol_val, dcp_sol_val, atol=1e-4) \ No newline at end of file diff --git a/cvxpy/tests/nlp_tests/stress_tests_diff_engine/test_matmul_sparse.py b/cvxpy/tests/nlp_tests/stress_tests_diff_engine/test_matmul_sparse.py new file mode 100644 index 0000000000..63308c839d --- /dev/null +++ b/cvxpy/tests/nlp_tests/stress_tests_diff_engine/test_matmul_sparse.py @@ -0,0 +1,59 @@ +import numpy as np +import pytest +import scipy.sparse as sp + +import cvxpy as cp +from cvxpy.reductions.solvers.defines import INSTALLED_SOLVERS +from cvxpy.reductions.solvers.nlp_solvers.nlp_solver import DerivativeChecker + + +@pytest.mark.skipif('IPOPT' not in INSTALLED_SOLVERS, reason='IPOPT is not installed.') +class TestMatmulDifferentFormats: + + def test_dense_sparse_sparse(self): + n = 10 + A = np.random.rand(n, n) + c = np.random.rand(n, 1) + x = cp.Variable((n, 1), nonneg=True) + x0 = np.random.rand(n, 1) + b = A @ x0 + + x.value = 10 * np.ones((n, 1)) + obj = cp.Minimize(c.T @ x) + + # solve problem with dense A + constraints = [A @ x == b] + problem = cp.Problem(obj, constraints) + checker = DerivativeChecker(problem) + checker.run_and_assert() + problem.solve(solver=cp.IPOPT, nlp=True, verbose=False) + dense_val = problem.value + dense_sol = x.value + + x.value = 10 * np.ones((n, 1)) + + # solve problem with sparse A CSR + A_sparse = sp.csr_matrix(A) + constraints = [A_sparse @ x == b] + problem = cp.Problem(obj, constraints) + checker = DerivativeChecker(problem) + checker.run_and_assert() + problem.solve(solver=cp.IPOPT, nlp=True, verbose=False) + sparse_val = problem.value + sparse_sol = x.value + + x.value = 10 * np.ones((n, 1)) + # solve problem with sparse A CSC + A_sparse = sp.csc_matrix(A) + constraints = [A_sparse @ x == b] + problem = cp.Problem(obj, constraints) + checker = DerivativeChecker(problem) + checker.run_and_assert() + problem.solve(solver=cp.IPOPT, nlp=True, verbose=False) + csc_val = problem.value + csc_sol = x.value + + assert np.allclose(dense_val, sparse_val) + assert np.allclose(dense_val, csc_val) + assert np.allclose(dense_sol, sparse_sol) + assert np.allclose(dense_sol, csc_sol) \ No newline at end of file diff --git a/cvxpy/tests/nlp_tests/stress_tests_diff_engine/test_multiply_sparse.py b/cvxpy/tests/nlp_tests/stress_tests_diff_engine/test_multiply_sparse.py new file mode 100644 index 0000000000..d1e87bc298 --- /dev/null +++ b/cvxpy/tests/nlp_tests/stress_tests_diff_engine/test_multiply_sparse.py @@ -0,0 +1,52 @@ +import numpy as np +import pytest +import scipy.sparse as sp + +import cvxpy as cp +from cvxpy.reductions.solvers.defines import INSTALLED_SOLVERS +from cvxpy.reductions.solvers.nlp_solvers.nlp_solver import DerivativeChecker + + +@pytest.mark.skipif('IPOPT' not in INSTALLED_SOLVERS, reason='IPOPT is not installed.') +class TestMultiplyDifferentFormats: + + def test_dense_sparse_sparse(self): + np.random.seed(0) + n = 5 + + # dense + x = cp.Variable((n, n), bounds=[-2, 2]) + A = np.random.rand(n, n) - 0.5 + obj = cp.Minimize(cp.sum(cp.multiply(A, x))) + prob = cp.Problem(obj) + x.value = np.random.rand(n, n) + checker = DerivativeChecker(prob) + checker.run_and_assert() + prob.solve(nlp=True, verbose=False) + assert np.allclose(x.value[(A > 0)], -2) + assert np.allclose(x.value[(A < 0)], 2) + + # CSR + x = cp.Variable((n, n), bounds=[-2, 2]) + A = sp.csr_matrix(A) + obj = cp.Minimize(cp.sum(cp.multiply(A, x))) + prob = cp.Problem(obj) + x.value = np.random.rand(n, n) + checker = DerivativeChecker(prob) + checker.run_and_assert() + prob.solve(nlp=True, verbose=False) + assert np.allclose(x.value[(A > 0).todense()], -2) + assert np.allclose(x.value[(A < 0).todense()], 2) + + # CSC + x = cp.Variable((n, n), bounds=[-2, 2]) + A = sp.csc_matrix(A) + obj = cp.Minimize(cp.sum(cp.multiply(A, x))) + prob = cp.Problem(obj) + x.value = np.random.rand(n, n) + checker = DerivativeChecker(prob) + checker.run_and_assert() + prob.solve(nlp=True, verbose=False) + assert np.allclose(x.value[(A > 0).todense()], -2) + assert np.allclose(x.value[(A < 0).todense()], 2) + \ No newline at end of file diff --git a/cvxpy/tests/nlp_tests/stress_tests_diff_engine/test_quad_form.py b/cvxpy/tests/nlp_tests/stress_tests_diff_engine/test_quad_form.py new file mode 100644 index 0000000000..3bfae96144 --- /dev/null +++ b/cvxpy/tests/nlp_tests/stress_tests_diff_engine/test_quad_form.py @@ -0,0 +1,98 @@ +import numpy as np +import pytest +import scipy.sparse as sp + +import cvxpy as cp +from cvxpy.reductions.solvers.defines import INSTALLED_SOLVERS + + +@pytest.mark.skipif('IPOPT' not in INSTALLED_SOLVERS, reason='IPOPT is not installed.') +class TestQuadFormDifferentFormats: + + def test_quad_form_dense_sparse_sparse(self): + # Generate a random non-trivial quadratic program. + m = 15 + n = 10 + p = 5 + np.random.seed(1) + P = np.random.randn(n, n) + P = P.T @ P + q = np.random.randn(n) + G = np.random.randn(m, n) + h = G @ np.random.randn(n, 1) + A = np.random.randn(p, n) + b = np.random.randn(p, 1) + x = cp.Variable((n, 1)) + + constraints = [G @ x <= h, + A @ x == b] + + # dense problem + x.value = None + prob = cp.Problem(cp.Minimize((1/2)*cp.quad_form(x, P) + q.T @ x), + constraints) + prob.solve(nlp=True, verbose=False) + dense_val = x.value + + # CSR problem + x.value = None + P_csr = sp.csr_matrix(P) + prob = cp.Problem(cp.Minimize((1/2)*cp.quad_form(x, P_csr) + q.T @ x), + constraints) + prob.solve(nlp=True, verbose=False) + csr_val = x.value + + # CSC problem + x.value = None + P_csc = sp.csc_matrix(P) + prob = cp.Problem(cp.Minimize((1/2)*cp.quad_form(x, P_csc) + q.T @ x), + constraints) + prob.solve(nlp=True, verbose=False) + csc_val = x.value + + assert np.allclose(dense_val, csr_val) + assert np.allclose(dense_val, csc_val) + + def test_quad_form_dense_sparse_sparse_different_x(self): + # Generate a random non-trivial quadratic program. + m = 15 + n = 10 + p = 5 + np.random.seed(1) + P = np.random.randn(n, n) + P = P.T @ P + q = np.random.randn(n) + G = np.random.randn(m, n) + h = G @ np.random.randn(n) + A = np.random.randn(p, n) + b = np.random.randn(p) + x = cp.Variable(n) + + constraints = [G @ x <= h, + A @ x == b] + + # dense problem + x.value = None + prob = cp.Problem(cp.Minimize((1/2)*cp.quad_form(x, P) + q.T @ x), + constraints) + prob.solve(nlp=True, verbose=False) + dense_val = x.value + + # CSR problem + x.value = None + P_csr = sp.csr_matrix(P) + prob = cp.Problem(cp.Minimize((1/2)*cp.quad_form(x, P_csr) + q.T @ x), + constraints) + prob.solve(nlp=True, verbose=False) + csr_val = x.value + + # CSC problem + x.value = None + P_csc = sp.csc_matrix(P) + prob = cp.Problem(cp.Minimize((1/2)*cp.quad_form(x, P_csc) + q.T @ x), + constraints) + prob.solve(nlp=True, verbose=False) + csc_val = x.value + + assert np.allclose(dense_val, csr_val) + assert np.allclose(dense_val, csc_val) \ No newline at end of file diff --git a/cvxpy/tests/nlp_tests/test_ML_Gaussian_stress.py b/cvxpy/tests/nlp_tests/test_ML_Gaussian_stress.py new file mode 100644 index 0000000000..461cd7af20 --- /dev/null +++ b/cvxpy/tests/nlp_tests/test_ML_Gaussian_stress.py @@ -0,0 +1,122 @@ +import numpy as np +import numpy.linalg as LA +import pytest + +import cvxpy as cp + +# TODO (DCED): should try eg. student-t regression + +#@pytest.mark.skipif('IPOPT' not in INSTALLED_SOLVERS, reason='IPOPT is not installed.') +@pytest.mark.skipif(True, reason='We skip now.') +class TestStressMLE(): + + def test_zero_mean(self): + np.random.seed(1234) + TOL = 1e-3 + METHODS = [1, 2, 3, 4, 5] + all_n = np.arange(2, 100, 5) + #scaling_factors = [0.1, 1e0, 10] + scaling_factors = [1e0] + + for n in all_n: + np.random.seed(n) + for factor in scaling_factors: + data = factor*np.random.randn(n) + sigma_opt = (1 / np.sqrt(n)) * LA.norm(data) + res = LA.norm(data) ** 2 + for method in METHODS: + print("Method, n, scale factor: ", method, n, factor) + if method == 1: + sigma = cp.Variable((1, ), nonneg=True) + obj = (n / 2) * cp.log(2*np.pi*cp.square(sigma)) + \ + (1 / (2 * cp.square(sigma))) * res + constraints = [] + elif method == 2: + sigma2 = cp.Variable((1, )) + obj = (n / 2) * cp.log( 2 * np.pi * sigma2) + (1 / (2 * sigma2)) * res + constraints = [] + sigma = cp.sqrt(sigma2) + elif method == 3: + sigma = cp.Variable((1, )) + obj = n * cp.log(np.sqrt(2*np.pi)*sigma) + \ + (1 / (2 * cp.square(sigma))) * res + constraints = [] + elif method == 4: + sigma2 = cp.Variable((1, )) + obj = (n / 2) * cp.log(sigma2 * 2 * np.pi * -1 * -1) + \ + (1 / (2 * sigma2)) * res + constraints = [] + sigma = cp.sqrt(sigma2) + elif method == 5: + sigma = cp.Variable((1, )) + obj = n * cp.log(np.sqrt(2*np.pi)*sigma * -1 * -1 * 2 * 0.5) + \ + (1 / (2 * cp.square(sigma))) * res + constraints = [] + + problem = cp.Problem(cp.Minimize(obj), constraints) + problem.solve(solver=cp.IPOPT, nlp=True, hessian_approximation="exact", + derivative_test='none') + print("sigma.value: ", sigma.value) + print("sigma_opt: ", sigma_opt) + assert(np.abs(sigma.value - sigma_opt) / np.max([1, np.abs(sigma_opt)]) <= TOL) + + + def test_nonzero_mean(self): + np.random.seed(1234) + TOL = 1e-3 + # we do not run method 1 because it fails sometimes + METHODS = [2, 3, 4, 5] + all_n = np.arange(2, 100, 5) + scaling_factors = [1e0] + mu = cp.Variable((1, ), name="mu") + + for n in all_n: + np.random.seed(n) + for factor in scaling_factors: + data = factor*np.random.randn(n) + sigma_opt = (1 / np.sqrt(n)) * LA.norm(data - np.mean(data)) + mu_opt = np.mean(data) + for method in METHODS: + mu.value = None + print("Method, n, scale factor: ", method, n, factor) + if method == 1: + # here we wont deduce that sigma is nonnegative so it can be useful + # to mention it + sigma = cp.Variable((1, ), nonneg=True) + obj = (n / 2) * cp.log(2*np.pi*cp.square(sigma)) + \ + (1 / (2 * cp.square(sigma))) * cp.sum(cp.square(data-mu)) + constraints = [] + elif method == 2: + # here we will deduce that sigma2 is nonnegative so no need to mention it + sigma2 = cp.Variable((1, ), name="Sigma2") + obj = (n / 2) * cp.log( 2 * np.pi * sigma2) + \ + (1 / (2 * sigma2)) * cp.sum(cp.square(data-mu)) + constraints = [] + sigma = cp.sqrt(sigma2) + elif method == 3: + # here we will deduce that sigma is nonnegative so no need to mention it + sigma = cp.Variable((1, ), name="Sigma") + obj = n * cp.log(np.sqrt(2*np.pi)*sigma) + \ + (1 / (2 * cp.square(sigma))) * cp.sum(cp.square(data-mu)) + constraints = [] + elif method == 4: + # here we will deduce that sigma is nonnegative so no need to mention it + sigma2 = cp.Variable((1, ), name="Sigma2") + obj = (n / 2) * cp.log(sigma2 * 2 * np.pi * -1 * -1) + \ + (1 / (2 * sigma2)) * cp.sum(cp.square(data-mu)) + constraints = [] + sigma = cp.sqrt(sigma2) + elif method == 5: + # here we will deduce that sigma is nonnegative so no need to mention it + sigma = cp.Variable((1, ), name="Sigma") + obj = n * cp.log(np.sqrt(2*np.pi)*sigma * -1 * -1 * 2 * 0.5) + \ + (1 / (2 * cp.square(sigma))) * cp.sum(cp.square(data-mu)) + constraints = [] + + problem = cp.Problem(cp.Minimize(obj), constraints) + problem.solve(solver=cp.IPOPT, nlp=True, hessian_approximation="exact", + derivative_test='none') + print("sigma.value: ", sigma.value) + print("sigma_opt: ", sigma_opt) + assert(np.abs(sigma.value - sigma_opt) / np.max([1, np.abs(sigma_opt)]) <= TOL) + assert(np.abs(mu.value - mu_opt) / np.max([1, np.abs(mu_opt)]) <= TOL) diff --git a/cvxpy/tests/nlp_tests/test_Sharpe_ratio.py b/cvxpy/tests/nlp_tests/test_Sharpe_ratio.py new file mode 100644 index 0000000000..486909e429 --- /dev/null +++ b/cvxpy/tests/nlp_tests/test_Sharpe_ratio.py @@ -0,0 +1,44 @@ +import numpy as np +import pytest + +import cvxpy as cp +from cvxpy.reductions.solvers.defines import INSTALLED_SOLVERS +from cvxpy.reductions.solvers.nlp_solvers.nlp_solver import DerivativeChecker + +np.random.seed(0) + +@pytest.mark.skipif('IPOPT' not in INSTALLED_SOLVERS, reason='IPOPT is not installed.') +class TestSharpeRatio(): + + def test_formulation_one(self): + n = 100 + Sigma = np.random.rand(n, n) + Sigma = Sigma @ Sigma.T + mu = np.random.rand(n, ) + + x = cp.Variable((n, ), nonneg=True) + + # This type of initialization makes ipopt muich more robust. + # With no initialization it sometimes fails. Perhaps this is + # because we initialize in a very infeasible point? + x.value = np.ones(n) / n + + obj = cp.square(mu @ x) / cp.quad_form(x, Sigma) + constraints = [cp.sum(x) == 1] + problem = cp.Problem(cp.Maximize(obj), constraints) + problem.solve(solver=cp.IPOPT, nlp=True, verbose=True, hessian_approximation='exact') + x_noncvx = x.value + + # global solution computed using convex optimization + obj_convex = cp.quad_form(x, Sigma) + constraints_convex = [mu @ x == 1] + problem_convex = cp.Problem(cp.Minimize(obj_convex), constraints_convex) + problem_convex.solve(solver=cp.CLARABEL, verbose=True) + x_cvx = x.value / np.sum(x.value) + + sharpe_ratio1 = mu @ x_noncvx / np.sqrt(x_noncvx @ Sigma @ x_noncvx) + sharpe_ratio2 = mu @ x_cvx / np.sqrt(x_cvx @ Sigma @ x_cvx) + assert(np.abs(sharpe_ratio1 - sharpe_ratio2) < 1e-6) + + checker = DerivativeChecker(problem) + checker.run_and_assert() diff --git a/cvxpy/tests/nlp_tests/test_abs.py b/cvxpy/tests/nlp_tests/test_abs.py new file mode 100644 index 0000000000..be3c10ec42 --- /dev/null +++ b/cvxpy/tests/nlp_tests/test_abs.py @@ -0,0 +1,119 @@ +import numpy as np +import numpy.linalg as LA +import pytest + +import cvxpy as cp +from cvxpy.reductions.solvers.defines import INSTALLED_SOLVERS +from cvxpy.reductions.solvers.nlp_solvers.nlp_solver import DerivativeChecker + + +@pytest.mark.skipif('IPOPT' not in INSTALLED_SOLVERS, reason='IPOPT is not installed.') +class TestAbs(): + + def test_lasso_square_small(self): + np.random.seed(0) + m, n = 10, 10 + factors = np.linspace(0.1, 1, 20) + + for factor in factors: + b = np.random.randn(m) + A = np.random.randn(m, n) + lmbda_max = 2 * LA.norm(A.T @ b, np.inf) + lmbda = factor * lmbda_max + + x = cp.Variable((n, ), name='x') + obj = cp.sum(cp.square((A @ x - b))) + lmbda * cp.sum(cp.abs(x)) + problem = cp.Problem(cp.Minimize(obj)) + problem.solve(solver=cp.CLARABEL) + obj_star_dcp = obj.value + + x = cp.Variable((n, ), name='x') + obj = cp.sum(cp.square((A @ x - b))) + lmbda * cp.sum(cp.abs(x)) + problem = cp.Problem(cp.Minimize(obj)) + problem.solve(solver=cp.IPOPT, nlp=True, hessian_approximation='exact', + derivative_test='none', verbose=False) + obj_star_nlp = obj.value + assert(np.abs(obj_star_nlp - obj_star_dcp) / obj_star_nlp <= 1e-4) + + checker = DerivativeChecker(problem) + checker.run_and_assert() + + + def test_lasso_square(self): + np.random.seed(0) + m, n = 50, 50 + factors = np.linspace(0.1, 1, 20) + + for factor in factors: + b = np.random.randn(m) + A = np.random.randn(m, n) + lmbda_max = 2 * LA.norm(A.T @ b, np.inf) + lmbda = factor * lmbda_max + + x = cp.Variable((n, ), name='x') + obj = cp.sum(cp.square((A @ x - b))) + lmbda * cp.sum(cp.abs(x)) + problem = cp.Problem(cp.Minimize(obj)) + problem.solve(solver=cp.CLARABEL) + obj_star_dcp = obj.value + + x = cp.Variable((n, ), name='x') + obj = cp.sum(cp.square((A @ x - b))) + lmbda * cp.sum(cp.abs(x)) + problem = cp.Problem(cp.Minimize(obj)) + problem.solve(solver=cp.IPOPT, nlp=True, hessian_approximation='exact', + derivative_test='none', verbose=False) + obj_star_nlp = obj.value + assert(np.abs(obj_star_nlp - obj_star_dcp) / obj_star_nlp <= 1e-4) + + + + def test_lasso_underdetermined(self): + np.random.seed(0) + m, n = 100, 200 + factors = np.linspace(0.1, 1, 20) + + for factor in factors: + b = np.random.randn(m) + A = np.random.randn(m, n) + lmbda_max = 2 * LA.norm(A.T @ b, np.inf) + lmbda = factor * lmbda_max + + x = cp.Variable((n, ), name='x') + obj = cp.sum(cp.square((A @ x - b))) + lmbda * cp.sum(cp.abs(x)) + problem = cp.Problem(cp.Minimize(obj)) + problem.solve(solver=cp.CLARABEL) + obj_star_dcp = obj.value + + x = cp.Variable((n, ), name='x') + obj = cp.sum(cp.square((A @ x - b))) + lmbda * cp.sum(cp.abs(x)) + problem = cp.Problem(cp.Minimize(obj)) + problem.solve(solver=cp.IPOPT, nlp=True, hessian_approximation='exact', + derivative_test='none', verbose=False) + obj_star_nlp = obj.value + assert(np.abs(obj_star_nlp - obj_star_dcp) / obj_star_nlp <= 1e-4) + + + + def test_lasso_overdetermined(self): + np.random.seed(0) + m, n = 200, 100 + factors = np.linspace(0.1, 1, 20) + + for factor in factors: + b = np.random.randn(m) + A = np.random.randn(m, n) + lmbda_max = 2 * LA.norm(A.T @ b, np.inf) + lmbda = factor * lmbda_max + + x = cp.Variable((n, ), name='x') + obj = cp.sum(cp.square((A @ x - b))) + lmbda * cp.sum(cp.abs(x)) + problem = cp.Problem(cp.Minimize(obj)) + problem.solve(solver=cp.CLARABEL) + obj_star_dcp = obj.value + + x = cp.Variable((n, ), name='x') + obj = cp.sum(cp.square((A @ x - b))) + lmbda * cp.sum(cp.abs(x)) + problem = cp.Problem(cp.Minimize(obj)) + problem.solve(solver=cp.IPOPT, nlp=True, hessian_approximation='exact', + derivative_test='none', verbose=False) + obj_star_nlp = obj.value + assert(np.abs(obj_star_nlp - obj_star_dcp) / obj_star_nlp <= 1e-4) diff --git a/cvxpy/tests/nlp_tests/test_best_of.py b/cvxpy/tests/nlp_tests/test_best_of.py new file mode 100644 index 0000000000..70d80d0739 --- /dev/null +++ b/cvxpy/tests/nlp_tests/test_best_of.py @@ -0,0 +1,99 @@ +import numpy as np +import pytest + +import cvxpy as cp +from cvxpy.reductions.solvers.defines import INSTALLED_SOLVERS + + +@pytest.mark.skipif('IPOPT' not in INSTALLED_SOLVERS, reason='IPOPT is not installed.') +class TestBestOf(): + + def test_circle_packing_best_of_one(self): + np.random.seed(0) + rng = np.random.default_rng(5) + n = 5 + radius = rng.uniform(1.0, 3.0, n) + centers = cp.Variable((n, 2), name='c') + constraints = [] + for i in range(n - 1): + constraints += [cp.sum((centers[i, :] - centers[i+1:, :]) ** 2, axis=1) >= + (radius[i] + radius[i+1:]) ** 2] + obj = cp.Minimize(cp.max(cp.norm_inf(centers, axis=1) + radius)) + prob = cp.Problem(obj, constraints) + centers.sample_bounds = [-5.0, 5.0] + n_runs = 10 + prob.solve(nlp=True, verbose=True, derivative_test='none', best_of=n_runs) + obj_best_of = obj.value + best_centers = centers.value + all_objs = prob.solver_stats.extra_stats['all_objs_from_best_of'] + + assert len(all_objs) == n_runs + manual_obj = np.max(np.linalg.norm(best_centers, ord=np.inf, axis=1) + radius) + assert manual_obj == obj_best_of + assert manual_obj == np.min(all_objs) + + def test_path_planning_best_of_two(self): + # test that if sample bounds and the value of the variables are set, + # then best_of still initializes randomly within the bounds + np.random.seed(0) + rng = np.random.default_rng(5) + n = 5 + radius = rng.uniform(1.0, 3.0, n) + centers = cp.Variable((n, 2), name='c') + constraints = [] + for i in range(n - 1): + constraints += [cp.sum((centers[i, :] - centers[i+1:, :]) ** 2, axis=1) >= + (radius[i] + radius[i+1:]) ** 2] + obj = cp.Minimize(cp.max(cp.norm_inf(centers, axis=1) + radius)) + prob = cp.Problem(obj, constraints) + + centers.value = np.random.rand(n, 2) + centers.sample_bounds = [-5.0, 5.0] + n_runs = 10 + prob.solve(nlp=True, verbose=True, derivative_test='none', best_of=n_runs) + obj_best_of = obj.value + best_centers = centers.value + all_objs = prob.solver_stats.extra_stats['all_objs_from_best_of'] + _, counts = np.unique(all_objs, return_counts=True) + + assert np.max(counts) == 1 + assert len(all_objs) == n_runs + manual_obj = np.max(np.linalg.norm(best_centers, ord=np.inf, axis=1) + radius) + assert manual_obj == obj_best_of + assert manual_obj == np.min(all_objs) + + def test_path_planning_best_of_three(self): + # test that no error is raised when best_of > 1 and all variables have finite bounds + x = cp.Variable(bounds=[-5, 5]) + y = cp.Variable(bounds=[-3, 3]) + obj = cp.Minimize((x - 1) ** 2 + (y - 2) ** 2) + prob = cp.Problem(obj) + prob.solve(nlp=True, best_of=3) + + all_objs = prob.solver_stats.extra_stats['all_objs_from_best_of'] + assert len(all_objs) == 3 + + def test_path_planning_best_of_four(self): + # test that an error is raised it there is a variable with one + # infinite bound and no sample_bounds when best_of > 1 + x = cp.Variable(bounds=[-5, 5]) + y = cp.Variable(bounds=[-3, None]) + obj = cp.Minimize((x - 1) ** 2 + (y - 2) ** 2) + prob = cp.Problem(obj) + + # test that it raises an error + with pytest.raises(ValueError): + prob.solve(nlp=True, best_of=3) + + def test_path_planning_best_of_five(self): + # test that no error is raised it there is a variable with + # no bounds and no sample bounds, but it has been assigned + # a value + x = cp.Variable(bounds=[-5, 5]) + y = cp.Variable() + y.value = 5 + obj = cp.Minimize((x - 1) ** 2 + (y - 2) ** 2) + prob = cp.Problem(obj) + prob.solve(nlp=True, best_of=3) + all_objs = prob.solver_stats.extra_stats['all_objs_from_best_of'] + assert len(all_objs) == 3 \ No newline at end of file diff --git a/cvxpy/tests/nlp_tests/test_broadcast.py b/cvxpy/tests/nlp_tests/test_broadcast.py new file mode 100644 index 0000000000..285735ea49 --- /dev/null +++ b/cvxpy/tests/nlp_tests/test_broadcast.py @@ -0,0 +1,79 @@ +import numpy as np +import pytest + +import cvxpy as cp +from cvxpy.reductions.solvers.defines import INSTALLED_SOLVERS +from cvxpy.reductions.solvers.nlp_solvers.nlp_solver import DerivativeChecker + + +@pytest.mark.skipif('IPOPT' not in INSTALLED_SOLVERS, reason='IPOPT is not installed.') +class TestBroadcast(): + + def test_scalar_to_matrix(self): + np.random.seed(0) + x = cp.Variable(name='x') + A = np.random.randn(200, 6) + obj = cp.sum(cp.square(x - A)) + problem = cp.Problem(cp.Minimize(obj)) + + problem.solve(solver=cp.IPOPT, nlp=True, hessian_approximation='exact', + derivative_test='none', verbose=True) + assert(problem.status == cp.OPTIMAL) + assert(np.allclose(x.value, np.mean(A))) + + checker = DerivativeChecker(problem) + checker.run_and_assert() + + + def test_row_broadcast(self): + np.random.seed(0) + x = cp.Variable(6, name='x') + A = np.random.randn(5, 6) + obj = cp.sum(cp.square(x - A)) + problem = cp.Problem(cp.Minimize(obj)) + + problem.solve(solver=cp.IPOPT, nlp=True, hessian_approximation='exact', + derivative_test='none', verbose=True) + assert(problem.status == cp.OPTIMAL) + assert(np.allclose(x.value, np.mean(A, axis=0))) + + checker = DerivativeChecker(problem) + checker.run_and_assert() + + + def test_column_broadcast(self): + np.random.seed(0) + x = cp.Variable((5, 1), name='x') + A = np.random.randn(5, 6) + obj = cp.sum(cp.square(x - A)) + problem = cp.Problem(cp.Minimize(obj)) + + problem.solve(solver=cp.IPOPT, nlp=True, hessian_approximation='exact', + derivative_test='none', verbose=True) + assert(problem.status == cp.OPTIMAL) + assert(np.allclose(x.value.flatten(), np.mean(A, axis=1))) + + checker = DerivativeChecker(problem) + checker.run_and_assert() + + def test_subtle_broadcast1(self): + n = 5 + x = cp.Variable((n, 1)) + b = np.ones(n) + constraints = [cp.log(x) == b] + x.value = np.random.rand(n, 1) + 0.1 + + prob = cp.Problem(cp.Minimize(0), constraints) + checker = DerivativeChecker(prob) + checker.run_and_assert() + + def test_subtle_broadcast2(self): + n = 5 + x = cp.Variable((n, 1)) + b = np.ones((1, n)) + constraints = [cp.log(x) == b] + x.value = np.random.rand(n, 1) + 0.1 + + prob = cp.Problem(cp.Minimize(0), constraints) + checker = DerivativeChecker(prob) + checker.run_and_assert() diff --git a/cvxpy/tests/nlp_tests/test_dnlp.py b/cvxpy/tests/nlp_tests/test_dnlp.py new file mode 100644 index 0000000000..a627625c16 --- /dev/null +++ b/cvxpy/tests/nlp_tests/test_dnlp.py @@ -0,0 +1,125 @@ +import pytest + +import cvxpy as cp +from cvxpy import error + + +class TestDNLP(): + """ + This class tests whether problems are correctly identified as DNLP + (disciplined nonlinear programs) and whether the objective and constraints + are correctly identified as smooth, esr or hsr. + + We adopt the convention that a function is smooth if and only if it is + both esr and hsr. This convention is analogous to DCP and convex programming + where a function is affine if and only if it is both convex and concave. + """ + + def test_simple_smooth(self): + # Define a simple nonlinear program + x = cp.Variable() + y = cp.Variable() + objective = cp.Minimize(cp.log(x - 1) + cp.exp(y - 2)) + constraints = [x + y == 1] + prob = cp.Problem(objective, constraints) + + assert prob.is_dnlp() + assert prob.objective.expr.is_smooth() + assert prob.constraints[0].expr.is_smooth() + + def test_abs_esr(self): + x = cp.Variable() + objective = cp.Minimize(cp.abs(x)) + prob = cp.Problem(objective) + assert objective.expr.is_esr() + assert prob.is_dnlp() + + def test_sqrt_hsr(self): + x = cp.Variable() + objective = cp.Maximize(cp.sqrt(x)) + prob = cp.Problem(objective) + assert objective.expr.is_hsr() + assert prob.is_dnlp() + + def test_simple_neg_expr(self): + x = cp.Variable() + y = cp.Variable() + constraints = [cp.abs(x) - cp.sqrt(y) <= 5] + assert constraints[0].is_dnlp() + assert constraints[0].expr.is_esr() + + def test_non_dnlp(self): + """ + The constraint abs(x) >= 5 is hsr but makes + the problem non-DNLP. + """ + x = cp.Variable() + constraints = [cp.abs(x) >= 5] + # the expression is 5 + -abs(x) + assert constraints[0].expr.is_hsr() + assert not constraints[0].is_dnlp() + + def test_simple_composition(self): + x = cp.Variable() + obj1 = cp.Minimize(cp.log(cp.abs(x))) + assert obj1.is_dnlp() + + obj2 = cp.Minimize(cp.exp(cp.norm1(x))) + assert obj2.is_dnlp() + + expr = cp.sqrt(cp.abs(x)) + # we treat sqrt as smooth + assert expr.is_dnlp() + + def test_complicated_composition(self): + x = cp.Variable() + y = cp.Variable() + expr = cp.minimum(cp.sqrt(cp.exp(x)), -cp.abs(y)) + assert expr.is_hsr() + + # cannot minimize an hsr function + obj = cp.Minimize(expr) + prob = cp.Problem(obj) + assert not prob.is_dnlp() + + +class TestNonDNLP: + + def test_max(self): + x = cp.Variable(1) + y = cp.Variable(1) + + objective = cp.Maximize(cp.maximum(x, y)) + + constraints = [x - 14 == 0, y - 6 == 0] + + # assert raises DNLP error + problem = cp.Problem(objective, constraints) + with pytest.raises(error.DNLPError): + problem.solve(solver=cp.IPOPT, nlp=True) + + def test_min(self): + x = cp.Variable(1) + y = cp.Variable(1) + + objective = cp.Minimize(cp.minimum(x, y)) + + constraints = [x - 14 == 0, y - 6 == 0] + + problem = cp.Problem(objective, constraints) + with pytest.raises(error.DNLPError): + problem.solve(solver=cp.IPOPT, nlp=True) + + def test_max_2(self): + # Define variables + x = cp.Variable(3) + y = cp.Variable(3) + + objective = cp.Maximize(cp.sum(cp.maximum(x, y))) + + constraints = [x <= 14, y <= 14] + + problem = cp.Problem(objective, constraints) + with pytest.raises(error.DNLPError): + problem.solve(solver=cp.IPOPT, nlp=True) + diff --git a/cvxpy/tests/nlp_tests/test_entropy_related.py b/cvxpy/tests/nlp_tests/test_entropy_related.py new file mode 100644 index 0000000000..998655fda1 --- /dev/null +++ b/cvxpy/tests/nlp_tests/test_entropy_related.py @@ -0,0 +1,171 @@ +import numpy as np +import numpy.linalg as LA +import pytest + +import cvxpy as cp +from cvxpy.reductions.solvers.defines import INSTALLED_SOLVERS +from cvxpy.reductions.solvers.nlp_solvers.nlp_solver import DerivativeChecker + + +@pytest.mark.skipif('IPOPT' not in INSTALLED_SOLVERS, reason='IPOPT is not installed.') +class TestEntropy(): + + # convex problem, standard entropy f(x) = - x log x. + def test_entropy_one(self): + np.random.seed(0) + n = 100 + q = cp.Variable(n, nonneg=True) + A = np.random.rand(n, n) + obj = cp.sum(cp.entr(A @ q)) + constraints = [cp.sum(q) == 1] + problem = cp.Problem(cp.Maximize(obj), constraints) + problem.solve(solver=cp.IPOPT, nlp=True, verbose=True, derivative_test='none') + q_opt_nlp = q.value + problem.solve(solver=cp.CLARABEL, verbose=True) + q_opt_clarabel = q.value + assert(LA.norm(q_opt_nlp - q_opt_clarabel) <= 1e-4) + + checker = DerivativeChecker(problem) + checker.run_and_assert() + + # nonconvex problem, compute minimum entropy distribution + # over simplex (the analytical solution is any of the vertices) + def test_entropy_two(self): + np.random.seed(0) + n = 10 + q = cp.Variable((n, ), nonneg=True) + q.value = np.random.rand(n) + q.value = q.value / np.sum(q.value) + obj = cp.sum(cp.entr(q)) + constraints = [cp.sum(q) == 1] + problem = cp.Problem(cp.Minimize(obj), constraints) + problem.solve(solver=cp.IPOPT, nlp=True, verbose=True, + hessian_approximation='limited-memory') + q_opt_nlp = q.value + assert(np.sum(q_opt_nlp > 1e-8) == 1) + + checker = DerivativeChecker(problem) + checker.run_and_assert() + + # convex formulation, relative entropy f(x, y) = x log (x / y) + def test_rel_entropy_one(self): + np.random.seed(0) + n = 40 + p = np.random.rand(n, ) + p = p / np.sum(p) + q = cp.Variable(n, nonneg=True) + A = np.random.rand(n, n) + obj = cp.sum(cp.rel_entr(A @ q, p)) + constraints = [cp.sum(q) == 1] + problem = cp.Problem(cp.Minimize(obj), constraints) + problem.solve(solver=cp.IPOPT, nlp=True, verbose=True, derivative_test='none') + q_opt_nlp = q.value + problem.solve(solver=cp.CLARABEL, verbose=True) + q_opt_clarabel = q.value + assert(LA.norm(q_opt_nlp - q_opt_clarabel) <= 1e-4) + + checker = DerivativeChecker(problem) + checker.run_and_assert() + + def test_rel_entropy_one_switched_arguments(self): + np.random.seed(0) + n = 40 + p = np.random.rand(n, ) + p = p / np.sum(p) + q = cp.Variable(n, nonneg=True) + A = np.random.rand(n, n) + obj = cp.sum(cp.rel_entr(p, A @ q)) + constraints = [cp.sum(q) == 1] + problem = cp.Problem(cp.Minimize(obj), constraints) + problem.solve(solver=cp.IPOPT, nlp=True, verbose=True, derivative_test='none') + q_opt_nlp = q.value + problem.solve(solver=cp.CLARABEL, verbose=True) + q_opt_clarabel = q.value + assert(LA.norm(q_opt_nlp - q_opt_clarabel) <= 1e-4) + + checker = DerivativeChecker(problem) + checker.run_and_assert() + + def test_KL_one(self): + np.random.seed(0) + n = 40 + p = np.random.rand(n, ) + p = p / np.sum(p) + q = cp.Variable(n, nonneg=True) + A = np.random.rand(n, n) + obj = cp.sum(cp.kl_div(A @ q, p)) + constraints = [cp.sum(q) == 1] + problem = cp.Problem(cp.Minimize(obj), constraints) + problem.solve(solver=cp.IPOPT, nlp=True, verbose=True, derivative_test='none') + q_opt_nlp = q.value + problem.solve(solver=cp.CLARABEL, verbose=True) + q_opt_clarabel = q.value + assert(LA.norm(q_opt_nlp - q_opt_clarabel) <= 1e-4) + + checker = DerivativeChecker(problem) + checker.run_and_assert() + + def test_KL_two(self): + np.random.seed(0) + n = 40 + p = np.random.rand(n, ) + p = p / np.sum(p) + q = cp.Variable(n, nonneg=True) + A = np.random.rand(n, n) + obj = cp.sum(cp.kl_div(p, A @ q)) + constraints = [cp.sum(q) == 1] + problem = cp.Problem(cp.Minimize(obj), constraints) + problem.solve(solver=cp.IPOPT, nlp=True, verbose=True, derivative_test='none') + q_opt_nlp = q.value + problem.solve(solver=cp.CLARABEL, verbose=True) + q_opt_clarabel = q.value + assert(LA.norm(q_opt_nlp - q_opt_clarabel) <= 1e-4) + + checker = DerivativeChecker(problem) + checker.run_and_assert() + + # nonnegative matrix factorization with KL objective (nonconvex) + def test_KL_three_graph_form(self): + np.random.seed(0) + n, m, k = 40, 20, 4 + X_true = np.random.rand(n, k) + Y_true = np.random.rand(k, m) + A = X_true @ Y_true + A = np.clip(A, 0, None) + X = cp.Variable((n, k), bounds=[0, None]) + Y = cp.Variable((k, m), bounds=[0, None]) + # without random initialization we converge to a very structured + # point that is not the global minimizer + X.value = np.random.rand(n, k) + Y.value = np.random.rand(k, m) + + # graph form + T = cp.Variable((n, m)) + obj = cp.sum(cp.kl_div(A, T)) + constraints = [T == X @ Y] + problem = cp.Problem(cp.Minimize(obj), constraints) + problem.solve(solver=cp.IPOPT, nlp=True, verbose=False, derivative_test='none') + assert(obj.value <= 1e-10) + + checker = DerivativeChecker(problem) + checker.run_and_assert() + + # nonnegative matrix factorization with KL objective (nonconvex) + def test_KL_three_not_graph_form(self): + np.random.seed(0) + n, m, k = 40, 20, 4 + X_true = np.random.rand(n, k) + Y_true = np.random.rand(k, m) + A = X_true @ Y_true + A = np.clip(A, 0, None) + X = cp.Variable((n, k), bounds=[0, None]) + Y = cp.Variable((k, m), bounds=[0, None]) + X.value = np.random.rand(n, k) + Y.value = np.random.rand(k, m) + obj = cp.sum(cp.kl_div(A, X @ Y)) + problem = cp.Problem(cp.Minimize(obj)) + problem.solve(solver=cp.IPOPT, nlp=True, verbose=False, derivative_test='none') + assert(obj.value <= 1e-10) + + checker = DerivativeChecker(problem) + checker.run_and_assert() \ No newline at end of file diff --git a/cvxpy/tests/nlp_tests/test_fast_params.py b/cvxpy/tests/nlp_tests/test_fast_params.py new file mode 100644 index 0000000000..add595bd76 --- /dev/null +++ b/cvxpy/tests/nlp_tests/test_fast_params.py @@ -0,0 +1,660 @@ +"""Tests for fast parameter support in the diff engine. + +Verifies that CVXPY Parameters are correctly handled as live nodes in the +C expression tree, allowing parameter updates without tree rebuilding. + +All tests use **affine** parameter expressions only — matching what DCP2Cone +produces. After canonicalization the diff engine sees c^T x objectives and +Ax + b constraints where c, A, b depend on parameters. + +Copyright 2025, the CVXPY developers + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import numpy as np +import pytest +from scipy import sparse as sp + +import cvxpy as cp +from cvxpy.reductions.solvers.defines import INSTALLED_SOLVERS +from cvxpy.reductions.solvers.nlp_solvers.diff_engine import C_problem + + +@pytest.mark.skipif('IPOPT' not in INSTALLED_SOLVERS, reason='IPOPT is not installed.') +class TestFastParams: + + # ------------------------------------------------------------------ + # Parameter node creation and forward pass + # ------------------------------------------------------------------ + def test_parameter_node_forward(self): + """Parameter node value updates correctly via update_params.""" + P = cp.Parameter((2, 2)) + x = cp.Variable(2, bounds=[-10, 10]) + P.value = np.eye(2) + x.value = np.array([1.0, 2.0]) + + prob = cp.Problem(cp.Minimize(cp.sum(P @ x))) + c_prob = C_problem(prob, verbose=False) + c_prob.init_jacobian() + c_prob.init_hessian() + c_prob.update_params() + + u = np.array([1.0, 2.0]) + val = c_prob.objective_forward(u) + np.testing.assert_allclose(val, 3.0) # sum(I @ [1,2]) = 3 + + # Update parameter to 2*I + P.value = 2 * np.eye(2) + c_prob.update_params() + val = c_prob.objective_forward(u) + np.testing.assert_allclose(val, 6.0) # sum(2I @ [1,2]) = 6 + + # ------------------------------------------------------------------ + # P @ x — Parameter matrix × variable vector + # ------------------------------------------------------------------ + def test_param_matmul_gradient(self): + """P @ x: gradient = P.T @ 1 for sum objective.""" + P = cp.Parameter((3, 3)) + x = cp.Variable(3, bounds=[-10, 10]) + + P.value = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]], dtype=float) + x.value = np.ones(3) + + prob = cp.Problem(cp.Minimize(cp.sum(P @ x))) + c_prob = C_problem(prob, verbose=False) + c_prob.init_jacobian() + c_prob.init_hessian() + c_prob.update_params() + + u = np.array([1.0, 2.0, 3.0]) + c_prob.objective_forward(u) + grad = c_prob.gradient() + + expected = P.value.T @ np.ones(3) # [12, 15, 18] + np.testing.assert_allclose(grad, expected, atol=1e-10) + + def test_param_matmul_update(self): + """After updating P, gradient changes without tree rebuild.""" + P = cp.Parameter((3, 3)) + x = cp.Variable(3, bounds=[-10, 10]) + x.value = np.ones(3) + + # First: P = I + P.value = np.eye(3) + prob = cp.Problem(cp.Minimize(cp.sum(P @ x))) + c_prob = C_problem(prob, verbose=False) + c_prob.init_jacobian() + c_prob.init_hessian() + c_prob.update_params() + + u = np.array([1.0, 2.0, 3.0]) + c_prob.objective_forward(u) + grad1 = c_prob.gradient() + np.testing.assert_allclose(grad1, [1.0, 1.0, 1.0], atol=1e-10) + + # Update P to [[1,2,3],[4,5,6],[7,8,9]] + P.value = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]], dtype=float) + c_prob.update_params() # NO rebuild + c_prob.objective_forward(u) + grad2 = c_prob.gradient() + expected = P.value.T @ np.ones(3) + np.testing.assert_allclose(grad2, expected, atol=1e-10) + + def test_param_matmul_finite_diff(self): + """P @ x: verify gradient with central differences.""" + np.random.seed(42) + P = cp.Parameter((3, 4)) + x = cp.Variable(4, bounds=[-10, 10]) + + P.value = np.random.randn(3, 4) + x.value = np.random.randn(4) + + prob = cp.Problem(cp.Minimize(cp.sum(P @ x))) + c_prob = C_problem(prob, verbose=False) + c_prob.init_jacobian() + c_prob.init_hessian() + c_prob.update_params() + + u = np.random.randn(4) + c_prob.objective_forward(u) + grad = c_prob.gradient() + + eps = 1e-7 + for j in range(4): + u_p, u_m = u.copy(), u.copy() + u_p[j] += eps + u_m[j] -= eps + numerical = (c_prob.objective_forward(u_p) - c_prob.objective_forward(u_m)) / (2 * eps) + np.testing.assert_allclose(grad[j], numerical, atol=1e-5, + err_msg=f"Gradient mismatch at index {j}") + + # ------------------------------------------------------------------ + # gamma * expr — Scalar parameter × affine expression + # ------------------------------------------------------------------ + def test_param_scalar_mult_gradient(self): + """gamma * sum(x): gradient = gamma * 1.""" + gamma = cp.Parameter(nonneg=True) + x = cp.Variable(4, bounds=[-10, 10]) + + gamma.value = 2.0 + x.value = np.array([1.0, 2.0, 3.0, 4.0]) + + prob = cp.Problem(cp.Minimize(gamma * cp.sum(x))) + c_prob = C_problem(prob, verbose=False) + c_prob.init_jacobian() + c_prob.init_hessian() + c_prob.update_params() + + u = np.array([1.0, 2.0, 3.0, 4.0]) + c_prob.objective_forward(u) + grad = c_prob.gradient() + np.testing.assert_allclose(grad, 2.0 * np.ones(4), atol=1e-10) + + # Update gamma + gamma.value = 5.0 + c_prob.update_params() + c_prob.objective_forward(u) + grad = c_prob.gradient() + np.testing.assert_allclose(grad, 5.0 * np.ones(4), atol=1e-10) + + def test_param_scalar_mult_hessian(self): + """gamma * sum(x): Hessian = 0 (affine objective).""" + gamma = cp.Parameter(nonneg=True) + x = cp.Variable(4, bounds=[-10, 10]) + + gamma.value = 5.0 + x.value = np.ones(4) + + prob = cp.Problem(cp.Minimize(gamma * cp.sum(x))) + c_prob = C_problem(prob, verbose=False) + c_prob.init_jacobian() + c_prob.init_hessian() + c_prob.update_params() + + u = np.array([1.0, 2.0, 3.0, 4.0]) + c_prob.objective_forward(u) + c_prob.constraint_forward(u) + hess = c_prob.hessian(1.0, np.array([])) + np.testing.assert_allclose(hess.toarray(), np.zeros((4, 4)), atol=1e-10) + + # ------------------------------------------------------------------ + # p ∘ x — Vector parameter × variable + # ------------------------------------------------------------------ + def test_param_vector_mult(self): + """p ∘ x: gradient = p for sum(p * x).""" + p = cp.Parameter(4, pos=True) + x = cp.Variable(4, bounds=[-10, 10]) + + p.value = np.array([1.0, 2.0, 3.0, 4.0]) + x.value = np.ones(4) + + prob = cp.Problem(cp.Minimize(cp.sum(cp.multiply(p, x)))) + c_prob = C_problem(prob, verbose=False) + c_prob.init_jacobian() + c_prob.init_hessian() + c_prob.update_params() + + u = np.array([1.0, 1.0, 1.0, 1.0]) + c_prob.objective_forward(u) + grad = c_prob.gradient() + np.testing.assert_allclose(grad, p.value, atol=1e-10) + + # Update p + p.value = np.array([10.0, 20.0, 30.0, 40.0]) + c_prob.update_params() + c_prob.objective_forward(u) + grad = c_prob.gradient() + np.testing.assert_allclose(grad, p.value, atol=1e-10) + + # ------------------------------------------------------------------ + # Affine parametric objective: sum(A @ x - b) + # ------------------------------------------------------------------ + def test_param_affine_objective(self): + """sum(A @ x - b): gradient = A.T @ 1, Hessian = 0.""" + np.random.seed(0) + n = 3 + A = cp.Parameter((n, n)) + b = cp.Parameter(n) + x = cp.Variable(n, bounds=[-10, 10]) + + A.value = np.random.randn(n, n) + b.value = np.random.randn(n) + x.value = np.random.randn(n) + + prob = cp.Problem(cp.Minimize(cp.sum(A @ x - b))) + c_prob = C_problem(prob, verbose=False) + c_prob.init_jacobian() + c_prob.init_hessian() + c_prob.update_params() + + u = np.random.randn(n) + c_prob.objective_forward(u) + grad = c_prob.gradient() + + # Analytical: gradient of sum(A @ x - b) w.r.t. x = A.T @ 1 + Av = A.value + expected_grad = Av.T @ np.ones(n) + np.testing.assert_allclose(grad, expected_grad, atol=1e-8) + + # Hessian = 0 (affine) + c_prob.constraint_forward(u) + hess = c_prob.hessian(1.0, np.array([])) + np.testing.assert_allclose(hess.toarray(), np.zeros((n, n)), atol=1e-8) + + # Update A and b, check again + A.value = np.eye(n) * 3 + b.value = np.ones(n) * 2 + c_prob.update_params() + c_prob.objective_forward(u) + grad2 = c_prob.gradient() + expected_grad2 = (3 * np.eye(n)).T @ np.ones(n) + np.testing.assert_allclose(grad2, expected_grad2, atol=1e-8) + + # ------------------------------------------------------------------ + # Finite difference checks for gradient/Jacobian/Hessian + # ------------------------------------------------------------------ + def test_param_finite_diff_gradient(self): + """Central differences for objective gradient with parameters.""" + np.random.seed(123) + gamma = cp.Parameter(nonneg=True) + P = cp.Parameter((3, 3)) + x = cp.Variable(3, bounds=[-5, 5]) + + gamma.value = 2.5 + P.value = np.random.randn(3, 3) + x.value = np.random.randn(3) + + prob = cp.Problem(cp.Minimize(gamma * cp.sum(P @ x))) + c_prob = C_problem(prob, verbose=False) + c_prob.init_jacobian() + c_prob.init_hessian() + c_prob.update_params() + + u = np.random.randn(3) * 0.5 + c_prob.objective_forward(u) + grad = c_prob.gradient() + + eps = 1e-7 + for j in range(3): + u_p, u_m = u.copy(), u.copy() + u_p[j] += eps + u_m[j] -= eps + numerical = (c_prob.objective_forward(u_p) - c_prob.objective_forward(u_m)) / (2 * eps) + np.testing.assert_allclose(grad[j], numerical, atol=1e-5, + err_msg=f"Gradient mismatch at index {j}") + + def test_param_finite_diff_jacobian(self): + """Central differences for constraint Jacobian with parameters.""" + np.random.seed(456) + A = cp.Parameter((3, 4)) + b = cp.Parameter(3) + x = cp.Variable(4, bounds=[-5, 5]) + + A.value = np.random.randn(3, 4) + b.value = np.random.randn(3) + x.value = np.random.randn(4) + + prob = cp.Problem(cp.Minimize(0), [A @ x - b <= 0]) + c_prob = C_problem(prob, verbose=False) + c_prob.init_jacobian() + c_prob.init_hessian() + c_prob.update_params() + + u = np.random.randn(4) * 0.5 + c_prob.constraint_forward(u) + jac = c_prob.jacobian().toarray() + + eps = 1e-7 + n_vars = 4 + n_constraints = 3 + numerical_jac = np.zeros((n_constraints, n_vars)) + for j in range(n_vars): + u_p, u_m = u.copy(), u.copy() + u_p[j] += eps + u_m[j] -= eps + c_p = c_prob.constraint_forward(u_p) + c_m = c_prob.constraint_forward(u_m) + numerical_jac[:, j] = (c_p - c_m) / (2 * eps) + + np.testing.assert_allclose(jac, numerical_jac, atol=1e-5) + + def test_param_finite_diff_hessian(self): + """Central differences for Lagrangian Hessian with parameters (affine = 0).""" + np.random.seed(789) + gamma = cp.Parameter(nonneg=True) + x = cp.Variable(3, bounds=[-5, 5]) + + gamma.value = 3.0 + x.value = np.random.randn(3) + + prob = cp.Problem(cp.Minimize(gamma * cp.sum(x))) + c_prob = C_problem(prob, verbose=False) + c_prob.init_jacobian() + c_prob.init_hessian() + c_prob.update_params() + + u = np.random.randn(3) * 0.5 + c_prob.objective_forward(u) + c_prob.constraint_forward(u) + hess = c_prob.hessian(1.0, np.array([])).toarray() + + # Affine objective → Hessian is zero + np.testing.assert_allclose(hess, np.zeros((3, 3)), atol=1e-10) + + # Also verify via finite differences of the gradient + eps = 1e-5 + n_vars = 3 + numerical_hess = np.zeros((n_vars, n_vars)) + for j in range(n_vars): + u_p, u_m = u.copy(), u.copy() + u_p[j] += eps + u_m[j] -= eps + c_prob.objective_forward(u_p) + grad_p = c_prob.gradient().copy() + c_prob.objective_forward(u_m) + grad_m = c_prob.gradient().copy() + numerical_hess[:, j] = (grad_p - grad_m) / (2 * eps) + + np.testing.assert_allclose(hess, numerical_hess, atol=1e-4) + + # ------------------------------------------------------------------ + # No-rebuild performance: update_params doesn't require re-init + # ------------------------------------------------------------------ + def test_no_rebuild(self): + """Verify update_params + re-eval works without init_jacobian/init_hessian.""" + gamma = cp.Parameter(nonneg=True) + x = cp.Variable(3, bounds=[-10, 10]) + gamma.value = 1.0 + x.value = np.ones(3) + + prob = cp.Problem(cp.Minimize(gamma * cp.sum(x))) + c_prob = C_problem(prob, verbose=False) + c_prob.init_jacobian() + c_prob.init_hessian() + c_prob.update_params() + + u = np.array([1.0, 2.0, 3.0]) + + # First evaluation + c_prob.objective_forward(u) + grad1 = c_prob.gradient() + + # Update parameter — NO re-init of structures + gamma.value = 10.0 + c_prob.update_params() + c_prob.objective_forward(u) + grad2 = c_prob.gradient() + + np.testing.assert_allclose(grad1, np.ones(3), atol=1e-10) + np.testing.assert_allclose(grad2, 10.0 * np.ones(3), atol=1e-10) + + # ------------------------------------------------------------------ + # Finance-style: affine parametric objective + # ------------------------------------------------------------------ + def test_param_finance(self): + """Affine finance-style: gamma * (c.T @ x) - mu @ x.""" + np.random.seed(99) + n = 5 + gamma = cp.Parameter(nonneg=True) + x = cp.Variable(n, bounds=[0, 1]) + + c_vec = np.random.randn(n) + mu = np.random.randn(n) + + gamma.value = 1.0 + x.value = np.ones(n) / n + + obj = gamma * (c_vec @ x) - mu @ x + prob = cp.Problem(cp.Minimize(obj), [cp.sum(x) == 1]) + c_prob = C_problem(prob, verbose=False) + c_prob.init_jacobian() + c_prob.init_hessian() + c_prob.update_params() + + u = np.ones(n) / n + c_prob.objective_forward(u) + grad = c_prob.gradient() + + # Analytical: gradient = gamma * c - mu + expected = gamma.value * c_vec - mu + np.testing.assert_allclose(grad, expected, atol=1e-8) + + # Update gamma, check gradient changes + gamma.value = 5.0 + c_prob.update_params() + c_prob.objective_forward(u) + grad2 = c_prob.gradient() + expected2 = 5.0 * c_vec - mu + np.testing.assert_allclose(grad2, expected2, atol=1e-8) + + # ------------------------------------------------------------------ + # Problem with no parameters (backward compatibility) + # ------------------------------------------------------------------ + def test_no_params_backward_compat(self): + """Problems without parameters still work normally (affine).""" + x = cp.Variable(3, bounds=[-10, 10]) + x.value = np.ones(3) + prob = cp.Problem(cp.Minimize(cp.sum(x))) + c_prob = C_problem(prob, verbose=False) + c_prob.init_jacobian() + c_prob.init_hessian() + + u = np.array([1.0, 2.0, 3.0]) + c_prob.objective_forward(u) + grad = c_prob.gradient() + np.testing.assert_allclose(grad, np.ones(3), atol=1e-10) + + # ------------------------------------------------------------------ + # DerivativeChecker with parameters + # ------------------------------------------------------------------ + def test_derivative_checker_with_params(self): + """Run DerivativeChecker on a parametric affine problem.""" + from cvxpy.reductions.solvers.nlp_solvers.nlp_solver import DerivativeChecker + + np.random.seed(42) + P = cp.Parameter((3, 3)) + x = cp.Variable(3, bounds=[-5, 5]) + + P.value = np.random.randn(3, 3) + x.value = np.random.randn(3) + + prob = cp.Problem(cp.Minimize(cp.sum(P @ x))) + checker = DerivativeChecker(prob) + checker.run_and_assert() + + def test_derivative_checker_scalar_param(self): + """Run DerivativeChecker on gamma * sum(x).""" + from cvxpy.reductions.solvers.nlp_solvers.nlp_solver import DerivativeChecker + + np.random.seed(42) + gamma = cp.Parameter(nonneg=True) + x = cp.Variable(4, bounds=[-5, 5]) + + gamma.value = 3.0 + x.value = np.random.randn(4) + + prob = cp.Problem(cp.Minimize(gamma * cp.sum(x))) + checker = DerivativeChecker(prob) + checker.run_and_assert() + + def test_derivative_checker_vector_param(self): + """Run DerivativeChecker on sum(p * x).""" + from cvxpy.reductions.solvers.nlp_solvers.nlp_solver import DerivativeChecker + + np.random.seed(42) + p = cp.Parameter(4, pos=True) + x = cp.Variable(4, bounds=[-5, 5]) + + p.value = np.abs(np.random.randn(4)) + 0.1 + x.value = np.random.randn(4) + + prob = cp.Problem(cp.Minimize(cp.sum(cp.multiply(p, x)))) + checker = DerivativeChecker(prob) + checker.run_and_assert() + + # ------------------------------------------------------------------ + # Multiple parameter updates + # ------------------------------------------------------------------ + def test_multiple_param_updates(self): + """Multiple update_params calls with different values.""" + gamma = cp.Parameter(nonneg=True) + x = cp.Variable(2, bounds=[-10, 10]) + gamma.value = 1.0 + x.value = np.ones(2) + + prob = cp.Problem(cp.Minimize(gamma * cp.sum(x))) + c_prob = C_problem(prob, verbose=False) + c_prob.init_jacobian() + c_prob.init_hessian() + + u = np.array([1.0, 2.0]) + for gamma_val in [1.0, 2.0, 0.5, 10.0, 0.1]: + gamma.value = gamma_val + c_prob.update_params() + c_prob.objective_forward(u) + grad = c_prob.gradient() + np.testing.assert_allclose(grad, gamma_val * np.ones(2), atol=1e-10, + err_msg=f"Failed for gamma={gamma_val}") + + # ------------------------------------------------------------------ + # Sparse parameters (sparsity kwarg) + # ------------------------------------------------------------------ + def test_sparse_param_matmul_forward(self): + """Sparse P @ x: forward pass with dense sparsity pattern.""" + sparsity = ([0, 0, 1, 1], [0, 1, 0, 1]) + P = cp.Parameter((2, 2), sparsity=sparsity) + x = cp.Variable(2, bounds=[-10, 10]) + + P.value_sparse = sp.coo_array( + np.array([[1.0, 2.0], [3.0, 4.0]]) + ) + x.value = np.array([1.0, 2.0]) + + prob = cp.Problem(cp.Minimize(cp.sum(P @ x))) + c_prob = C_problem(prob, verbose=False) + c_prob.init_jacobian() + c_prob.init_hessian() + + u = np.array([1.0, 2.0]) + val = c_prob.objective_forward(u) + # P @ [1,2] = [5, 11], sum = 16 + np.testing.assert_allclose(val, 16.0) + + def test_sparse_param_matmul_gradient(self): + """Sparse P @ x: gradient = P.T @ 1 for sum objective.""" + sparsity = ([0, 0, 1, 1], [0, 1, 0, 1]) + P = cp.Parameter((2, 2), sparsity=sparsity) + x = cp.Variable(2, bounds=[-10, 10]) + + P.value_sparse = sp.coo_array( + np.array([[1.0, 2.0], [3.0, 4.0]]) + ) + x.value = np.ones(2) + + prob = cp.Problem(cp.Minimize(cp.sum(P @ x))) + c_prob = C_problem(prob, verbose=False) + c_prob.init_jacobian() + c_prob.init_hessian() + + u = np.array([1.0, 2.0]) + c_prob.objective_forward(u) + grad = c_prob.gradient() + + Pv = np.array([[1.0, 2.0], [3.0, 4.0]]) + expected = Pv.T @ np.ones(2) # [4, 6] + np.testing.assert_allclose(grad, expected, atol=1e-10) + + def test_sparse_param_matmul_update(self): + """After updating sparse P, gradient changes without tree rebuild.""" + sparsity = ([0, 0, 1, 1], [0, 1, 0, 1]) + P = cp.Parameter((2, 2), sparsity=sparsity) + x = cp.Variable(2, bounds=[-10, 10]) + x.value = np.ones(2) + + P.value_sparse = sp.coo_array( + ([1.0, 0.0, 0.0, 1.0], ([0, 0, 1, 1], [0, 1, 0, 1])), shape=(2, 2) + ) + prob = cp.Problem(cp.Minimize(cp.sum(P @ x))) + c_prob = C_problem(prob, verbose=False) + c_prob.init_jacobian() + c_prob.init_hessian() + + u = np.array([1.0, 2.0]) + c_prob.objective_forward(u) + grad1 = c_prob.gradient() + np.testing.assert_allclose(grad1, [1.0, 1.0], atol=1e-10) + + # Update P via value_sparse + P.value_sparse = sp.coo_array( + np.array([[1.0, 2.0], [3.0, 4.0]]) + ) + c_prob.update_params() + c_prob.objective_forward(u) + grad2 = c_prob.gradient() + expected = np.array([[1.0, 2.0], [3.0, 4.0]]).T @ np.ones(2) + np.testing.assert_allclose(grad2, expected, atol=1e-10) + + def test_sparse_param_truly_sparse(self): + """Sparse parameter with actual zeros (diagonal pattern).""" + sparsity = ([0, 1, 2], [0, 1, 2]) # diagonal only + P = cp.Parameter((3, 3), sparsity=sparsity) + x = cp.Variable(3, bounds=[-10, 10]) + + P.value_sparse = sp.coo_array( + ([2.0, 3.0, 4.0], ([0, 1, 2], [0, 1, 2])), shape=(3, 3) + ) + x.value = np.ones(3) + + prob = cp.Problem(cp.Minimize(cp.sum(P @ x))) + c_prob = C_problem(prob, verbose=False) + c_prob.init_jacobian() + c_prob.init_hessian() + + u = np.array([1.0, 2.0, 3.0]) + val = c_prob.objective_forward(u) + # diag([2,3,4]) @ [1,2,3] = [2,6,12], sum = 20 + np.testing.assert_allclose(val, 20.0) + + c_prob.objective_forward(u) + grad = c_prob.gradient() + # grad of sum(diag(p) @ x) = p (diagonal values) + np.testing.assert_allclose(grad, [2.0, 3.0, 4.0], atol=1e-10) + + # Update diagonal values + P.value_sparse = sp.coo_array( + ([10.0, 20.0, 30.0], ([0, 1, 2], [0, 1, 2])), shape=(3, 3) + ) + c_prob.update_params() + c_prob.objective_forward(u) + grad2 = c_prob.gradient() + np.testing.assert_allclose(grad2, [10.0, 20.0, 30.0], atol=1e-10) + + def test_sparse_param_derivative_checker(self): + """Run DerivativeChecker on a sparse parametric problem.""" + from cvxpy.reductions.solvers.nlp_solvers.nlp_solver import DerivativeChecker + + np.random.seed(42) + sparsity = ([0, 0, 1, 1, 2, 2], [0, 1, 1, 2, 0, 2]) + P = cp.Parameter((3, 3), sparsity=sparsity) + x = cp.Variable(3, bounds=[-5, 5]) + + vals = np.random.randn(6) + P.value_sparse = sp.coo_array( + (vals, ([0, 0, 1, 1, 2, 2], [0, 1, 1, 2, 0, 2])), shape=(3, 3) + ) + x.value = np.random.randn(3) + + prob = cp.Problem(cp.Minimize(cp.sum(P @ x))) + checker = DerivativeChecker(prob) + checker.run_and_assert() diff --git a/cvxpy/tests/nlp_tests/test_huber_sum_largest.py b/cvxpy/tests/nlp_tests/test_huber_sum_largest.py new file mode 100644 index 0000000000..5f9e6ce256 --- /dev/null +++ b/cvxpy/tests/nlp_tests/test_huber_sum_largest.py @@ -0,0 +1,90 @@ +import numpy as np +import pytest + +import cvxpy as cp +from cvxpy.reductions.solvers.defines import INSTALLED_SOLVERS +from cvxpy.reductions.solvers.nlp_solvers.nlp_solver import DerivativeChecker + + +@pytest.mark.skipif('IPOPT' not in INSTALLED_SOLVERS, reason='IPOPT is not installed.') +class TestNonsmoothNontrivial(): + + # convex optimization, huber regression + def test_huber(self): + np.random.seed(1) + n = 100 + SAMPLES = int(1.5 * n) + beta_true = 5 * np.random.normal(size=(n, 1)) + X = np.random.randn(n, SAMPLES) + Y = np.zeros((SAMPLES, 1)) + v = np.random.normal(size=(SAMPLES, 1)) + TESTS = 5 + p_vals = np.linspace(0, 0.15, num=TESTS) + for idx, p in enumerate(p_vals): + # generate the sign changes. + factor = 2 * np.random.binomial(1, 1 - p, size=(SAMPLES, 1)) - 1 + Y = factor * X.T.dot(beta_true) + v + + # form problem + beta = cp.Variable((n, 1)) + cost = cp.sum(cp.huber(X.T @ beta - Y, 1)) + prob = cp.Problem(cp.Minimize(cost)) + + # solve using NLP solver + prob.solve(nlp=True, solver=cp.IPOPT, verbose=False) + nlp_value = prob.value + + # solve using conic solver + prob.solve() + conic_value = prob.value + assert(np.abs(nlp_value - conic_value) <= 1e-4) + + checker = DerivativeChecker(prob) + checker.run_and_assert() + + # convex optimization, sum largest + def test_sum_largest(self): + x = cp.Variable(5) + w = np.array([0.1, 0.2, 0.3, 0.4, 0.5]) + + # form problem + k = 2 + cost = cp.sum_largest(cp.multiply(x, w), k) + prob = cp.Problem(cp.Minimize(cost), [cp.sum(x) == 1, x >= 0]) + + # solve using NLP solver + prob.solve(nlp=True, solver=cp.IPOPT) + nlp_value = prob.value + + # solve using conic solver + prob.solve() + conic_value = prob.value + + assert(np.abs(nlp_value - conic_value) <= 1e-4) + + checker = DerivativeChecker(prob) + checker.run_and_assert() + + # convex optimization, sum smallest + def test_sum_smallest(self): + x = cp.Variable(5) + w = np.array([0.1, 0.2, 0.3, 0.4, 0.5]) + + # form problem + k = 2 + cost = cp.sum_smallest(cp.multiply(x, w), k) + prob = cp.Problem(cp.Maximize(cost), [cp.sum(x) == 1, x >= 0]) + + # solve using NLP solver + prob.solve(nlp=True, solver=cp.IPOPT) + nlp_value = prob.value + + # solve using conic solver + prob.solve() + conic_value = prob.value + + assert(np.abs(nlp_value - conic_value) <= 1e-4) + + checker = DerivativeChecker(prob) + checker.run_and_assert() + diff --git a/cvxpy/tests/nlp_tests/test_hyperbolic.py b/cvxpy/tests/nlp_tests/test_hyperbolic.py new file mode 100644 index 0000000000..c166f11325 --- /dev/null +++ b/cvxpy/tests/nlp_tests/test_hyperbolic.py @@ -0,0 +1,53 @@ +import pytest + +import cvxpy as cp +from cvxpy.reductions.solvers.defines import INSTALLED_SOLVERS +from cvxpy.reductions.solvers.nlp_solvers.nlp_solver import DerivativeChecker + + +@pytest.mark.skipif('IPOPT' not in INSTALLED_SOLVERS, reason='IPOPT is not installed.') +class TestHyperbolic(): + + def test_sinh(self): + n = 10 + x = cp.Variable(n) + prob = cp.Problem(cp.Minimize(cp.sum(cp.sinh(cp.logistic(x * 2)))), + [x >= 0.1, cp.sum(x) == 10]) + prob.solve(nlp=True, solver=cp.IPOPT) + assert prob.status == cp.OPTIMAL + + checker = DerivativeChecker(prob) + checker.run_and_assert() + + def test_tanh(self): + n = 10 + x = cp.Variable(n) + prob = cp.Problem(cp.Minimize(cp.sum(cp.tanh(cp.logistic(x * 2)))), + [x >= 0.1, cp.sum(x) == 10]) + prob.solve(nlp=True, solver=cp.IPOPT) + assert prob.status == cp.OPTIMAL + + checker = DerivativeChecker(prob) + checker.run_and_assert() + + def test_asinh(self): + n = 10 + x = cp.Variable(n) + prob = cp.Problem(cp.Minimize(cp.sum(cp.asinh(cp.logistic(x * 3)))), + [x >= 0.1, cp.sum(x) == 10]) + prob.solve(nlp=True, solver=cp.IPOPT) + assert prob.status == cp.OPTIMAL + + checker = DerivativeChecker(prob) + checker.run_and_assert() + + def test_atanh(self): + n = 10 + x = cp.Variable(n) + prob = cp.Problem(cp.Minimize(cp.sum(cp.atanh(cp.logistic(x * 0.1)))), + [x >= 0.1, cp.sum(x) == 10]) + prob.solve(nlp=True, solver=cp.IPOPT) + assert prob.status == cp.OPTIMAL + + checker = DerivativeChecker(prob) + checker.run_and_assert() \ No newline at end of file diff --git a/cvxpy/tests/nlp_tests/test_initialization.py b/cvxpy/tests/nlp_tests/test_initialization.py new file mode 100644 index 0000000000..76c515f6a7 --- /dev/null +++ b/cvxpy/tests/nlp_tests/test_initialization.py @@ -0,0 +1,104 @@ +""" +Copyright 2026 CVXPY Developers + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import numpy as np +import pytest +import scipy.sparse as sp + +import cvxpy as cp +from cvxpy.reductions.solvers.defines import INSTALLED_SOLVERS + + +@pytest.mark.skipif('IPOPT' not in INSTALLED_SOLVERS, reason='IPOPT is not installed.') +class TestVariableAttributeInit: + """Tests initialization of variables with attributes.""" + + def test_simple_diag_variable(self): + """Test that diagonal variables work in NLP problems.""" + n = 3 + D = cp.Variable((n, n), diag=True) + prob = cp.Problem(cp.Maximize(cp.sum(cp.log(cp.exp(D))))) + + # Should not crash - tests diag_vec Jacobian and value propagation + prob.solve(solver=cp.IPOPT, nlp=True, max_iter=10) + + def test_diag_variable_value_sparse_init(self): + """Test that diagonal variables with sparse value initialization work. + + Tests the code path in cvx_attr2constr.py where a diag variable has + its value stored as a sparse matrix. + """ + n = 3 + D = cp.Variable((n, n), diag=True) + + # Set value using a sparse diagonal matrix + diag_values = np.array([1.0, 2.0, 3.0]) + D.value = sp.diags(diag_values, format='dia') + + # Verify value is sparse + assert sp.issparse(D.value) + + # Create a simple constrained problem (use sum instead of trace for NLP support) + prob = cp.Problem( + cp.Minimize(cp.sum_squares(D)), + [cp.sum(D) >= 1] + ) + + # Solve with NLP - the sparse initialization should propagate correctly + prob.solve(solver=cp.IPOPT, nlp=True) + + assert prob.status == cp.OPTIMAL + # The optimal solution should have sum = 1 (constraint active at minimum) + assert np.isclose(D.value.toarray().sum(), 1.0, atol=1e-4) + + def test_advanced_pricing_problem(self): + """ + Test a more complex non-convex problem from Max Schaller. + Bounds are added to prevent unboundedness. + """ + np.random.seed(42) + n, N = 5, 10 + rank = 2 + D = np.random.randn(n, N) + Pitilde = np.random.randn(n + 1, N) + + Etilde_cp = cp.Variable((n, n + 1)) + Ediag = cp.Variable((n, n), diag=True) + B = cp.Variable((n, rank)) + C = cp.Variable((rank, n)) + + problem = cp.Problem( + cp.Maximize(cp.sum(cp.multiply(D, Etilde_cp @ Pitilde) - cp.exp(Etilde_cp @ Pitilde))), + [ + Etilde_cp[:, :-1] == Ediag + B @ C, + Ediag <= 0, + Etilde_cp >= -10, Etilde_cp <= 10, + B >= -5, B <= 5, + C >= -5, C <= 5, + ] + ) + + assert not problem.is_dcp(), "Problem should be non-DCP" + + # Set initial values + Etilde_cp.value = np.random.randn(n, n + 1) + Ediag.value = -np.abs(np.diag(np.random.randn(n))) + B.value = np.random.randn(n, rank) + C.value = np.random.randn(rank, n) + + problem.solve(solver=cp.IPOPT, nlp=True, max_iter=200) + + assert problem.status == cp.OPTIMAL, "Problem did not solve to optimality" diff --git a/cvxpy/tests/nlp_tests/test_interfaces.py b/cvxpy/tests/nlp_tests/test_interfaces.py new file mode 100644 index 0000000000..630846dd2a --- /dev/null +++ b/cvxpy/tests/nlp_tests/test_interfaces.py @@ -0,0 +1,268 @@ +import numpy as np +import pytest + +import cvxpy as cp +from cvxpy.reductions.solvers.defines import INSTALLED_SOLVERS + + +def is_knitro_available(): + """Check if KNITRO is installed and a license is available.""" + import os + if 'KNITRO' not in INSTALLED_SOLVERS: + return False + # Only run KNITRO tests if license env var is explicitly set + # This prevents hanging in CI when KNITRO is installed but not licensed + return bool( + os.environ.get('ARTELYS_LICENSE') or + os.environ.get('ARTELYS_LICENSE_NETWORK_ADDR') + ) + + +@pytest.mark.skipif( + not is_knitro_available(), + reason='KNITRO is not installed or license is not available.' +) +class TestKNITROInterface: + """Tests for KNITRO solver interface options and algorithms.""" + + def test_knitro_basic_solve(self): + """Test that KNITRO can solve a basic NLP problem.""" + x = cp.Variable() + prob = cp.Problem(cp.Minimize((x - 2) ** 2), [x >= 1]) + prob.solve(solver=cp.KNITRO, nlp=True) + assert prob.status == cp.OPTIMAL + assert np.isclose(x.value, 2.0, atol=1e-5) + + def test_knitro_algorithm_bar_direct(self, capfd): + """Test Interior-Point/Barrier Direct algorithm (algorithm=1).""" + x = cp.Variable(2) + x.value = np.array([1.0, 1.0]) + prob = cp.Problem(cp.Minimize(x[0]**2 + x[1]**2), [x[0] + x[1] >= 1]) + + prob.solve(solver="knitro_ipm", nlp=True, verbose=True) + + captured = capfd.readouterr() + output = captured.out + captured.err + + assert prob.status == cp.OPTIMAL + assert np.allclose(x.value, [0.5, 0.5], atol=1e-4) + assert "Interior-Point/Barrier Direct" in output + + def test_knitro_algorithm_bar_cg(self, capfd): + """Test Interior-Point/Barrier CG algorithm (algorithm=2).""" + x = cp.Variable(2) + x.value = np.array([1.0, 1.0]) + prob = cp.Problem(cp.Minimize(x[0]**2 + x[1]**2), [x[0] + x[1] >= 1]) + + prob.solve(solver=cp.KNITRO, nlp=True, verbose=True, algorithm=2) + + captured = capfd.readouterr() + output = captured.out + captured.err + + assert prob.status == cp.OPTIMAL + assert np.allclose(x.value, [0.5, 0.5], atol=1e-4) + assert "Interior-Point/Barrier Conjugate Gradient" in output + + def test_knitro_algorithm_act_cg(self, capfd): + """Test Active-Set CG algorithm (algorithm=3).""" + x = cp.Variable(2) + x.value = np.array([1.0, 1.0]) + prob = cp.Problem(cp.Minimize(x[0]**2 + x[1]**2), [x[0] + x[1] >= 1]) + + prob.solve(solver=cp.KNITRO, nlp=True, verbose=True, algorithm=3) + + captured = capfd.readouterr() + output = captured.out + captured.err + + assert prob.status == cp.OPTIMAL + assert np.allclose(x.value, [0.5, 0.5], atol=1e-4) + assert "Active-Set" in output + + def test_knitro_algorithm_sqp(self, capfd): + """Test Active-Set SQP algorithm (algorithm=4).""" + x = cp.Variable(2) + x.value = np.array([1.0, 1.0]) + prob = cp.Problem(cp.Minimize(x[0]**2 + x[1]**2), [x[0] + x[1] >= 1]) + + prob.solve(solver="knitro_sqp", nlp=True, verbose=True) + + captured = capfd.readouterr() + output = captured.out + captured.err + + assert prob.status == cp.OPTIMAL + assert np.allclose(x.value, [0.5, 0.5], atol=1e-4) + assert "SQP" in output + + def test_knitro_algorithm_alm(self, capfd): + """Test Augmented Lagrangian Method algorithm (algorithm=6).""" + # ALM works best on unconstrained or simple problems + x = cp.Variable(2, name='x') + prob = cp.Problem( + cp.Minimize((1 - x[0])**2 + 100 * (x[1] - x[0]**2)**2), + [] + ) + prob.solve(solver="knitro_alm", nlp=True, verbose=True) + + captured = capfd.readouterr() + output = captured.out + captured.err + + assert prob.status == cp.OPTIMAL + assert np.allclose(x.value, [1.0, 1.0], atol=1e-3) + # ALM output shows "Sequential Quadratic Programming" as the subproblem solver + # but we can verify the algorithm parameter was set + assert "nlp_algorithm 6" in output + + def test_knitro_hessopt_exact(self, capfd): + """Test exact Hessian option (hessopt=1, default).""" + x = cp.Variable(2) + x.value = np.array([1.0, 1.0]) + prob = cp.Problem(cp.Minimize(x[0]**2 + x[1]**2), [x[0] + x[1] >= 1]) + + prob.solve(solver=cp.KNITRO, nlp=True, verbose=True, hessopt=1) + + captured = capfd.readouterr() + output = captured.out + captured.err + + assert prob.status == cp.OPTIMAL + assert np.allclose(x.value, [0.5, 0.5], atol=1e-4) + assert "hessopt 1" in output + + def test_knitro_hessopt_bfgs(self, capfd): + """Test BFGS Hessian approximation (hessopt=2).""" + x = cp.Variable(2) + x.value = np.array([1.0, 1.0]) + prob = cp.Problem(cp.Minimize(x[0]**2 + x[1]**2), [x[0] + x[1] >= 1]) + + prob.solve(solver=cp.KNITRO, nlp=True, verbose=True, hessopt=2) + + captured = capfd.readouterr() + output = captured.out + captured.err + + assert prob.status == cp.OPTIMAL + assert np.allclose(x.value, [0.5, 0.5], atol=1e-4) + assert "hessopt 2" in output + + def test_knitro_hessopt_lbfgs(self, capfd): + """Test L-BFGS Hessian approximation (hessopt=6).""" + x = cp.Variable(2) + x.value = np.array([1.0, 1.0]) + prob = cp.Problem(cp.Minimize(x[0]**2 + x[1]**2), [x[0] + x[1] >= 1]) + + prob.solve(solver=cp.KNITRO, nlp=True, verbose=True, hessopt=6) + + captured = capfd.readouterr() + output = captured.out + captured.err + + assert prob.status == cp.OPTIMAL + assert np.allclose(x.value, [0.5, 0.5], atol=1e-4) + assert "hessopt 6" in output + + def test_knitro_hessopt_sr1(self, capfd): + """Test SR1 Hessian approximation (hessopt=3).""" + x = cp.Variable(2) + x.value = np.array([1.0, 1.0]) + prob = cp.Problem(cp.Minimize(x[0]**2 + x[1]**2), [x[0] + x[1] >= 1]) + + prob.solve(solver=cp.KNITRO, nlp=True, verbose=True, hessopt=3) + + captured = capfd.readouterr() + output = captured.out + captured.err + + assert prob.status == cp.OPTIMAL + assert np.allclose(x.value, [0.5, 0.5], atol=1e-4) + assert "hessopt 3" in output + + def test_knitro_maxit(self): + """Test maximum iterations option.""" + x = cp.Variable(2) + x.value = np.array([1.0, 1.0]) + prob = cp.Problem(cp.Minimize(x[0]**2 + x[1]**2), [x[0] + x[1] >= 1]) + + # Use a reasonable maxit value that allows convergence + prob.solve(solver=cp.KNITRO, nlp=True, maxit=100) + assert prob.status == cp.OPTIMAL + assert np.allclose(x.value, [0.5, 0.5], atol=1e-4) + + def test_knitro_maxit_limit(self): + """Test that max iterations limit is respected.""" + # Rosenbrock is harder to solve - use very small maxit + x = cp.Variable(2, name='x') + prob = cp.Problem( + cp.Minimize((1 - x[0])**2 + 100 * (x[1] - x[0]**2)**2), + [] + ) + + # With only 1 iteration, solver should hit the limit + prob.solve(solver=cp.KNITRO, nlp=True, maxit=1) + # Status should be USER_LIMIT (iteration limit reached) + assert prob.status in [cp.USER_LIMIT, cp.OPTIMAL_INACCURATE, cp.OPTIMAL] + + def test_knitro_combined_options(self, capfd): + """Test combining multiple KNITRO options.""" + x = cp.Variable(2) + x.value = np.array([1.0, 1.0]) + prob = cp.Problem(cp.Minimize(x[0]**2 + x[1]**2), [x[0] + x[1] >= 1]) + + prob.solve( + solver=cp.KNITRO, + nlp=True, + verbose=True, + algorithm=1, # BAR_DIRECT + hessopt=2, # BFGS + feastol=1e-8, + opttol=1e-8, + ) + + captured = capfd.readouterr() + output = captured.out + captured.err + + assert prob.status == cp.OPTIMAL + assert np.allclose(x.value, [0.5, 0.5], atol=1e-4) + assert "Interior-Point/Barrier Direct" in output + assert "hessopt 2" in output + assert "feastol 1e-08" in output + assert "opttol 1e-08" in output + + def test_knitro_unknown_option_raises(self): + """Test that unknown options raise ValueError.""" + x = cp.Variable() + prob = cp.Problem(cp.Minimize((x - 2) ** 2), [x >= 1]) + + with pytest.raises(ValueError, match="Unknown KNITRO option"): + prob.solve(solver=cp.KNITRO, nlp=True, unknown_option=123) + + def test_knitro_solver_stats(self): + """Test that solver stats (num_iters, solve_time) are available.""" + x = cp.Variable(2) + x.value = np.array([1.0, 1.0]) + prob = cp.Problem(cp.Minimize(x[0]**2 + x[1]**2), [x[0] + x[1] >= 1]) + + prob.solve(solver=cp.KNITRO, nlp=True) + + assert prob.status == cp.OPTIMAL + assert prob.solver_stats is not None + assert prob.solver_stats.num_iters == 3 + assert prob.solver_stats.solve_time < 0.005 # 5ms solve time + + +@pytest.mark.skipif('COPT' not in INSTALLED_SOLVERS, reason='COPT is not installed.') +class TestCOPTInterface: + + def test_copt_basic_solve(self): + """Test that COPT can solve a basic NLP problem.""" + x = cp.Variable() + prob = cp.Problem(cp.Minimize((x - 2) ** 2), [x >= 1]) + prob.solve(solver=cp.COPT, nlp=True) + assert prob.status == cp.OPTIMAL + assert np.isclose(x.value, 2.0, atol=1e-5) + + def test_copt_maxit(self): + """Test maximum iterations option.""" + x = cp.Variable(2) + x.value = np.array([1.0, 1.0]) + prob = cp.Problem(cp.Minimize(x[0]**2 + x[1]**2), [x[0] + x[1] >= 1]) + + # Use a reasonable maxit value that allows convergence + prob.solve(solver=cp.COPT, nlp=True, NLPIterLimit=100) + assert prob.status == cp.OPTIMAL + assert np.allclose(x.value, [0.5, 0.5], atol=1e-4) diff --git a/cvxpy/tests/nlp_tests/test_log_sum_exp.py b/cvxpy/tests/nlp_tests/test_log_sum_exp.py new file mode 100644 index 0000000000..4315161dc5 --- /dev/null +++ b/cvxpy/tests/nlp_tests/test_log_sum_exp.py @@ -0,0 +1,61 @@ +import numpy as np +import pytest + +import cvxpy as cp +from cvxpy.reductions.solvers.defines import INSTALLED_SOLVERS +from cvxpy.reductions.solvers.nlp_solvers.nlp_solver import DerivativeChecker + + +@pytest.mark.skipif('IPOPT' not in INSTALLED_SOLVERS, reason='IPOPT is not installed.') +class TestLogSumExp(): + + def test_one(self): + x = cp.Variable(3, name='x') + obj = cp.Minimize(cp.log_sum_exp(x)) + constraints = [x >= 1] + prob = cp.Problem(obj, constraints) + prob.solve(nlp=True, verbose=True, derivative_test='none') + expected = np.log(3 * np.exp(1)) + assert np.isclose(obj.value, expected) + + checker = DerivativeChecker(prob) + checker.run_and_assert() + + def test_two(self): + m = 50 + n = 10 + A = np.random.randn(m, n) + x = cp.Variable(n) + obj = cp.Minimize(cp.log_sum_exp(A @ x)) + constraints = [x >= 0, cp.sum(x) == 1] + prob = cp.Problem(obj, constraints) + prob.solve(nlp=True, verbose=True, derivative_test='second-order') + DNLP_opt_val = obj.value + prob.solve(solver=cp.CLARABEL, verbose=True) + DCP_opt_val = obj.value + assert np.isclose(DNLP_opt_val, DCP_opt_val) + + x.value = x.value + 0.1 # perturb to avoid boundary issues + checker = DerivativeChecker(prob) + checker.run_and_assert() + + @pytest.mark.parametrize( + "m, n", + [(50, 25), (300, 100)] + ) + def test_three(self, m, n): + A = np.random.randn(m, n) + x = cp.Variable(n) + y = cp.Variable(n) + obj = cp.Minimize(cp.log_sum_exp(cp.square(A @ x))) + constraints = [x >= 0, x + y == 1, y >= 0] + prob = cp.Problem(obj, constraints) + prob.solve(nlp=True, verbose=True, derivative_test='none') + DNLP_opt_val = obj.value + prob.solve(solver=cp.CLARABEL, verbose=True) + DCP_opt_val = obj.value + assert np.isclose(DNLP_opt_val, DCP_opt_val) + + checker = DerivativeChecker(prob) + checker.run_and_assert() + diff --git a/cvxpy/tests/nlp_tests/test_matmul.py b/cvxpy/tests/nlp_tests/test_matmul.py new file mode 100644 index 0000000000..cd7096b6e7 --- /dev/null +++ b/cvxpy/tests/nlp_tests/test_matmul.py @@ -0,0 +1,96 @@ +import numpy as np +import pytest + +import cvxpy as cp +from cvxpy.reductions.solvers.defines import INSTALLED_SOLVERS +from cvxpy.reductions.solvers.nlp_solvers.nlp_solver import DerivativeChecker + + +@pytest.mark.skipif('IPOPT' not in INSTALLED_SOLVERS, reason='IPOPT is not installed.') +class TestMatmul(): + + def test_simple_matmul_graph_form(self): + np.random.seed(0) + m, n, p = 5, 7, 11 + X = cp.Variable((m, n), bounds=[-1, 1], name='X') + Y = cp.Variable((n, p), bounds=[-2, 2], name='Y') + t = cp.Variable(name='t') + X.value = np.random.rand(m, n) + Y.value = np.random.rand(n, p) + constraints = [t == cp.sum(cp.matmul(X, Y))] + problem = cp.Problem(cp.Minimize(t), constraints) + + problem.solve(solver=cp.IPOPT, nlp=True, hessian_approximation='exact', + derivative_test='none', verbose=False) + assert(problem.status == cp.OPTIMAL) + + checker = DerivativeChecker(problem) + checker.run_and_assert() + + + + def test_simple_matmul_not_graph_form(self): + np.random.seed(0) + m, n, p = 5, 7, 11 + X = cp.Variable((m, n), bounds=[-1, 1], name='X') + Y = cp.Variable((n, p), bounds=[-2, 2], name='Y') + X.value = np.random.rand(m, n) + Y.value = np.random.rand(n, p) + obj = cp.sum(cp.matmul(X, Y)) + problem = cp.Problem(cp.Minimize(obj)) + + problem.solve(solver=cp.IPOPT, nlp=True, hessian_approximation='exact', + derivative_test='none', verbose=False) + assert(problem.status == cp.OPTIMAL) + + checker = DerivativeChecker(problem) + checker.run_and_assert() + + def test_matmul_with_function_right(self): + np.random.seed(0) + m, n, p = 5, 7, 11 + X = np.random.rand(m, n) + Y = cp.Variable((n, p), bounds=[-2, 2], name='Y') + Y.value = np.random.rand(n, p) + obj = cp.sum(cp.matmul(X, cp.cos(Y))) + problem = cp.Problem(cp.Minimize(obj)) + + problem.solve(solver=cp.IPOPT, nlp=True, hessian_approximation='exact', + derivative_test='none', verbose=True) + assert(problem.status == cp.OPTIMAL) + + checker = DerivativeChecker(problem) + checker.run_and_assert() + + def test_matmul_with_function_left(self): + np.random.seed(0) + m, n, p = 5, 7, 11 + X = cp.Variable((m, n), bounds=[-2, 2], name='X') + Y = np.random.rand(n, p) + X.value = np.random.rand(m, n) + obj = cp.sum(cp.matmul(cp.cos(X), Y)) + problem = cp.Problem(cp.Minimize(obj)) + + problem.solve(solver=cp.IPOPT, nlp=True, hessian_approximation='exact', + derivative_test='none', verbose=True) + assert(problem.status == cp.OPTIMAL) + + checker = DerivativeChecker(problem) + checker.run_and_assert() + + def test_matmul_with_functions_both_sides(self): + np.random.seed(0) + m, n, p = 5, 7, 11 + X = cp.Variable((m, n), bounds=[-2, 2], name='X') + Y = cp.Variable((n, p), bounds=[-2, 2], name='Y') + X.value = np.random.rand(m, n) + Y.value = np.random.rand(n, p) + obj = cp.sum(cp.matmul(cp.cos(X), cp.sin(Y))) + problem = cp.Problem(cp.Minimize(obj)) + + problem.solve(solver=cp.IPOPT, nlp=True, hessian_approximation='exact', + derivative_test='none', verbose=True) + assert(problem.status == cp.OPTIMAL) + + checker = DerivativeChecker(problem) + checker.run_and_assert() \ No newline at end of file diff --git a/cvxpy/tests/nlp_tests/test_nlp_solvers.py b/cvxpy/tests/nlp_tests/test_nlp_solvers.py new file mode 100644 index 0000000000..f57a5bc7f3 --- /dev/null +++ b/cvxpy/tests/nlp_tests/test_nlp_solvers.py @@ -0,0 +1,464 @@ + +import numpy as np +import pytest + +import cvxpy as cp +from cvxpy.reductions.solvers.defines import INSTALLED_SOLVERS +from cvxpy.reductions.solvers.nlp_solvers.nlp_solver import DerivativeChecker +from cvxpy.tests.test_conic_solvers import is_knitro_available + +# Always parametrize all solvers, skip at runtime if not available +NLP_SOLVERS = [ + pytest.param('IPOPT', marks=pytest.mark.skipif( + 'IPOPT' not in INSTALLED_SOLVERS, reason='IPOPT is not installed.')), + pytest.param('KNITRO', marks=pytest.mark.skipif( + not is_knitro_available(), reason='KNITRO is not installed or license not available.')), + pytest.param('UNO', marks=pytest.mark.skipif( + 'UNO' not in INSTALLED_SOLVERS, reason='UNO is not installed.')), + pytest.param('COPT', marks=pytest.mark.skipif( + 'COPT' not in INSTALLED_SOLVERS, reason='COPT is not installed.')), +] + + +@pytest.mark.parametrize("solver", NLP_SOLVERS) +class TestNLPExamples: + """ + Nonlinear test problems taken from the IPOPT documentation and + the Julia documentation: https://jump.dev/JuMP.jl/stable/tutorials/nonlinear/simple_examples/. + """ + + def test_hs071(self, solver): + x = cp.Variable(4, bounds=[0, 6]) + x.value = np.array([1.0, 5.0, 5.0, 1.0]) + objective = cp.Minimize(x[0]*x[3]*(x[0] + x[1] + x[2]) + x[2]) + + constraints = [ + x[0]*x[1]*x[2]*x[3] >= 25, + cp.sum(cp.square(x)) == 40, + ] + problem = cp.Problem(objective, constraints) + problem.solve(solver=solver, nlp=True) + assert problem.status == cp.OPTIMAL + assert np.allclose(x.value, np.array([0.75450865, 4.63936861, 3.78856881, 1.88513184])) + + checker = DerivativeChecker(problem) + checker.run_and_assert() + + def test_mle(self, solver): + n = 1000 + np.random.seed(1234) + data = np.random.randn(n) + + mu = cp.Variable((1,), name="mu") + mu.value = np.array([0.0]) + sigma = cp.Variable((1,), name="sigma") + sigma.value = np.array([1.0]) + + constraints = [mu == sigma**2] + log_likelihood = ( + (n / 2) * cp.log(1 / (2 * np.pi * (sigma)**2)) + - cp.sum(cp.square(data-mu)) / (2 * (sigma)**2) + ) + + objective = cp.Maximize(log_likelihood) + problem = cp.Problem(objective, constraints) + problem.solve(solver=solver, nlp=True) + assert problem.status == cp.OPTIMAL + assert np.allclose(sigma.value, 0.77079388) + assert np.allclose(mu.value, 0.59412321) + + checker = DerivativeChecker(problem) + checker.run_and_assert() + + def test_portfolio_opt(self, solver): + # data taken from https://jump.dev/JuMP.jl/stable/tutorials/nonlinear/portfolio/ + # r and Q are pre-computed from historical data of 3 assets + r = np.array([0.026002150277777, 0.008101316405671, 0.073715909491990]) + Q = np.array([ + [0.018641039983891, 0.003598532927677, 0.001309759253660], + [0.003598532927677, 0.006436938322676, 0.004887265158407], + [0.001309759253660, 0.004887265158407, 0.068682765454814], + ]) + x = cp.Variable(3) + x.value = np.array([10.0, 10.0, 10.0]) + variance = cp.quad_form(x, Q) + expected_return = r @ x + problem = cp.Problem( + cp.Minimize(variance), + [ + cp.sum(x) <= 1000, + expected_return >= 50, + x >= 0 + ] + ) + problem.solve(solver=solver, nlp=True) + assert problem.status == cp.OPTIMAL + # Second element can be slightly negative due to numerical tolerance + assert np.allclose(x.value, np.array([4.97045504e+02, 0.0, 5.02954496e+02]), atol=1e-4) + + checker = DerivativeChecker(problem) + checker.run_and_assert() + + def test_portfolio_opt_sum_multiply(self, solver): + # data taken from https://jump.dev/JuMP.jl/stable/tutorials/nonlinear/portfolio/ + # r and Q are pre-computed from historical data of 3 assets + r = np.array([0.026002150277777, 0.008101316405671, 0.073715909491990]) + Q = np.array([ + [0.018641039983891, 0.003598532927677, 0.001309759253660], + [0.003598532927677, 0.006436938322676, 0.004887265158407], + [0.001309759253660, 0.004887265158407, 0.068682765454814], + ]) + x = cp.Variable(3) + x.value = np.array([10.0, 10.0, 10.0]) + variance = cp.quad_form(x, Q) + expected_return = cp.sum(cp.multiply(r, x)) + problem = cp.Problem( + cp.Minimize(variance), + [ + cp.sum(x) <= 1000, + expected_return >= 50, + x >= 0 + ] + ) + problem.solve(solver=solver, nlp=True) + assert problem.status == cp.OPTIMAL + # Second element can be slightly negative due to numerical tolerance + assert np.allclose(x.value, np.array([4.97045504e+02, 0.0, 5.02954496e+02]), atol=1e-4) + + checker = DerivativeChecker(problem) + checker.run_and_assert() + + def test_rosenbrock(self, solver): + x = cp.Variable(2, name='x') + objective = cp.Minimize((1 - x[0])**2 + 100 * (x[1] - x[0]**2)**2) + problem = cp.Problem(objective, []) + problem.solve(solver=solver, nlp=True) + assert problem.status == cp.OPTIMAL + assert np.allclose(x.value, np.array([1.0, 1.0])) + + checker = DerivativeChecker(problem) + checker.run_and_assert() + + def test_qcp(self, solver): + # Use IPM for UNO on this test, SQP converges to a suboptimal point: (0, 0, 1) + if solver == 'UNO': + solver = 'UNO_IPM' + x = cp.Variable(1) + y = cp.Variable(1, bounds=[0, np.inf]) + z = cp.Variable(1, bounds=[0, np.inf]) + + objective = cp.Maximize(x) + + constraints = [ + x + y + z == 1, + x**2 + y**2 - z**2 <= 0, + x**2 - cp.multiply(y, z) <= 0 + ] + problem = cp.Problem(objective, constraints) + problem.solve(solver=solver, nlp=True) + assert problem.status == cp.OPTIMAL + assert np.allclose(x.value, np.array([0.32699284])) + assert np.allclose(y.value, np.array([0.25706586])) + assert np.allclose(z.value, np.array([0.4159413])) + + checker = DerivativeChecker(problem) + checker.run_and_assert() + + def test_analytic_polytope_center(self, solver): + # Generate random data + np.random.seed(0) + m, n = 50, 4 + b = np.ones(m) + rand = np.random.randn(m - 2*n, n) + A = np.vstack((rand, np.eye(n), np.eye(n) * -1)) + + # Define the variable + x = cp.Variable(n) + # set initial value for x + objective = cp.Minimize(-cp.sum(cp.log(b - A @ x))) + problem = cp.Problem(objective, []) + # Solve the problem + problem.solve(solver=solver, nlp=True) + + assert problem.status == cp.OPTIMAL + + checker = DerivativeChecker(problem) + checker.run_and_assert() + + def test_analytic_polytope_center_x_column_vector(self, solver): + # Generate random data + np.random.seed(0) + m, n = 50, 4 + b = np.ones((m, 1)) + rand = np.random.randn(m - 2*n, n) + A = np.vstack((rand, np.eye(n), np.eye(n) * -1)) + + # Define the variable + x = cp.Variable((n, 1)) + # set initial value for x + objective = cp.Minimize(-cp.sum(cp.log(b - A @ x))) + problem = cp.Problem(objective, []) + # Solve the problem + problem.solve(solver=solver, nlp=True) + assert problem.status == cp.OPTIMAL + + checker = DerivativeChecker(problem) + checker.run_and_assert() + + + def test_socp(self, solver): + # Define variables + x = cp.Variable(3) + y = cp.Variable() + + # Define objective function + objective = cp.Minimize(3 * x[0] + 2 * x[1] + x[2]) + + # Define constraints + constraints = [ + cp.norm(x, 2) <= y, + x[0] + x[1] + 3*x[2] >= 1.0, + y <= 5 + ] + + # Create and solve the problem + problem = cp.Problem(objective, constraints) + problem.solve(solver=solver, nlp=True) + assert problem.status == cp.OPTIMAL + assert np.allclose(objective.value, -13.548638814247532) + assert np.allclose(x.value, [-3.87462191, -2.12978826, 2.33480343]) + assert np.allclose(y.value, 5) + + checker = DerivativeChecker(problem) + checker.run_and_assert() + + def test_portfolio_socp(self, solver): + np.random.seed(858) + n = 100 + x = cp.Variable(n, name='x') + mu = np.random.randn(n) + Sigma = np.random.randn(n, n) + Sigma = Sigma.T @ Sigma + gamma = 0.1 + t = cp.Variable(name='t', bounds=[0, None]) + L = np.linalg.cholesky(Sigma, upper=False) + + objective = cp.Minimize(- mu.T @ x + gamma * t) + constraints = [cp.norm(L.T @ x, 2) <= t, + cp.sum(x) == 1, + x >= 0] + problem = cp.Problem(objective, constraints) + problem.solve(solver=solver, nlp=True) + assert problem.status == cp.OPTIMAL + assert np.allclose(problem.value, -1.93414338e+00) + + checker = DerivativeChecker(problem) + checker.run_and_assert() + + def test_portfolio_socp_x_column_vector(self, solver): + np.random.seed(858) + n = 100 + x = cp.Variable((n, 1), name='x') + mu = np.random.randn(n, 1) + Sigma = np.random.randn(n, n) + Sigma = Sigma.T @ Sigma + gamma = 0.1 + t = cp.Variable(name='t', bounds=[0, None]) + L = np.linalg.cholesky(Sigma, upper=False) + + objective = cp.Minimize(-cp.sum(cp.multiply(mu, x)) + gamma * t) + constraints = [cp.norm(L.T @ x, 2) <= t, + cp.sum(x) == 1, + x >= 0] + problem = cp.Problem(objective, constraints) + problem.solve(solver=solver, nlp=True) + assert problem.status == cp.OPTIMAL + assert np.allclose(problem.value, -1.93414338e+00) + + checker = DerivativeChecker(problem) + checker.run_and_assert() + + def test_localization(self, solver): + np.random.seed(42) + m = 10 + dim = 2 + x_true = np.array([2.0, -1.5]) + a = np.random.uniform(-5, 5, (m, dim)) + rho = np.linalg.norm(a - x_true, axis=1) # no noise + x = cp.Variable(2, name='x') + t = cp.Variable(m, name='t') + constraints = [t == cp.sqrt(cp.sum(cp.square(x - a), axis=1))] + objective = cp.Minimize(cp.sum_squares(t - rho)) + problem = cp.Problem(objective, constraints) + problem.solve(solver=solver, nlp=True) + assert problem.status == cp.OPTIMAL + assert np.allclose(x.value, x_true) + + checker = DerivativeChecker(problem) + checker.run_and_assert() + + def test_localization2(self, solver): + np.random.seed(42) + m = 10 + dim = 2 + x_true = np.array([2.0, -1.5]) + a = np.random.uniform(-5, 5, (m, dim)) + rho = np.linalg.norm(a - x_true, axis=1) # no noise + x = cp.Variable((1, 2), name='x') + t = cp.Variable(m, name='t') + constraints = [t == cp.sqrt(cp.sum(cp.square(x - a), axis=1))] + objective = cp.Minimize(cp.sum_squares(t - rho)) + problem = cp.Problem(objective, constraints) + problem.solve(solver=solver, nlp=True) + assert problem.status == cp.OPTIMAL + assert np.allclose(x.value, x_true) + + checker = DerivativeChecker(problem) + checker.run_and_assert() + + def test_circle_packing_formulation_one(self, solver): + """Epigraph formulation.""" + rng = np.random.default_rng(5) + n = 3 + radius = rng.uniform(1.0, 3.0, n) + + centers = cp.Variable((2, n), name='c') + constraints = [] + for i in range(n - 1): + for j in range(i + 1, n): + constraints += [cp.sum(cp.square(centers[:, i] - centers[:, j])) >= + (radius[i] + radius[j]) ** 2] + + centers.value = rng.uniform(-5.0, 5.0, (2, n)) + t = cp.Variable() + obj = cp.Minimize(t) + constraints += [cp.max(cp.norm_inf(centers, axis=0) + radius) <= t] + problem = cp.Problem(obj, constraints) + problem.solve(solver=solver, nlp=True) + + true_sol = np.array([[1.73655994, -1.98685738, 2.57208783], + [1.99273311, -1.67415425, -2.57208783]]) + assert np.allclose(centers.value, true_sol) + + checker = DerivativeChecker(problem) + checker.run_and_assert() + + def test_circle_packing_formulation_two(self, solver): + """Using norm_inf. This test revealed a very subtle bug in the unpacking of + the ipopt solution. Some variables were mistakenly reordered. It was fixed + in https://github.com/cvxgrp/cvxpy-ipopt/pull/82""" + rng = np.random.default_rng(5) + n = 3 + radius = rng.uniform(1.0, 3.0, n) + + centers = cp.Variable((2, n), name='c') + constraints = [] + for i in range(n - 1): + for j in range(i + 1, n): + constraints += [cp.sum(cp.square(centers[:, i] - centers[:, j])) >= + (radius[i] + radius[j]) ** 2] + + centers.value = rng.uniform(-5.0, 5.0, (2, n)) + obj = cp.Minimize(cp.max(cp.norm_inf(centers, axis=0) + radius)) + prob = cp.Problem(obj, constraints) + prob.solve(solver=solver, nlp=True) + + assert np.allclose(obj.value, 4.602738956101437) + + residuals = [] + for i in range(n - 1): + for j in range(i + 1, n): + dist_sq = np.linalg.norm(centers.value[:, i] - centers.value[:, j]) ** 2 + min_dist_sq = (radius[i] + radius[j]) ** 2 + residuals.append(dist_sq - min_dist_sq) + + assert(np.all(np.array(residuals) <= 1e-6)) + + # Ipopt finds these centers, but Knitro rotates them (but finds the same + # objective value) + #true_sol = np.array([[1.73655994, -1.98685738, 2.57208783], + # [1.99273311, -1.67415425, -2.57208783]]) + #assert np.allclose(centers.value, true_sol) + + checker = DerivativeChecker(prob) + checker.run_and_assert() + + def test_circle_packing_formulation_three(self, solver): + """Using max max abs.""" + rng = np.random.default_rng(5) + n = 3 + radius = rng.uniform(1.0, 3.0, n) + + centers = cp.Variable((2, n), name='c') + constraints = [] + for i in range(n - 1): + for j in range(i + 1, n): + constraints += [cp.sum(cp.square(centers[:, i] - centers[:, j])) >= + (radius[i] + radius[j]) ** 2] + + centers.value = rng.uniform(-5.0, 5.0, (2, n)) + obj = cp.Minimize(cp.max(cp.max(cp.abs(centers), axis=0) + radius)) + prob = cp.Problem(obj, constraints) + prob.solve(solver=solver, nlp=True) + + true_sol = np.array([[1.73655994, -1.98685738, 2.57208783], + [1.99273311, -1.67415425, -2.57208783]]) + assert np.allclose(centers.value, true_sol) + + checker = DerivativeChecker(prob) + checker.run_and_assert() + + def test_geo_mean(self, solver): + x = cp.Variable(3, pos=True) + geo_mean = cp.geo_mean(x) + objective = cp.Maximize(geo_mean) + constraints = [cp.sum(x) == 1] + problem = cp.Problem(objective, constraints) + problem.solve(solver=solver, nlp=True) + assert problem.status == cp.OPTIMAL + assert np.allclose(x.value, np.array([1/3, 1/3, 1/3])) + + checker = DerivativeChecker(problem) + checker.run_and_assert() + + def test_geo_mean2(self, solver): + p = np.array([.07, .12, .23, .19, .39]) + x = cp.Variable(5, nonneg=True) + prob = cp.Problem(cp.Maximize(cp.geo_mean(x, p)), [cp.sum(x) <= 1]) + prob.solve(solver=solver, nlp=True) + x_true = p/sum(p) + assert prob.status == cp.OPTIMAL + assert np.allclose(x.value, x_true) + + checker = DerivativeChecker(prob) + checker.run_and_assert() + + def test_clnlbeam(self, solver): + N = 1000 + h = 1 / N + alpha = 350 + + t = cp.Variable(N+1, bounds=[-1, 1]) + x = cp.Variable(N+1, bounds=[-0.05, 0.05]) + u = cp.Variable(N+1) + u.value = np.zeros(N+1) + control_terms = cp.multiply(0.5 * h, cp.power(u[1:], 2) + cp.power(u[:-1], 2)) + trigonometric_terms = cp.multiply(0.5 * alpha * h, cp.cos(t[1:]) + cp.cos(t[:-1])) + objective_terms = cp.sum(control_terms + trigonometric_terms) + + objective = cp.Minimize(objective_terms) + constraints = [] + position_constraints = (x[1:] - x[:-1] - + cp.multiply(0.5 * h, cp.sin(t[1:]) + cp.sin(t[:-1])) == 0) + constraints.append(position_constraints) + angle_constraint = (t[1:] - t[:-1] - 0.5 * h * (u[1:] + u[:-1]) == 0) + constraints.append(angle_constraint) + + problem = cp.Problem(objective, constraints) + problem.solve(solver=solver, nlp=True) + assert problem.status == cp.OPTIMAL + assert np.allclose(problem.value, 3.500e+02) + + # the derivative checker takes more than 10 seconds on this problem + #checker = DerivativeChecker(problem) + #checker.run_and_assert() diff --git a/cvxpy/tests/nlp_tests/test_power_flow.py b/cvxpy/tests/nlp_tests/test_power_flow.py new file mode 100644 index 0000000000..7d98b8217c --- /dev/null +++ b/cvxpy/tests/nlp_tests/test_power_flow.py @@ -0,0 +1,100 @@ +import numpy as np +import pytest +from scipy.sparse import csr_matrix, diags + +import cvxpy as cp +from cvxpy.reductions.solvers.defines import INSTALLED_SOLVERS +from cvxpy.reductions.solvers.nlp_solvers.nlp_solver import DerivativeChecker + + +@pytest.mark.skipif('IPOPT' not in INSTALLED_SOLVERS, reason='IPOPT is not installed.') +class TestPowerFlowIPOPT: + """Power flow problem from DNLP paper""" + + def test_power_flow_dense_formulation(self): + # ----------------------------------------------------------------------------------- + # Define prob data + # ----------------------------------------------------------------------------------- + N = 9 + p_min = np.zeros(N) + p_max = np.zeros(N) + q_min = np.zeros(N) + q_max = np.zeros(N) + p_min[[0, 1, 2]] = [10, 10, 10] + p_max[[0, 1, 2]] = [250, 300, 270] + q_min[[0, 1, 2]] = [-5, -5, -5] + p_min[[4,6,8]] = p_max[[4,6,8]] = [-54, -60, -75] + q_min[[4,6,8]] = q_max[[4,6,8]] = [-18, -21, -30] + v_min, v_max = 0.9, 1.1 + + # ----------------------------------------------------------------------------------- + # Define admittance matrices + # ----------------------------------------------------------------------------------- + # Branch data: (from_bus, to_bus, resistance, reactance, susceptance) + branch_data = np.array([ + [0, 3, 0.0, 0.0576, 0.0], + [3, 4, 0.017, 0.092, 0.158], + [5, 4, 0.039, 0.17, 0.358], + [2, 5, 0.0, 0.0586, 0.0], + [5, 6, 0.0119, 0.1008, 0.209], + [7, 6, 0.0085, 0.072, 0.149], + [1, 7, 0.0, 0.0625, 0.0], + [7, 8, 0.032, 0.161, 0.306], + [3, 8, 0.01, 0.085, 0.176], + ]) + + M = branch_data.shape[0] # Number of branches + base_MVA = 100 + + # Build incidence matrix A + from_bus = branch_data[:, 0].astype(int) + to_bus = branch_data[:, 1].astype(int) + A = csr_matrix((np.ones(M), (from_bus, np.arange(M))), shape=(N, M)) + \ + csr_matrix((-np.ones(M), (to_bus, np.arange(M))), shape=(N, M)) + + # Network impedance + z = (branch_data[:, 2] + 1j * branch_data[:, 3]) / base_MVA + + # Bus admittance matrix Y_0 + Y_0 = A @ diags(1.0 / z) @ A.T + + # Shunt admittance from line charging + y_sh = 0.5 * (1j * branch_data[:, 4]) * base_MVA + Y_sh_diag = np.array((A @ diags(y_sh) @ A.T).diagonal()).flatten() + Y_sh = diags(Y_sh_diag) + + # Extract conductance and susceptance matrices + G0 = np.real(Y_0.toarray()) # Conductance matrix + B0 = np.imag(Y_0.toarray()) # Susceptance matrix + G_sh = np.real(Y_sh.toarray()) # Shunt conductance + B_sh = np.imag(Y_sh.toarray()) # + G = G0 + G_sh + B = B0 + B_sh + + + # ----------------------------------------------------------------------------------- + # Define optimization prob + # ----------------------------------------------------------------------------------- + theta, P, Q = cp.Variable((N, 1)), cp.Variable((N, N)), cp.Variable((N, N)) + v = cp.Variable((N, 1), bounds=[v_min, v_max]) + p = cp.Variable(N, bounds=[p_min, p_max]) + q = cp.Variable(N, bounds=[q_min, q_max]) + C, S = cp.cos(theta - theta.T), cp.sin(theta - theta.T) + + constr = [theta[0] == 0, p == cp.sum(P, axis=1), q == cp.sum(Q, axis=1), + P == cp.multiply(v @ v.T, cp.multiply(G, C) + cp.multiply(B, S)), + Q == cp.multiply(v @ v.T, cp.multiply(G, S) - cp.multiply(B, C))] + cost = (0.11 * p[0]**2 + 5 * p[0] + 150 + 0.085 * p[1]**2 + 1.2 * p[1] + 600 + + 0.1225 * p[2]**2 + p[2] + 335) + prob = cp.Problem(cp.Minimize(cost), constr) + + # ----------------------------------------------------------------------------------- + # Solve prob + # ----------------------------------------------------------------------------------- + prob.solve(nlp=True, solver=cp.IPOPT, verbose=False) + + assert prob.status == cp.OPTIMAL + assert np.abs(prob.value - 3087.84) / prob.value <= 1e-4 + + checker = DerivativeChecker(prob) + checker.run_and_assert() \ No newline at end of file diff --git a/cvxpy/tests/nlp_tests/test_problem.py b/cvxpy/tests/nlp_tests/test_problem.py new file mode 100644 index 0000000000..b9ec4641d6 --- /dev/null +++ b/cvxpy/tests/nlp_tests/test_problem.py @@ -0,0 +1,116 @@ + +import numpy as np + +import cvxpy as cp + + +class TestProblem(): + """ + This class can be used to test internal function for Problem that have been added + in the DNLP extension. + """ + + def test_set_initial_point_both_bounds_infinity(self): + # when both bounds are infinity, the initial point should be zero vector + + # test 1 + x = cp.Variable((3, )) + prob = cp.Problem(cp.Minimize(cp.sum(x))) + prob.set_NLP_initial_point() + assert (x.value == np.zeros((3, ))).all() + + # test 2 + x = cp.Variable((3, ), bounds=[None, None]) + prob = cp.Problem(cp.Minimize(cp.sum(x))) + prob.set_NLP_initial_point() + assert (x.value == np.zeros((3, ))).all() + + # test 3 + x = cp.Variable((3, ), bounds=[-np.inf, np.inf]) + prob = cp.Problem(cp.Minimize(cp.sum(x))) + prob.set_NLP_initial_point() + assert (x.value == np.zeros((3, ))).all() + + # test 4 + x = cp.Variable((3, ), bounds=[None, np.inf]) + prob = cp.Problem(cp.Minimize(cp.sum(x))) + prob.set_NLP_initial_point() + assert (x.value == np.zeros((3, ))).all() + + # test 5 + x = cp.Variable((3, ), bounds=[-np.inf, None]) + prob = cp.Problem(cp.Minimize(cp.sum(x))) + prob.set_NLP_initial_point() + assert (x.value == np.zeros((3, ))).all() + + + def test_set_initial_point_lower_bound_infinity(self): + # when one bound is infinity, the initial point should be one unit + # away from the finite bound + + # test 1 + x = cp.Variable((3, ), bounds=[None, 3.5]) + prob = cp.Problem(cp.Minimize(cp.sum(x))) + prob.set_NLP_initial_point() + assert (x.value == 2.5 * np.ones((3, ))).all() + + # test 2 + x = cp.Variable((3, ), bounds=[-np.inf, 3.5]) + prob = cp.Problem(cp.Minimize(cp.sum(x))) + prob.set_NLP_initial_point() + assert (x.value == 2.5 * np.ones((3, ))).all() + + def test_set_initial_point_upper_bound_infinity(self): + # when one bound is infinity, the initial point should be one unit + # away from the finite bound + + # test 1 + x = cp.Variable((3, ), bounds=[3.5, None]) + prob = cp.Problem(cp.Minimize(cp.sum(x))) + prob.set_NLP_initial_point() + assert (x.value == 4.5 * np.ones((3, ))).all() + + # test 2 + x = cp.Variable((3, ), bounds=[3.5, np.inf]) + prob = cp.Problem(cp.Minimize(cp.sum(x))) + prob.set_NLP_initial_point() + assert (x.value == 4.5 * np.ones((3, ))).all() + + def test_set_initial_point_both_bounds_finite(self): + # when both bounds are finite, the initial point should be the midpoint + # between the two bounds + + # test 1 + x = cp.Variable((3, ), bounds=[3.5, 4.5]) + prob = cp.Problem(cp.Minimize(cp.sum(x))) + prob.set_NLP_initial_point() + assert (x.value == 4.0 * np.ones((3, ))).all() + + def test_set_initial_point_mixed_inf_and_finite(self): + lb = np.array([-np.inf, 3.5, -np.inf, -1.5, 2, 2.5]) + ub = np.array([-4, 4.5, np.inf, 4.5, np.inf, 4.5]) + x = cp.Variable((6, ), bounds=[lb, ub]) + prob = cp.Problem(cp.Minimize(cp.sum(x))) + prob.set_NLP_initial_point() + expected = np.array([-5, 4.0, 0.0, 1.5, 3, 3.5]) + assert (x.value == expected).all() + + def test_set_initial_point_two_variables(self): + x = cp.Variable((2, ), bounds=[-np.inf, np.inf]) + y = cp.Variable((2, ), bounds=[-3, np.inf]) + prob = cp.Problem(cp.Minimize(cp.sum(x) + cp.sum(y))) + prob.set_NLP_initial_point() + assert (x.value == np.zeros((2, ))).all() + assert (y.value == -2 * np.ones((2, ))).all() + + def test_set_initial_point_nonnegative_attributes(self): + x = cp.Variable((2, ), nonneg=True) + prob = cp.Problem(cp.Minimize(cp.sum(x))) + prob.set_NLP_initial_point() + assert (x.value == np.ones((2, ))).all() + + def test_set_initial_point_nonnegative_attributes_and_bounds(self): + x = cp.Variable((2, ), nonneg=True, bounds=[1, None]) + prob = cp.Problem(cp.Minimize(cp.sum(x))) + prob.set_NLP_initial_point() + assert (x.value == 2 * np.ones((2, ))).all() \ No newline at end of file diff --git a/cvxpy/tests/nlp_tests/test_prod.py b/cvxpy/tests/nlp_tests/test_prod.py new file mode 100644 index 0000000000..387554bb5e --- /dev/null +++ b/cvxpy/tests/nlp_tests/test_prod.py @@ -0,0 +1,271 @@ +import numpy as np +import pytest + +import cvxpy as cp +from cvxpy.reductions.solvers.defines import INSTALLED_SOLVERS +from cvxpy.reductions.solvers.nlp_solvers.nlp_solver import DerivativeChecker + + +class TestProdDNLP: + """Test that prod expressions are correctly identified as DNLP.""" + + def test_prod_is_smooth(self): + """Test that prod is identified as smooth (both ESR and HSR).""" + x = cp.Variable(3, pos=True) + p = cp.prod(x) + assert p.is_atom_esr() + assert p.is_atom_hsr() + assert p.is_smooth() + + def test_prod_is_esr(self): + """Test that prod is ESR.""" + x = cp.Variable(3, pos=True) + p = cp.prod(x) + assert p.is_esr() + + def test_prod_is_hsr(self): + """Test that prod is HSR.""" + x = cp.Variable(3, pos=True) + p = cp.prod(x) + assert p.is_hsr() + + def test_prod_composition_smooth(self): + """Test that compositions with prod are smooth.""" + x = cp.Variable(3, pos=True) + expr = cp.log(cp.prod(x)) + assert expr.is_smooth() + + def test_prod_problem_is_dnlp(self): + """Test that a problem with prod is DNLP.""" + x = cp.Variable(3, pos=True) + obj = cp.Maximize(cp.prod(x)) + constr = [cp.sum(x) <= 3] + prob = cp.Problem(obj, constr) + assert prob.is_dnlp() + + +@pytest.mark.skipif('IPOPT' not in INSTALLED_SOLVERS, reason='IPOPT is not installed.') +class TestProdIPOPT: + """Test solving prod problems with IPOPT.""" + + def test_prod_maximize_positive(self): + """Test maximizing prod with positive variables.""" + x = cp.Variable(3, pos=True) + obj = cp.Maximize(cp.prod(x)) + constr = [cp.sum(x) <= 3] + prob = cp.Problem(obj, constr) + prob.solve(solver=cp.IPOPT, nlp=True, print_level=0) + + # Optimal is x = [1, 1, 1] by AM-GM inequality + assert np.allclose(x.value, [1.0, 1.0, 1.0], atol=1e-4) + assert np.isclose(prob.value, 1.0, atol=1e-4) + + checker = DerivativeChecker(prob) + checker.run_and_assert() + + def test_prod_minimize_squared(self): + """Test minimizing prod squared with mixed sign variables.""" + x = cp.Variable(3) + x.value = np.array([1.0, -1.0, 2.0]) + obj = cp.Minimize(cp.prod(x)**2) + constr = [x[0] >= 0.5, x[1] <= -0.5, x[2] >= 1, cp.sum(x) == 2] + prob = cp.Problem(obj, constr) + prob.solve(solver=cp.IPOPT, nlp=True, print_level=0) + + # Check constraints are satisfied + assert x.value[0] >= 0.5 - 1e-4 + assert x.value[1] <= -0.5 + 1e-4 + assert x.value[2] >= 1 - 1e-4 + assert np.isclose(np.sum(x.value), 2.0, atol=1e-4) + + # Check objective + assert np.isclose(prob.value, np.prod(x.value)**2, atol=1e-4) + + checker = DerivativeChecker(prob) + checker.run_and_assert() + + def test_prod_with_axis(self): + """Test prod with axis parameter.""" + X = cp.Variable((2, 3), pos=True) + obj = cp.Maximize(cp.sum(cp.prod(X, axis=1))) + # Constraint per row so AM-GM applies independently to each row + constr = [cp.sum(X, axis=1) <= 3] + prob = cp.Problem(obj, constr) + prob.solve(solver=cp.IPOPT, nlp=True, print_level=0) + + # By AM-GM, each row should be [1, 1, 1] for max prod + assert np.allclose(X.value, np.ones((2, 3)), atol=1e-4) + assert np.isclose(prob.value, 2.0, atol=1e-4) + + checker = DerivativeChecker(prob) + checker.run_and_assert() + + def test_prod_with_axis_large_matrix(self): + """Test prod with axis parameter on a larger matrix.""" + X = cp.Variable((4, 5), pos=True) + obj = cp.Maximize(cp.sum(cp.prod(X, axis=1))) + # Constraint per row so AM-GM applies independently to each row + constr = [cp.sum(X, axis=1) <= 5] + prob = cp.Problem(obj, constr) + prob.solve(solver=cp.IPOPT, nlp=True, print_level=0) + + # By AM-GM, each row should be [1, 1, 1, 1, 1] for max prod + assert np.allclose(X.value, np.ones((4, 5)), atol=1e-4) + assert np.isclose(prob.value, 4.0, atol=1e-4) + + checker = DerivativeChecker(prob) + checker.run_and_assert() + + def test_prod_with_zero_start(self): + """Test prod when starting near zero.""" + x = cp.Variable(3) + x.value = np.array([1.0, 0.1, 2.0]) + obj = cp.Minimize((cp.prod(x) - 1)**2) + constr = [x[0] >= 0.5, x[2] >= 1, cp.sum(x) == 3] + prob = cp.Problem(obj, constr) + prob.solve(solver=cp.IPOPT, nlp=True, print_level=0) + + # Should find a point where prod(x) ≈ 1 + assert np.isclose(np.prod(x.value), 1.0, atol=1e-3) + + checker = DerivativeChecker(prob) + checker.run_and_assert() + + def test_prod_negative_values(self): + """Test prod with negative values in solution.""" + x = cp.Variable(2) + x.value = np.array([2.0, -2.0]) + obj = cp.Minimize((cp.prod(x) + 4)**2) # min (x1*x2 + 4)^2 + constr = [x[0] == 2, x[1] == -2] + prob = cp.Problem(obj, constr) + prob.solve(solver=cp.IPOPT, nlp=True, print_level=0) + + # With x = [2, -2], prod = -4, so (prod + 4)^2 = 0 + assert np.isclose(x.value[0], 2.0, atol=1e-3) + assert np.isclose(x.value[1], -2.0, atol=1e-3) + assert np.isclose(np.prod(x.value), -4.0, atol=1e-3) + assert np.isclose(prob.value, 0.0, atol=1e-4) + + checker = DerivativeChecker(prob) + checker.run_and_assert() + + def test_prod_in_constraint(self): + """Test prod in a constraint.""" + x = cp.Variable(3, pos=True) + obj = cp.Minimize(cp.sum(x)) + constr = [cp.prod(x) >= 8] + prob = cp.Problem(obj, constr) + prob.solve(solver=cp.IPOPT, nlp=True, print_level=0) + + # By AM-GM, min sum when prod = 8 is at x = [2, 2, 2] + assert np.allclose(x.value, [2.0, 2.0, 2.0], atol=1e-3) + assert np.isclose(prob.value, 6.0, atol=1e-3) + + checker = DerivativeChecker(prob) + checker.run_and_assert() + + def test_prod_log_composition(self): + """Test log(prod(x)) = sum(log(x)) equivalence.""" + n = 4 + x = cp.Variable(n, pos=True) + + # Using log(prod(x)) + obj1 = cp.Maximize(cp.log(cp.prod(x))) + constr = [cp.sum(x) <= n] + prob1 = cp.Problem(obj1, constr) + prob1.solve(solver=cp.IPOPT, nlp=True, print_level=0) + val1 = prob1.value + x1 = x.value.copy() + + checker = DerivativeChecker(prob1) + checker.run_and_assert() + + # Using sum(log(x)) + obj2 = cp.Maximize(cp.sum(cp.log(x))) + prob2 = cp.Problem(obj2, constr) + prob2.solve(solver=cp.IPOPT, nlp=True, print_level=0) + val2 = prob2.value + x2 = x.value.copy() + + # Both should give the same result + assert np.isclose(val1, val2, atol=1e-4) + assert np.allclose(x1, x2, atol=1e-3) + + checker = DerivativeChecker(prob2) + checker.run_and_assert() + + def test_prod_exp_log_relationship(self): + """Test relationship: prod(x) = exp(sum(log(x))).""" + n = 3 + x = cp.Variable(n, pos=True) + + # Maximize prod(x) + obj1 = cp.Maximize(cp.prod(x)) + constr = [cp.sum(x) <= n] + prob1 = cp.Problem(obj1, constr) + prob1.solve(solver=cp.IPOPT, nlp=True, print_level=0) + prod_val = prob1.value + x1 = x.value.copy() + + checker = DerivativeChecker(prob1) + checker.run_and_assert() + + # Maximize exp(sum(log(x))) which equals prod(x) + obj2 = cp.Maximize(cp.exp(cp.sum(cp.log(x)))) + prob2 = cp.Problem(obj2, constr) + prob2.solve(solver=cp.IPOPT, nlp=True, print_level=0) + exp_sum_log_val = prob2.value + x2 = x.value.copy() + + # Both should give the same result + assert np.isclose(prod_val, exp_sum_log_val, atol=1e-4) + assert np.allclose(x1, x2, atol=1e-3) + + checker = DerivativeChecker(prob2) + checker.run_and_assert() + + def test_prod_with_axis_zero(self): + """Test prod with axis=0 parameter on a small matrix.""" + X = cp.Variable((3, 2), pos=True) + obj = cp.Maximize(cp.sum(cp.prod(X, axis=0))) + # Constraint per column so AM-GM applies independently to each column + constr = [cp.sum(X, axis=0) <= 3] + prob = cp.Problem(obj, constr) + prob.solve(solver=cp.IPOPT, nlp=True, print_level=0) + + # By AM-GM, each column should be [1, 1, 1] for max prod + assert np.allclose(X.value, np.ones((3, 2)), atol=1e-4) + assert np.isclose(prob.value, 2.0, atol=1e-4) + + checker = DerivativeChecker(prob) + checker.run_and_assert() + + def test_prod_with_axis_zero_large_matrix(self): + """Test prod with axis=0 parameter on a larger matrix.""" + X = cp.Variable((5, 4), pos=True) + obj = cp.Maximize(cp.sum(cp.prod(X, axis=0))) + # Constraint per column so AM-GM applies independently to each column + constr = [cp.sum(X, axis=0) <= 5] + prob = cp.Problem(obj, constr) + prob.solve(solver=cp.IPOPT, nlp=True, print_level=0) + + # By AM-GM, each column should be [1, 1, 1, 1, 1] for max prod + assert np.allclose(X.value, np.ones((5, 4)), atol=1e-4) + assert np.isclose(prob.value, 4.0, atol=1e-4) + + checker = DerivativeChecker(prob) + checker.run_and_assert() + + def test_prod_single_element(self): + """Test prod of single element.""" + x = cp.Variable(1, pos=True) + obj = cp.Maximize(cp.prod(x)) + constr = [x <= 5] + prob = cp.Problem(obj, constr) + prob.solve(solver=cp.IPOPT, nlp=True, print_level=0) + + assert np.isclose(x.value[0], 5.0, atol=1e-4) + assert np.isclose(prob.value, 5.0, atol=1e-4) + + checker = DerivativeChecker(prob) + checker.run_and_assert() \ No newline at end of file diff --git a/cvxpy/tests/nlp_tests/test_quasi_newton.py b/cvxpy/tests/nlp_tests/test_quasi_newton.py new file mode 100644 index 0000000000..336b603f53 --- /dev/null +++ b/cvxpy/tests/nlp_tests/test_quasi_newton.py @@ -0,0 +1,185 @@ +""" +Tests for quasi-Newton (L-BFGS) mode in NLP solvers. + +These tests verify that the quasi-Newton (hessian_approximation='limited-memory') +mode works correctly with IPOPT. In this mode, the Hessian is approximated using +L-BFGS rather than computed exactly, which can be faster for large problems +and allows solving problems where the Hessian is difficult to compute. +""" + +import numpy as np +import pytest + +import cvxpy as cp +from cvxpy.reductions.solvers.defines import INSTALLED_SOLVERS + + +@pytest.mark.skipif('IPOPT' not in INSTALLED_SOLVERS, reason='IPOPT is not installed.') +class TestQuasiNewton: + """Tests for quasi-Newton (L-BFGS) mode with IPOPT.""" + + def test_rosenbrock_lbfgs(self): + """Test Rosenbrock function with L-BFGS - classic unconstrained optimization.""" + x = cp.Variable(2, name='x') + objective = cp.Minimize((1 - x[0])**2 + 100 * (x[1] - x[0]**2)**2) + problem = cp.Problem(objective, []) + problem.solve(solver=cp.IPOPT, nlp=True, + hessian_approximation='limited-memory') + assert problem.status == cp.OPTIMAL + assert np.allclose(x.value, np.array([1.0, 1.0]), atol=1e-4) + + def test_hs071_lbfgs(self): + """Test HS071 problem with L-BFGS - standard NLP test problem with constraints.""" + x = cp.Variable(4, bounds=[0, 6]) + x.value = np.array([1.0, 5.0, 5.0, 1.0]) + objective = cp.Minimize(x[0]*x[3]*(x[0] + x[1] + x[2]) + x[2]) + + constraints = [ + x[0]*x[1]*x[2]*x[3] >= 25, + cp.sum(cp.square(x)) == 40, + ] + problem = cp.Problem(objective, constraints) + problem.solve(solver=cp.IPOPT, nlp=True, + hessian_approximation='limited-memory') + assert problem.status == cp.OPTIMAL + # L-BFGS may converge to a slightly different point, use looser tolerance + expected = np.array([0.75450865, 4.63936861, 3.78856881, 1.88513184]) + assert np.allclose(x.value, expected, atol=1e-3) + + def test_portfolio_lbfgs(self): + """Test portfolio optimization with L-BFGS - quadratic objective.""" + r = np.array([0.026002150277777, 0.008101316405671, 0.073715909491990]) + Q = np.array([ + [0.018641039983891, 0.003598532927677, 0.001309759253660], + [0.003598532927677, 0.006436938322676, 0.004887265158407], + [0.001309759253660, 0.004887265158407, 0.068682765454814], + ]) + x = cp.Variable(3) + x.value = np.array([10.0, 10.0, 10.0]) + variance = cp.quad_form(x, Q) + expected_return = r @ x + problem = cp.Problem( + cp.Minimize(variance), + [ + cp.sum(x) <= 1000, + expected_return >= 50, + x >= 0 + ] + ) + problem.solve(solver=cp.IPOPT, nlp=True, + hessian_approximation='limited-memory') + assert problem.status == cp.OPTIMAL + expected = np.array([4.97045504e+02, 0.0, 5.02954496e+02]) + assert np.allclose(x.value, expected, atol=1e-3) + + def test_analytic_center_lbfgs(self): + """Test analytic center problem with L-BFGS - log-barrier problem.""" + np.random.seed(0) + m, n = 50, 4 + b = np.ones(m) + rand = np.random.randn(m - 2*n, n) + A = np.vstack((rand, np.eye(n), np.eye(n) * -1)) + + x = cp.Variable(n) + objective = cp.Minimize(-cp.sum(cp.log(b - A @ x))) + problem = cp.Problem(objective, []) + problem.solve(solver=cp.IPOPT, nlp=True, + hessian_approximation='limited-memory') + assert problem.status == cp.OPTIMAL + + def test_exact_vs_lbfgs_solution_quality(self): + """Compare solution quality between exact Hessian and L-BFGS.""" + x = cp.Variable(2, name='x') + objective = cp.Minimize((1 - x[0])**2 + 100 * (x[1] - x[0]**2)**2) + + # Solve with exact Hessian + problem_exact = cp.Problem(objective, []) + problem_exact.solve(solver=cp.IPOPT, nlp=True, + hessian_approximation='exact') + x_exact = x.value.copy() + + # Solve with L-BFGS + problem_lbfgs = cp.Problem(objective, []) + problem_lbfgs.solve(solver=cp.IPOPT, nlp=True, + hessian_approximation='limited-memory') + x_lbfgs = x.value.copy() + + # Both should converge to the same solution + assert np.allclose(x_exact, x_lbfgs, atol=1e-4) + # Both should be close to [1, 1] + assert np.allclose(x_exact, np.array([1.0, 1.0]), atol=1e-4) + assert np.allclose(x_lbfgs, np.array([1.0, 1.0]), atol=1e-4) + + def test_large_scale_lbfgs(self): + """Test L-BFGS on a larger problem where it's more useful.""" + np.random.seed(42) + n = 100 + + # Create a simple quadratic problem with better scaling + Q = np.random.randn(n, n) + Q = Q.T @ Q / n # Make positive semidefinite and scale + c = np.random.randn(n) / n + + x = cp.Variable(n) + # Initialize to a feasible point + x.value = np.ones(n) / n + objective = cp.Minimize(0.5 * cp.quad_form(x, Q) + c @ x) + constraints = [cp.sum(x) == 1, x >= 0] + problem = cp.Problem(objective, constraints) + + # Solve with L-BFGS + problem.solve(solver=cp.IPOPT, nlp=True, + hessian_approximation='limited-memory', + print_level=0) + assert problem.status == cp.OPTIMAL + + def test_socp_lbfgs(self): + """Test second-order cone problem with L-BFGS.""" + x = cp.Variable(3) + y = cp.Variable() + + objective = cp.Minimize(3 * x[0] + 2 * x[1] + x[2]) + constraints = [ + cp.norm(x, 2) <= y, + x[0] + x[1] + 3*x[2] >= 1.0, + y <= 5 + ] + + problem = cp.Problem(objective, constraints) + problem.solve(solver=cp.IPOPT, nlp=True, + hessian_approximation='limited-memory') + assert problem.status == cp.OPTIMAL + assert np.allclose(objective.value, -13.548638814247532, atol=1e-3) + + def test_constrained_log_lbfgs(self): + """Test constrained log problem with L-BFGS.""" + np.random.seed(123) + n = 20 + + x = cp.Variable(n, pos=True) + x.value = np.ones(n) / n + + objective = cp.Minimize(-cp.sum(cp.log(x))) + constraints = [cp.sum(x) == 1] + problem = cp.Problem(objective, constraints) + + problem.solve(solver=cp.IPOPT, nlp=True, + hessian_approximation='limited-memory') + assert problem.status == cp.OPTIMAL + # Optimal solution is uniform: x_i = 1/n + assert np.allclose(x.value, np.ones(n) / n, atol=1e-4) + + def test_entropy_lbfgs(self): + """Test entropy minimization with L-BFGS (nonconvex).""" + np.random.seed(0) + n = 10 + q = cp.Variable((n, ), nonneg=True) + q.value = np.random.rand(n) + q.value = q.value / np.sum(q.value) + obj = cp.sum(cp.entr(q)) + constraints = [cp.sum(q) == 1] + problem = cp.Problem(cp.Minimize(obj), constraints) + problem.solve(solver=cp.IPOPT, nlp=True, + hessian_approximation='limited-memory') + # Minimum entropy distribution is concentrated on one point + assert np.sum(q.value > 1e-8) == 1 diff --git a/cvxpy/tests/nlp_tests/test_risk_parity.py b/cvxpy/tests/nlp_tests/test_risk_parity.py new file mode 100644 index 0000000000..92f2254aaa --- /dev/null +++ b/cvxpy/tests/nlp_tests/test_risk_parity.py @@ -0,0 +1,107 @@ +import numpy as np +import numpy.linalg as LA +import pytest + +import cvxpy as cp +from cvxpy.reductions.solvers.defines import INSTALLED_SOLVERS +from cvxpy.reductions.solvers.nlp_solvers.nlp_solver import DerivativeChecker + +np.random.seed(0) + +@pytest.fixture +def Sigma(): + return 1e-5 * np.array([ + [41.16, 22.03, 18.64, -4.74, 6.27, 10.1 , 14.52, 3.18], + [22.03, 58.57, 32.92, -5.04, 4.02, 3.7 , 26.76, 2.17], + [18.64, 32.92, 81.02, 0.53, 6.05, 2.02, 25.52, 1.56], + [-4.74, -5.04, 0.53, 20.6 , 2.52, 0.57, 0.2 , 3.6 ], + [6.27, 4.02, 6.05, 2.52, 10.13, 2.59, 4.32, 3.13], + [10.1 , 3.7 , 2.02, 0.57, 2.59, 22.89, 3.97, 3.26], + [14.52, 26.76, 25.52, 0.2 , 4.32, 3.97, 29.91, 3.25], + [3.18, 2.17, 1.56, 3.6 , 3.13, 3.26, 3.25, 13.63] + ]) + +@pytest.mark.skipif('IPOPT' not in INSTALLED_SOLVERS, reason='IPOPT is not installed.') +class TestRiskParity: + def test_vanilla_risk_parity_formulation_one(self, Sigma): + n = 8 + risk_target = (1 / n) * np.ones(n) + + w = cp.Variable((n,), nonneg=True, name='w') + t = cp.Variable((n,), name='t') + constraints = [cp.sum(w) == 1, t == Sigma @ w] + + term1 = cp.sum(cp.multiply(cp.square(w), cp.square(t))) / cp.quad_form(w, Sigma) + term2 = (LA.norm(risk_target) ** 2) * cp.quad_form(w, Sigma) + term3 = - 2 * cp.sum(cp.multiply(risk_target, cp.multiply(w, t))) + obj = cp.Minimize(term1 + term2 + term3) + problem = cp.Problem(obj, constraints) + problem.solve(solver=cp.IPOPT, nlp=True, verbose=True, derivative_test='none') + + risk_contributions = w.value * (Sigma @ w.value) + risk_contributions /= np.sum(risk_contributions) + assert np.linalg.norm(risk_contributions - risk_target) < 1e-5 + + checker = DerivativeChecker(problem) + checker.run_and_assert() + + # we do not expand the objective, and use square roots + def test_vanilla_risk_parity_formulation_two(self): + pass + + # we expand the objective manually to get rid of the square root + def test_group_risk_parity_formulation_one(self, Sigma): + n = 8 + b = np.array([0.4, 0.6]) + + w = cp.Variable((n, ), nonneg=True, name='w') + t = cp.Variable((n, ), name='t') + constraints = [cp.sum(w) == 1, t == Sigma @ w] + w.value = np.ones(n) / n + groups = [[0, 1, 5], [3, 4, 2, 6, 7]] + + term1 = 0 + term2 = 0 + term3 = 0 + + for k, g in enumerate(groups): + term1 += cp.square(cp.sum(cp.multiply(w[g], t[g]))) / cp.quad_form(w, Sigma) + term2 += (LA.norm(b[k]) ** 2) * cp.quad_form(w, Sigma) + term3 += - 2 * b[k] * cp.sum(cp.multiply(w[g], t[g])) + + obj = cp.Minimize(term1 + term2 + term3) + problem = cp.Problem(obj, constraints) + problem.solve(solver=cp.IPOPT, nlp=True, verbose=True, derivative_test='none') + risk_contributions = w.value * (Sigma @ w.value) + risk_contributions /= np.sum(risk_contributions) + risk_contributions = np.array([np.sum(risk_contributions[g]) for g in groups]) + assert np.linalg.norm(risk_contributions - b) < 1e-5 + + checker = DerivativeChecker(problem) + checker.run_and_assert() + + # other formulation + def test_group_risk_parity_formulation_two(self, Sigma): + n = 8 + b = np.array([0.4, 0.6]) + + w = cp.Variable((n, ), nonneg=True, name='w') + t = cp.Variable((n, ), name='t') + constraints = [cp.sum(w) == 1, t == Sigma @ w] + w.value = np.ones(n) / n + groups = [[0, 1, 5], [3, 4, 2, 6, 7]] + + obj = 0 + for k, g in enumerate(groups): + obj += cp.square(cp.sum(cp.multiply(w[g], t[g])) / cp.quad_form(w, Sigma) - b[k]) + + problem = cp.Problem(cp.Minimize(obj), constraints) + problem.solve(solver=cp.IPOPT, nlp=True, verbose=True, derivative_test='none') + + risk_contributions = w.value * (Sigma @ w.value) + risk_contributions /= np.sum(risk_contributions) + risk_contributions = np.array([np.sum(risk_contributions[g]) for g in groups]) + assert np.linalg.norm(risk_contributions - b) < 1e-5 + + checker = DerivativeChecker(problem) + checker.run_and_assert() \ No newline at end of file diff --git a/cvxpy/tests/nlp_tests/test_scalar_and_matrix_problems.py b/cvxpy/tests/nlp_tests/test_scalar_and_matrix_problems.py new file mode 100644 index 0000000000..4d8e62b046 --- /dev/null +++ b/cvxpy/tests/nlp_tests/test_scalar_and_matrix_problems.py @@ -0,0 +1,291 @@ +import numpy as np +import pytest + +import cvxpy as cp +from cvxpy.reductions.solvers.defines import INSTALLED_SOLVERS +from cvxpy.reductions.solvers.nlp_solvers.nlp_solver import DerivativeChecker + + +@pytest.mark.skipif('IPOPT' not in INSTALLED_SOLVERS, reason='IPOPT is not installed.') +class TestScalarProblems(): + + def test_exp(self): + x = cp.Variable() + prob = cp.Problem(cp.Minimize(cp.exp(x)), [x >= 4]) + prob.solve(nlp=True, solver=cp.IPOPT) + assert prob.status == cp.OPTIMAL + checker = DerivativeChecker(prob) + checker.run_and_assert() + + def test_entropy(self): + x = cp.Variable() + prob = cp.Problem(cp.Maximize(cp.entr(x)), [x >= 0.1]) + prob.solve(nlp=True, solver=cp.IPOPT) + assert prob.status == cp.OPTIMAL + checker = DerivativeChecker(prob) + checker.run_and_assert() + + def test_KL(self): + p = cp.Variable() + q = cp.Variable() + prob = cp.Problem(cp.Minimize(cp.kl_div(p, q)), [p >= 0.1, q >= 0.1, p + q == 2]) + prob.solve(nlp=True, solver=cp.IPOPT) + assert prob.status == cp.OPTIMAL + checker = DerivativeChecker(prob) + checker.run_and_assert() + + def test_KL_matrix(self): + Y = cp.Variable((3, 3)) + X = cp.Variable((3, 3)) + prob = cp.Problem(cp.Minimize(cp.sum(cp.kl_div(X, Y))), + [X >= 0.1, Y >= 0.1, cp.sum(X) + cp.sum(Y) == 6]) + prob.solve(nlp=True, solver=cp.IPOPT) + assert prob.status == cp.OPTIMAL + checker = DerivativeChecker(prob) + checker.run_and_assert() + + def test_entropy_matrix(self): + x = cp.Variable((3, 2)) + prob = cp.Problem(cp.Maximize(cp.sum(cp.entr(x))), [x >= 0.1]) + prob.solve(nlp=True, solver=cp.IPOPT) + assert prob.status == cp.OPTIMAL + checker = DerivativeChecker(prob) + checker.run_and_assert() + + def test_logistic(self): + x = cp.Variable() + prob = cp.Problem(cp.Minimize(cp.logistic(x)), [x >= 0.4]) + prob.solve(nlp=True, solver=cp.IPOPT) + assert prob.status == cp.OPTIMAL + checker = DerivativeChecker(prob) + checker.run_and_assert() + + def test_logistic_matrix(self): + x = cp.Variable((3, 2)) + prob = cp.Problem(cp.Minimize(cp.sum(cp.logistic(x))), [x >= 0.4]) + prob.solve(nlp=True, solver=cp.IPOPT) + assert prob.status == cp.OPTIMAL + checker = DerivativeChecker(prob) + checker.run_and_assert() + + def test_power(self): + x = cp.Variable() + prob = cp.Problem(cp.Minimize(cp.power(x, 3)), [x >= 2]) + prob.solve(nlp=True, solver=cp.IPOPT) + assert prob.status == cp.OPTIMAL + checker = DerivativeChecker(prob) + checker.run_and_assert() + + def test_power_matrix(self): + x = cp.Variable((3, 2)) + prob = cp.Problem(cp.Minimize(cp.sum(cp.power(x, 3))), [x >= 2]) + prob.solve(nlp=True, solver=cp.IPOPT) + assert prob.status == cp.OPTIMAL + checker = DerivativeChecker(prob) + checker.run_and_assert() + + def test_power_fractional(self): + x = cp.Variable() + prob = cp.Problem(cp.Minimize(cp.power(x, 1.5)), [x >= 2]) + prob.solve(nlp=True, solver=cp.IPOPT) + assert prob.status == cp.OPTIMAL + checker = DerivativeChecker(prob) + checker.run_and_assert() + + x = cp.Variable((3, 2)) + prob = cp.Problem(cp.Minimize(cp.sum(cp.power(x, 0.6))), [x >= 2]) + prob.solve(nlp=True, solver=cp.IPOPT) + assert prob.status == cp.OPTIMAL + checker = DerivativeChecker(prob) + checker.run_and_assert() + + def test_power_fractional_matrix(self): + x = cp.Variable((3, 2)) + prob = cp.Problem(cp.Minimize(cp.sum(cp.power(x, 1.5))), [x >= 2]) + prob.solve(nlp=True, solver=cp.IPOPT) + assert prob.status == cp.OPTIMAL + checker = DerivativeChecker(prob) + checker.run_and_assert() + + x = cp.Variable((3, 2)) + prob = cp.Problem(cp.Minimize(cp.sum(cp.power(x, 0.6))), [x >= 2]) + prob.solve(nlp=True, solver=cp.IPOPT) + assert prob.status == cp.OPTIMAL + checker = DerivativeChecker(prob) + checker.run_and_assert() + + def test_scalar_trig(self): + x = cp.Variable() + prob = cp.Problem(cp.Minimize(cp.tan(x)), [x >= 0.1]) + prob.solve(nlp=True, solver=cp.IPOPT) + assert prob.status == cp.OPTIMAL + checker = DerivativeChecker(prob) + checker.run_and_assert() + + prob = cp.Problem(cp.Minimize(cp.sin(x)), [x >= 0.1]) + prob.solve(nlp=True, solver=cp.IPOPT) + assert prob.status == cp.OPTIMAL + checker = DerivativeChecker(prob) + checker.run_and_assert() + + prob = cp.Problem(cp.Minimize(cp.cos(x)), [x >= 0.1]) + prob.solve(nlp=True, solver=cp.IPOPT) + assert prob.status == cp.OPTIMAL + checker = DerivativeChecker(prob) + checker.run_and_assert() + + def test_matrix_trig(self): + x = cp.Variable((3, 2)) + prob = cp.Problem(cp.Minimize(cp.sum(cp.tan(x))), [x >= 0.1]) + prob.solve(nlp=True, solver=cp.IPOPT) + assert prob.status == cp.OPTIMAL + checker = DerivativeChecker(prob) + checker.run_and_assert() + + prob = cp.Problem(cp.Minimize(cp.sum(cp.sin(x))), [x >= 0.1]) + prob.solve(nlp=True, solver=cp.IPOPT) + assert prob.status == cp.OPTIMAL + checker = DerivativeChecker(prob) + checker.run_and_assert() + + prob = cp.Problem(cp.Minimize(cp.sum(cp.cos(x))), [x >= 0.1]) + prob.solve(nlp=True, solver=cp.IPOPT) + assert prob.status == cp.OPTIMAL + checker = DerivativeChecker(prob) + checker.run_and_assert() + + def test_matrix_hyperbolic(self): + x = cp.Variable((3, 2)) + prob = cp.Problem(cp.Minimize(cp.sum(cp.sinh(x))), [x >= 0.1]) + prob.solve(nlp=True, solver=cp.IPOPT) + assert prob.status == cp.OPTIMAL + checker = DerivativeChecker(prob) + checker.run_and_assert() + + prob = cp.Problem(cp.Minimize(cp.sum(cp.tanh(x))), [x >= 0.1]) + prob.solve(nlp=True, solver=cp.IPOPT) + assert prob.status == cp.OPTIMAL + checker = DerivativeChecker(prob) + checker.run_and_assert() + + def test_scalar_hyperbolic(self): + x = cp.Variable() + prob = cp.Problem(cp.Minimize(cp.sinh(x)), [x >= 0.1]) + prob.solve(nlp=True, solver=cp.IPOPT) + assert prob.status == cp.OPTIMAL + checker = DerivativeChecker(prob) + checker.run_and_assert() + + prob = cp.Problem(cp.Minimize(cp.tanh(x)), [x >= 0.1]) + prob.solve(nlp=True, solver=cp.IPOPT) + assert prob.status == cp.OPTIMAL + checker = DerivativeChecker(prob) + checker.run_and_assert() + + def test_xexp(self): + x = cp.Variable() + prob = cp.Problem(cp.Minimize(cp.xexp(x)), [x >= 0.1]) + prob.solve(nlp=True, solver=cp.IPOPT) + assert prob.status == cp.OPTIMAL + checker = DerivativeChecker(prob) + checker.run_and_assert() + + x = cp.Variable((3, 2)) + prob = cp.Problem(cp.Minimize(cp.sum(cp.xexp(x))), [x >= 0.1]) + prob.solve(nlp=True, solver=cp.IPOPT) + assert prob.status == cp.OPTIMAL + checker = DerivativeChecker(prob) + checker.run_and_assert() + + def test_scalar_quad_form(self): + x = cp.Variable((1, )) + P = np.array([[3]]) + prob = cp.Problem(cp.Minimize(cp.quad_form(x, P)), [x >= 1]) + prob.solve(nlp=True, solver=cp.IPOPT) + assert prob.status == cp.OPTIMAL + checker = DerivativeChecker(prob) + checker.run_and_assert() + + def test_scalar_quad_over_lin(self): + x = cp.Variable() + y = cp.Variable() + prob = cp.Problem(cp.Minimize(cp.quad_over_lin(x, y)), [x >= 1, y <= 1]) + prob.solve(nlp=True, solver=cp.IPOPT) + assert prob.status == cp.OPTIMAL + checker = DerivativeChecker(prob) + checker.run_and_assert() + + def test_matrix_quad_over_lin(self): + x = cp.Variable((3, 2)) + y = cp.Variable((1, )) + prob = cp.Problem(cp.Minimize(cp.quad_over_lin(x, y)), [x >= 1, y <= 1]) + prob.solve(nlp=True, solver=cp.IPOPT) + assert prob.status == cp.OPTIMAL + checker = DerivativeChecker(prob) + checker.run_and_assert() + + y = cp.Variable() + prob = cp.Problem(cp.Minimize(cp.quad_over_lin(x, y)), [x >= 1, y <= 1]) + prob.solve(nlp=True, solver=cp.IPOPT) + assert prob.status == cp.OPTIMAL + checker = DerivativeChecker(prob) + checker.run_and_assert() + + def test_rel_entr_both_scalar_variables(self): + x = cp.Variable() + y = cp.Variable() + prob = cp.Problem(cp.Minimize(cp.rel_entr(x, y)), + [x >= 0.1, y >= 0.1, x <= 2, y <= 2]) + prob.solve(nlp=True, solver=cp.IPOPT) + assert prob.status == cp.OPTIMAL + checker = DerivativeChecker(prob) + checker.run_and_assert() + + x = cp.Variable((1, )) + y = cp.Variable((1, )) + prob = cp.Problem(cp.Minimize(cp.rel_entr(x, y)), + [x >= 0.1, y >= 0.1, x <= 2, y <= 2]) + prob.solve(nlp=True, solver=cp.IPOPT) + assert prob.status == cp.OPTIMAL + checker = DerivativeChecker(prob) + checker.run_and_assert() + + def test_rel_entr_matrix_variable_and_scalar_variable(self): + x = cp.Variable((3, 2)) + y = cp.Variable() + prob = cp.Problem(cp.Minimize(cp.sum(cp.rel_entr(x, y))), + [x >= 0.1, y >= 0.1, x <= 2, y <= 2]) + prob.solve(nlp=True, solver=cp.IPOPT) + assert prob.status == cp.OPTIMAL + checker = DerivativeChecker(prob) + checker.run_and_assert() + + def test_rel_entr_scalar_variable_and_matrix_variable(self): + x = cp.Variable() + y = cp.Variable((3, 2)) + prob = cp.Problem(cp.Minimize(cp.sum(cp.rel_entr(x, y))), + [x >= 0.1, y >= 0.1, x <= 2, y <= 2]) + prob.solve(nlp=True, solver=cp.IPOPT) + assert prob.status == cp.OPTIMAL + checker = DerivativeChecker(prob) + checker.run_and_assert() + + def test_rel_entr_both_matrix_variables(self): + x = cp.Variable((3, 2)) + y = cp.Variable((3, 2)) + prob = cp.Problem(cp.Minimize(cp.sum(cp.rel_entr(x, y))), + [x >= 0.1, y >= 0.1, x <= 2, y <= 2]) + prob.solve(nlp=True, solver=cp.IPOPT) + assert prob.status == cp.OPTIMAL + checker = DerivativeChecker(prob) + checker.run_and_assert() + + def test_rel_entr_both_vector_variables(self): + x = cp.Variable((3, )) + y = cp.Variable((3, )) + prob = cp.Problem(cp.Minimize(cp.sum(cp.rel_entr(x, y))), + [x >= 0.1, y >= 0.1, x <= 2, y <= 2]) + prob.solve(nlp=True, solver=cp.IPOPT) + assert prob.status == cp.OPTIMAL + checker = DerivativeChecker(prob) + checker.run_and_assert() diff --git a/cvxpy/tests/nlp_tests/test_sum.py b/cvxpy/tests/nlp_tests/test_sum.py new file mode 100644 index 0000000000..2764576143 --- /dev/null +++ b/cvxpy/tests/nlp_tests/test_sum.py @@ -0,0 +1,88 @@ +import numpy as np +import pytest + +import cvxpy as cp +from cvxpy.reductions.solvers.defines import INSTALLED_SOLVERS +from cvxpy.reductions.solvers.nlp_solvers.nlp_solver import DerivativeChecker + + +@pytest.mark.skipif('IPOPT' not in INSTALLED_SOLVERS, reason='IPOPT is not installed.') +class TestSumIPOPT: + """Test solving sum problems with IPOPT.""" + + def test_sum_without_axis(self): + x = cp.Variable((2, 1)) + obj = cp.Minimize((cp.sum(x) - 3)**2) + constr = [x <= 1] + prob = cp.Problem(obj, constr) + prob.solve(solver=cp.IPOPT, nlp=True) + assert np.allclose(x.value, [[1.0], [1.0]]) + + checker = DerivativeChecker(prob) + checker.run_and_assert() + + + def test_sum_with_axis(self): + """Test sum with axis parameter.""" + X = cp.Variable((2, 3)) + obj = cp.Minimize(cp.sum((cp.sum(X, axis=1) - 4)**2)) + constr = [X >= 0, X <= 1] + prob = cp.Problem(obj, constr) + prob.solve(solver=cp.IPOPT, nlp=True) + expected = np.full((2, 3), 1) + assert np.allclose(X.value, expected) + + checker = DerivativeChecker(prob) + checker.run_and_assert() + + def test_two_sum_with_axis(self): + """Test sum with axis parameter.""" + np.random.seed(0) + X = cp.Variable((2, 3)) + A = np.random.rand(4, 2) + obj = cp.Minimize(cp.prod(cp.sum(A @ X, axis=1))) + constr = [X >= 0, X <= 1] + prob = cp.Problem(obj, constr) + prob.solve(solver=cp.IPOPT, nlp=True) + checker = DerivativeChecker(prob) + checker.run_and_assert() + + def test_sum_with_other_axis(self): + """Test sum with axis parameter.""" + X = cp.Variable((2, 3)) + obj = cp.Minimize(cp.sum((cp.sum(X, axis=0) - 4)**2)) + constr = [X >= 0, X <= 1] + prob = cp.Problem(obj, constr) + prob.solve(solver=cp.IPOPT, nlp=True) + expected = np.full((2, 3), 1) + assert np.allclose(X.value, expected) + + checker = DerivativeChecker(prob) + checker.run_and_assert() + + def test_two_sum_with_other_axis(self): + """Test sum with axis parameter.""" + np.random.seed(0) + X = cp.Variable((2, 3)) + A = np.random.rand(4, 2) + obj = cp.Minimize(cp.prod(cp.sum(A @ X, axis=0))) + constr = [X >= 0, X <= 1] + prob = cp.Problem(obj, constr) + prob.solve(solver=cp.IPOPT, nlp=True) + checker = DerivativeChecker(prob) + checker.run_and_assert() + + def test_sum_matrix_arg(self): + np.random.seed(0) + n, m, k = 40, 20, 4 + A = np.random.rand(n, k) @ np.random.rand(k, m) + T = cp.Variable((n, m), name='T') + obj = cp.sum(cp.multiply(A, T)) + constraints = [T >= 1, T <= 2] + problem = cp.Problem(cp.Minimize(obj), constraints) + problem.solve(solver=cp.IPOPT, nlp=True, verbose=True, derivative_test='none') + assert(np.allclose(T.value, 1)) + assert problem.status == cp.OPTIMAL + + checker = DerivativeChecker(problem) + checker.run_and_assert() \ No newline at end of file diff --git a/cvxpy/tests/test_conic_solvers.py b/cvxpy/tests/test_conic_solvers.py index 9aeca5610c..5e3885156f 100644 --- a/cvxpy/tests/test_conic_solvers.py +++ b/cvxpy/tests/test_conic_solvers.py @@ -548,6 +548,7 @@ def test_clarabel_sdp_2(self) -> None: sth.check_complementarity(places) sth.check_dual_domains(places) + @unittest.skipUnless('CUCLARABEL' in INSTALLED_SOLVERS, 'CLARABEL is not installed.') class TestCuClarabel(BaseTest): diff --git a/cvxpy/tests/test_diff_engine_conic.py b/cvxpy/tests/test_diff_engine_conic.py new file mode 100644 index 0000000000..e918790fab --- /dev/null +++ b/cvxpy/tests/test_diff_engine_conic.py @@ -0,0 +1,367 @@ +"""Tests for the diff engine conic backend. + +Verifies that the DIFF_ENGINE canon backend produces identical results +to the tensor-based SCIPY backend for LP, QP, and SOCP problems. + +Copyright 2025, the CVXPY developers + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import numpy as np +import scipy.sparse as sp + +import cvxpy as cp +from cvxpy.tests.base_test import BaseTest + + +def _dense(M): + """Convert sparse matrix to dense array.""" + if sp.issparse(M): + return M.toarray() + return np.asarray(M) + + +class TestDiffEngineConic(BaseTest): + """Tests comparing DIFF_ENGINE backend against SCIPY backend.""" + + def _get_data(self, prob, backend): + """Get problem data with a specific backend (clearing cache).""" + prob._cache = type(prob._cache)() + data, _, _ = prob.get_problem_data(cp.CLARABEL, canon_backend=backend) + return data + + def _compare_data(self, prob, atol=1e-10): + """Compare problem data from DIFF_ENGINE vs SCIPY backends.""" + data_de = self._get_data(prob, 'DIFF_ENGINE') + data_sp = self._get_data(prob, 'SCIPY') + + A_de = _dense(data_de['A']) + A_sp = _dense(data_sp['A']) + np.testing.assert_allclose(A_de, A_sp, atol=atol, + err_msg="A matrices don't match") + np.testing.assert_allclose(data_de['b'], data_sp['b'], atol=atol, + err_msg="b vectors don't match") + np.testing.assert_allclose(data_de['c'], data_sp['c'], atol=atol, + err_msg="c vectors don't match") + + if 'P' in data_de and 'P' in data_sp: + P_de = _dense(data_de['P']) + P_sp = _dense(data_sp['P']) + np.testing.assert_allclose(P_de, P_sp, atol=atol, + err_msg="P matrices don't match") + + # ---- Matrix comparison tests ---- + + def test_lp_matrix_comparison(self) -> None: + """Compare A, b, c matrices for a simple LP.""" + np.random.seed(42) + n = 5 + x = cp.Variable(n) + A_param = cp.Parameter((3, n)) + b_param = cp.Parameter(3) + c = np.random.randn(n) + + prob = cp.Problem(cp.Minimize(c @ x), [A_param @ x <= b_param, x >= 0]) + A_param.value = np.random.randn(3, n) + b_param.value = np.random.randn(3) + 10 + + self._compare_data(prob) + + def test_lp_with_equality(self) -> None: + """LP with both equality and inequality constraints.""" + np.random.seed(123) + n = 4 + x = cp.Variable(n) + A_param = cp.Parameter((2, n)) + b_param = cp.Parameter(2) + c = np.random.randn(n) + + prob = cp.Problem(cp.Minimize(c @ x), + [A_param @ x <= b_param, cp.sum(x) == 1, x >= 0]) + A_param.value = np.random.randn(2, n) + b_param.value = np.random.randn(2) + 10 + + self._compare_data(prob) + + def test_qp_matrix_comparison(self) -> None: + """Compare P, q, A, b matrices for a QP.""" + np.random.seed(42) + n = 5 + x = cp.Variable(n) + Q_diag = cp.Parameter(n, nonneg=True) + A_param = cp.Parameter((3, n)) + b_param = cp.Parameter(3) + c = np.random.randn(n) + + obj = 0.5 * cp.sum(cp.multiply(Q_diag, cp.square(x))) + c @ x + prob = cp.Problem(cp.Minimize(obj), + [A_param @ x <= b_param, x >= -5, x <= 5]) + + Q_diag.value = np.abs(np.random.randn(n)) + 0.1 + A_param.value = np.random.randn(3, n) + b_param.value = np.random.randn(3) + 10 + + self._compare_data(prob, atol=1e-8) + + def test_no_params(self) -> None: + """Problem with no parameters.""" + np.random.seed(42) + n = 3 + x = cp.Variable(n) + A = np.random.randn(2, n) + b = np.random.randn(2) + 5 + + prob = cp.Problem(cp.Minimize(cp.sum(x)), [A @ x <= b, x >= 0]) + self._compare_data(prob) + + # ---- End-to-end solve tests ---- + + def test_lp_solve(self) -> None: + """Solve an LP with both backends and compare results.""" + np.random.seed(42) + n = 5 + x = cp.Variable(n) + A_param = cp.Parameter((3, n)) + b_param = cp.Parameter(3) + c = np.random.randn(n) + + prob = cp.Problem(cp.Minimize(c @ x), [A_param @ x <= b_param, x >= 0]) + A_param.value = np.random.randn(3, n) + b_param.value = np.random.randn(3) + 10 + + prob.solve(solver=cp.CLARABEL, canon_backend='DIFF_ENGINE') + val_de = prob.value + x_de = x.value.copy() + + prob._cache = type(prob._cache)() + prob.solve(solver=cp.CLARABEL, canon_backend='SCIPY') + val_sp = prob.value + + self.assertAlmostEqual(val_de, val_sp, places=5) + np.testing.assert_allclose(x_de, x.value, atol=1e-5) + + def test_qp_solve(self) -> None: + """Solve a QP with both backends and compare results.""" + np.random.seed(42) + n = 5 + x = cp.Variable(n) + Q_diag = cp.Parameter(n, nonneg=True) + c = np.random.randn(n) + + obj = 0.5 * cp.sum(cp.multiply(Q_diag, cp.square(x))) + c @ x + prob = cp.Problem(cp.Minimize(obj), [x >= -5, x <= 5]) + + Q_diag.value = np.abs(np.random.randn(n)) + 0.1 + + prob.solve(solver=cp.CLARABEL, canon_backend='DIFF_ENGINE') + val_de = prob.value + + prob._cache = type(prob._cache)() + prob.solve(solver=cp.CLARABEL, canon_backend='SCIPY') + val_sp = prob.value + + self.assertAlmostEqual(val_de, val_sp, places=5) + + # ---- Parametric warm-path tests ---- + + def test_warm_path_lp(self) -> None: + """Test parameter updates on the warm path for LP.""" + np.random.seed(42) + n = 6 + x = cp.Variable(n) + A_param = cp.Parameter((3, n)) + b_param = cp.Parameter(3) + c = np.random.randn(n) + + prob = cp.Problem(cp.Minimize(c @ x), [A_param @ x <= b_param, x >= 0]) + + # Cold path + A_param.value = np.random.randn(3, n) + b_param.value = np.abs(np.random.randn(3)) + 5 + prob.solve(solver=cp.CLARABEL, canon_backend='DIFF_ENGINE') + + # Warm path with new parameters + for trial in range(5): + np.random.seed(trial * 100 + 1000) + A_param.value = np.random.randn(3, n) + b_param.value = np.abs(np.random.randn(3)) + 5 + + prob.solve(solver=cp.CLARABEL, canon_backend='DIFF_ENGINE') + val_de = prob.value + + # Compare with scipy + A_save = A_param.value.copy() + b_save = b_param.value.copy() + prob._cache = type(prob._cache)() + A_param.value = A_save + b_param.value = b_save + prob.solve(solver=cp.CLARABEL, canon_backend='SCIPY') + val_sp = prob.value + + np.testing.assert_allclose(val_de, val_sp, atol=1e-5, + err_msg=f"Warm path trial {trial}") + + # Restore DE cache for next warm solve + prob._cache = type(prob._cache)() + np.random.seed(42) + A_param.value = np.random.randn(3, n) + b_param.value = np.abs(np.random.randn(3)) + 5 + prob.solve(solver=cp.CLARABEL, canon_backend='DIFF_ENGINE') + + def test_warm_path_qp(self) -> None: + """Test parameter updates on the warm path for QP.""" + np.random.seed(42) + n = 5 + x = cp.Variable(n) + Q_diag = cp.Parameter(n, nonneg=True) + A_param = cp.Parameter((3, n)) + b_param = cp.Parameter(3) + c = np.random.randn(n) + + obj = 0.5 * cp.sum(cp.multiply(Q_diag, cp.square(x))) + c @ x + prob = cp.Problem(cp.Minimize(obj), + [A_param @ x <= b_param, x >= -5, x <= 5]) + + # Cold path + Q_diag.value = np.abs(np.random.randn(n)) + 0.1 + A_param.value = np.random.randn(3, n) + b_param.value = np.abs(np.random.randn(3)) + 5 + prob.solve(solver=cp.CLARABEL, canon_backend='DIFF_ENGINE') + + # Warm path + for trial in range(3): + np.random.seed(trial * 100 + 2000) + Q_diag.value = np.abs(np.random.randn(n)) + 0.1 + A_param.value = np.random.randn(3, n) + b_param.value = np.abs(np.random.randn(3)) + 5 + + prob.solve(solver=cp.CLARABEL, canon_backend='DIFF_ENGINE') + val_de = prob.value + + Q_save = Q_diag.value.copy() + A_save = A_param.value.copy() + b_save = b_param.value.copy() + prob._cache = type(prob._cache)() + Q_diag.value = Q_save + A_param.value = A_save + b_param.value = b_save + prob.solve(solver=cp.CLARABEL, canon_backend='SCIPY') + val_sp = prob.value + + np.testing.assert_allclose(val_de, val_sp, atol=1e-4, + err_msg=f"QP warm path trial {trial}") + + # Restore DE cache + prob._cache = type(prob._cache)() + Q_diag.value = np.abs(np.random.randn(n)) + 0.1 + A_param.value = np.random.randn(3, n) + b_param.value = np.abs(np.random.randn(3)) + 5 + prob.solve(solver=cp.CLARABEL, canon_backend='DIFF_ENGINE') + + # ---- Multiple parameter update tests ---- + + def test_multiple_param_updates(self) -> None: + """Loop through many parameter configurations.""" + np.random.seed(42) + n = 5 + x = cp.Variable(n) + A_param = cp.Parameter((3, n)) + b_param = cp.Parameter(3) + c_param = cp.Parameter(n) + + prob = cp.Problem(cp.Minimize(c_param @ x), + [A_param @ x <= b_param, x >= 0]) + + for trial in range(15): + np.random.seed(trial + 42) + A_param.value = np.random.randn(3, n) + b_param.value = np.abs(np.random.randn(3)) + 5 + c_param.value = np.random.randn(n) + + # DIFF_ENGINE + prob._cache = type(prob._cache)() + prob.solve(solver=cp.CLARABEL, canon_backend='DIFF_ENGINE') + val_de = prob.value + + # SCIPY + prob._cache = type(prob._cache)() + prob.solve(solver=cp.CLARABEL, canon_backend='SCIPY') + val_sp = prob.value + + np.testing.assert_allclose( + val_de, val_sp, atol=1e-5, + err_msg=f"Trial {trial}: DE={val_de}, SP={val_sp}") + + def test_scalar_param(self) -> None: + """Test with a single scalar parameter.""" + np.random.seed(42) + n = 3 + x = cp.Variable(n) + t = cp.Parameter(nonneg=True) + + prob = cp.Problem(cp.Minimize(cp.sum(x)), + [x >= 0, cp.sum(x) <= t]) + t.value = 10.0 + + prob.solve(solver=cp.CLARABEL, canon_backend='DIFF_ENGINE') + val_de = prob.value + + prob._cache = type(prob._cache)() + prob.solve(solver=cp.CLARABEL, canon_backend='SCIPY') + val_sp = prob.value + + self.assertAlmostEqual(val_de, val_sp, places=5) + + def test_objective_param(self) -> None: + """Test with parameters in the objective only.""" + np.random.seed(42) + n = 4 + x = cp.Variable(n) + c_param = cp.Parameter(n) + + prob = cp.Problem(cp.Minimize(c_param @ x), [x >= 0, cp.sum(x) == 1]) + c_param.value = np.random.randn(n) + + self._compare_data(prob) + + prob.solve(solver=cp.CLARABEL, canon_backend='DIFF_ENGINE') + val_de = prob.value + + prob._cache = type(prob._cache)() + prob.solve(solver=cp.CLARABEL, canon_backend='SCIPY') + val_sp = prob.value + + self.assertAlmostEqual(val_de, val_sp, places=5) + + def test_bounds(self) -> None: + """Test with variable bounds.""" + np.random.seed(42) + n = 4 + x = cp.Variable(n) + A_param = cp.Parameter((2, n)) + b_param = cp.Parameter(2) + + prob = cp.Problem(cp.Minimize(cp.sum(x)), + [A_param @ x <= b_param, x >= -1, x <= 5]) + A_param.value = np.random.randn(2, n) + b_param.value = np.abs(np.random.randn(2)) + 5 + + prob.solve(solver=cp.CLARABEL, canon_backend='DIFF_ENGINE') + val_de = prob.value + + prob._cache = type(prob._cache)() + prob.solve(solver=cp.CLARABEL, canon_backend='SCIPY') + val_sp = prob.value + + self.assertAlmostEqual(val_de, val_sp, places=5) diff --git a/cvxpy/utilities/citations.py b/cvxpy/utilities/citations.py index a0f3acc680..a5bf5ef3ff 100644 --- a/cvxpy/utilities/citations.py +++ b/cvxpy/utilities/citations.py @@ -598,6 +598,27 @@ } """ +CITATION_DICT["IPOPT"] = \ +""" +@article{wachter2006implementation, + title={On the implementation of a primal-dual interior point filter line search algorithm for + large-scale nonlinear programming}, + author={W{\"a}chter, Andreas and Biegler, Lorenz T}, + journal={Mathematical Programming}, + volume={106}, + } +""" + +CITATION_DICT["UNO"] = \ +""" +@unpublished{VanaretLeyffer2024, + author = {Vanaret, Charlie and Leyffer, Sven}, + title = {Implementing a unified solver for nonlinearly constrained optimization}, + year = {2024}, + note = {Submitted to Mathematical Programming Computation} +} +""" + CITATION_DICT["MOREAU"] = \ """ @misc{moreau2025, diff --git a/pyproject.toml b/pyproject.toml index 755b57d686..1d26d0f803 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,6 +55,7 @@ dependencies = [ "numpy >= 2.0.0", "scipy >= 1.13.0", "highspy >= 1.11.0", + "sparsediffpy", ] requires-python = ">=3.11" urls = {Homepage = "https://github.com/cvxpy/cvxpy"} @@ -93,6 +94,7 @@ SCS = [] XPRESS = ["xpress>=9.5"] DAQP = ["daqp"] KNITRO = ["knitro"] +# IPOPT = ["cyipopt"] testing = ["pytest", "hypothesis"] doc = ["sphinx", "sphinxcontrib.jquery", diff --git a/setup.py b/setup.py index 7fdaf7f273..bcfe75169e 100644 --- a/setup.py +++ b/setup.py @@ -52,7 +52,10 @@ def finalize_options(self) -> None: setup_versioning.write_version_py() VERSION = setup_versioning.VERSION -extensions = [setup_extensions.cvxcore, setup_extensions.sparsecholesky] +extensions = [ + setup_extensions.cvxcore, + setup_extensions.sparsecholesky, +] setup( name="cvxpy",