Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 24 additions & 3 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,14 @@ endif
endif

ifeq ("$(TEST_BENCHMARK)", "true")
TAGS := -bench=. -run=notests -cpu=1,2,4
# Check if TESTOPTIONS already contains a specific benchmark filter
ifneq (,$(findstring -bench=,$(TESTOPTIONS)))
# TESTOPTIONS contains a specific benchmark filter, don't add generic -bench=.
TAGS := -run=notests -cpu=1,2,4
else
# No specific benchmark filter, use generic -bench=. to run all benchmarks
TAGS := -bench=. -run=notests -cpu=1,2,4
endif
TESTS := $(REPOPATH)/test
endif

Expand Down Expand Up @@ -145,8 +152,8 @@ ifeq ("$(DEBUG)", "true")
DOCKER_RUN_CMD := $(DOCKER_DEBUG_ARGS) $(GOIMAGE) /go/bin/dlv --listen=:$(DEBUG_PORT) --headless=true --api-version=2 --accept-multiclient exec /test_debug.test -- $(TESTOPTIONS)
DOCKER_V2_RUN_CMD := $(DOCKER_RUN_CMD)
else
DOCKER_RUN_CMD := $(GOIMAGE) go test -timeout 120m $(GOBUILDTAGSOPT) $(TESTOPTIONS) $(TESTVERBOSEOPTIONS) $(TESTS)
DOCKER_V2_RUN_CMD := $(GOV2IMAGE) go test -timeout 120m $(GOBUILDTAGSOPT) $(TESTOPTIONS) $(TESTVERBOSEOPTIONS) -parallel $(TESTV2PARALLEL) ./tests
DOCKER_RUN_CMD := $(GOIMAGE) go test -timeout 120m $(GOBUILDTAGSOPT) $(TESTOPTIONS) $(TESTVERBOSEOPTIONS) $(TAGS) $(TESTS)
DOCKER_V2_RUN_CMD := $(GOV2IMAGE) go test -timeout 120m $(GOBUILDTAGSOPT) $(TESTOPTIONS) $(TESTVERBOSEOPTIONS) $(TAGS) -parallel $(TESTV2PARALLEL) ./tests
endif

ifeq ("$(ADD_TIMESTAMP)", "true")
Expand Down Expand Up @@ -508,6 +515,20 @@ run-benchmarks-single-vpack-no-auth:
@echo "Benchmarks: Single server, Velocypack, no authentication"
@${MAKE} TEST_MODE="single" TEST_AUTH="none" TEST_CONTENT_TYPE="vpack" TEST_BENCHMARK="true" __run_tests

run-benchmarks-v2-single-json-no-auth:
@echo "V2 Benchmarks: Single server, JSON no authentication"
@${MAKE} TEST_MODE="single" TEST_AUTH="none" TEST_CONTENT_TYPE="json" TEST_BENCHMARK="true" __run_v2_tests

# V1 Cluster benchmarks
run-benchmarks-cluster-json-no-auth:
@echo "V1 Benchmarks: Cluster server, JSON no authentication"
@${MAKE} TEST_MODE="cluster" TEST_AUTH="none" TEST_CONTENT_TYPE="json" TEST_BENCHMARK="true" __run_tests

# V2 Cluster benchmarks
run-benchmarks-v2-cluster-json-no-auth:
@echo "V2 Benchmarks: Cluster server, JSON no authentication"
@${MAKE} TEST_MODE="cluster" TEST_AUTH="none" TEST_CONTENT_TYPE="json" TEST_BENCHMARK="true" __run_v2_tests

## Lint

.PHONY: tools
Expand Down
146 changes: 146 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -41,3 +41,149 @@ Key:
* `+` Features included in the driver may be not present in the ArangoDB API.
Calls to ArangoDB may result in unexpected responses (404).
* `-` The ArangoDB version has features that are not supported by the driver.

## Running Benchmark Tests

The go-driver includes comprehensive benchmark tests to measure performance of both V1 and V2 APIs. These benchmarks help compare performance between different driver versions and HTTP protocols.

### Prerequisites

- Docker (for running ArangoDB test instances)
- Go 1.19+
- Make

### Available Benchmark Tests

#### V1 API Benchmarks
- **Collection Operations**: `BenchmarkCollectionExists`, `BenchmarkCollection`, `BenchmarkCollections`
- **Document Operations**: `BenchmarkCreateDocument`, `BenchmarkReadDocument`, `BenchmarkRemoveDocument`
- **Parallel Operations**: `BenchmarkCreateDocumentParallel`, `BenchmarkReadDocumentParallel`
- **Comprehensive CRUD**: `BenchmarkComprehensiveDocumentOperations_1K`, `BenchmarkComprehensiveDocumentOperations_10K`
- **Performance Tests**: `Benchmark_Insert`, `Benchmark_BatchInsert`

#### V2 API Benchmarks
- **Collection Operations**: `BenchmarkV2CollectionExists`, `BenchmarkV2Collection`, `BenchmarkV2Collections`
- **Document Operations**: `BenchmarkV2CreateDocument`, `BenchmarkV2ReadDocument`, `BenchmarkV2RemoveDocument`
- **Parallel Operations**: `BenchmarkV2CreateDocumentParallel`, `BenchmarkV2ReadDocumentParallel`
- **Comprehensive CRUD**: `BenchmarkV2ComprehensiveDocumentOperations_1K`, `BenchmarkV2ComprehensiveDocumentOperations_10K`
- **Performance Tests**: `Benchmark_Insert`, `Benchmark_BatchInsert`

### Running Benchmarks

#### Basic Benchmark Execution

```bash
# Run all V1 benchmarks
make run-benchmarks-single-json-no-auth

# Run all V2 benchmarks
make run-benchmarks-v2-single-json-no-auth
```

#### Running Specific Benchmarks

```bash
# V1 API - Specific benchmark
export TESTOPTIONS="-bench=BenchmarkCreateDocument -test.v"
make run-benchmarks-single-json-no-auth

# V2 API - Specific benchmark
export TESTOPTIONS="-bench=BenchmarkV2CreateDocument -test.v"
make run-benchmarks-v2-single-json-no-auth

# V1 API - Comprehensive CRUD with 10K documents
export TESTOPTIONS="-bench=BenchmarkComprehensiveDocumentOperations_10K -test.v"
make run-benchmarks-single-json-no-auth

# V2 API - Comprehensive CRUD with 10K documents
export TESTOPTIONS="-bench=BenchmarkV2ComprehensiveDocumentOperations_10K -test.v"
make run-benchmarks-v2-single-json-no-auth
```

#### Running Multiple Benchmarks

```bash
# Run multiple V2 benchmarks
export TESTOPTIONS="-bench=BenchmarkV2CreateDocument|BenchmarkV2ReadDocument -test.v"
make run-benchmarks-v2-single-json-no-auth

# Run both 1K and 10K comprehensive benchmarks
export TESTOPTIONS="-bench=BenchmarkV2ComprehensiveDocumentOperations_1K|BenchmarkV2ComprehensiveDocumentOperations_10K -test.v"
make run-benchmarks-v2-single-json-no-auth
```

#### Comparing V1 vs V2 Performance

```bash
# Run V1 10K comprehensive benchmark
export TESTOPTIONS="-bench=BenchmarkComprehensiveDocumentOperations_10K -test.v"
make run-benchmarks-single-json-no-auth

# Run V2 10K comprehensive benchmark
export TESTOPTIONS="-bench=BenchmarkV2ComprehensiveDocumentOperations_10K -test.v"
make run-benchmarks-v2-single-json-no-auth
```

### Benchmark Output Explanation

The benchmark output includes:

- **Operations per second**: How many operations completed in the benchmark duration
- **Time per operation**: Average time per operation (e.g., `720955 ns/op`)
- **Memory allocations**: Bytes allocated per operation (e.g., `9505 B/op`)
- **Allocation count**: Number of memory allocations per operation (e.g., `114 allocs/op`)
- **CPU scaling**: Results for different CPU counts (1, 2, 4 cores)

Example output:
```
BenchmarkV2CreateDocument/HTTP_JSON-4 1197 1055266 ns/op 10713 B/op 128 allocs/op
```

This means:
- 1197 operations completed
- 1,055,266 nanoseconds per operation (~1.06ms)
- 10,713 bytes allocated per operation
- 128 memory allocations per operation
- Tested with 4 CPU cores

### HTTP Protocol Comparison

V2 benchmarks automatically test both HTTP/1.1 and HTTP/2 protocols:

```
BenchmarkV2CreateDocument/HTTP_JSON # HTTP/1.1 results
BenchmarkV2CreateDocument/HTTP2_JSON # HTTP/2 results
```

### Benchmark Options

You can customize benchmark execution with additional options:

```bash
# Set benchmark duration
export TESTOPTIONS="-bench=BenchmarkV2CreateDocument -benchtime=10s -test.v"

# Set number of iterations
export TESTOPTIONS="-bench=BenchmarkV2CreateDocument -count=5 -test.v"

# Run with memory profiling
export TESTOPTIONS="-bench=BenchmarkV2CreateDocument -memprofile=mem.prof -test.v"

# Run with CPU profiling
export TESTOPTIONS="-bench=BenchmarkV2CreateDocument -cpuprofile=cpu.prof -test.v"
```

### Performance Tips

1. **Use specific benchmark names** to avoid running all tests
2. **Compare HTTP/1.1 vs HTTP/2** performance in V2 benchmarks
3. **Test with different document counts** (1K vs 10K) to understand scaling
4. **Run multiple iterations** for more reliable results
5. **Use profiling options** to identify performance bottlenecks

### Troubleshooting

- **"No endpoints found"**: Ensure Docker is running and the test environment is properly set up
- **Benchmark not found**: Check the exact function name (case-sensitive)
- **All benchmarks running**: Use specific benchmark names in `TESTOPTIONS`
- **HTTP/2 errors**: V2 benchmarks automatically handle HTTP/2 stream limits
Loading