NCM plays each Stockfish dev build 20,000 times against Stockfish 15. This yields an approximate Elo difference and establishes confidence in the strength of the dev builds.
| Host | Duration | Avg Base NPS | Games | WLD | Standard Elo | Ptnml(0-2) | Gamepair Elo |
|---|---|---|---|---|---|---|---|
| ncm-dbt-01 | 01:30:27 | 583992 | 872 | 310 163 399 | +59.13 ± 10.83 | 0 40 211 183 2 | +120.11 ± 23.4 |
| ncm-dbt-02 | 01:29:37 | 584643 | 868 | 306 129 433 | +71.86 ± 10.64 | 1 27 202 202 2 | +149.48 ± 23.86 |
| ncm-dbt-03 | 01:30:12 | 583218 | 878 | 299 154 425 | +57.91 ± 10.73 | 0 41 213 184 1 | +118.34 ± 23.29 |
| ncm-dbt-04 | 01:30:56 | 566713 | 886 | 315 157 414 | +62.63 ± 10.64 | 1 33 219 187 3 | +127.82 ± 22.86 |
| ncm-dbt-05 | 01:29:12 | 581908 | 864 | 293 158 413 | +54.74 ± 10.93 | 2 38 216 175 1 | +113.22 ± 23.07 |
| 4368 | 1523 761 2084 | +61.24 ± 4.81 | 4 179 1061 931 9 | +125.63 ± 10.4 | |||
| ID | Host | Base NPS | Games | WLD | Standard Elo | Ptnml(0-2) | Gamepair Elo | CLI | PGN | ||
|---|---|---|---|---|---|---|---|---|---|---|---|
| 422795 | ncm-dbt-05 | 579992 | 364 | 123 70 171 | +50.95 ± 17.28 | 2 15 94 70 1 | +106.28 ± 34.96 | ||||
cutechess-cli \
-rounds 266 \
-games 2 \
-concurrency 16 \
-srand 2044579809 \
-pgnout ncm-dbt-20230924-1804-010.pgn \
-openings \
file=UHO_4060_v2.epd \
format=epd \
order=random \
-repeat \
-resign \
movecount=3 \
score=600 \
-draw \
movenumber=34 \
movecount=8 \
score=5 \
-each \
tc=30+0.3 \
timemargin=10000 \
proto=uci \
option.Hash=128 \
option.Threads=8 \
-engine \
name=20230924-1804 \
cmd=docker \
arg=run \
arg=-i \
arg=--rm \
arg=--entrypoint=/engine \
arg=dev_build:22cdb6c1ea1f5ca429333bcbe26706c8b4dd38d7 \
-engine \
name=sf15 \
cmd=docker \
arg=run \
arg=-i \
arg=--rm \
arg=--entrypoint=/engine \
arg=stockfish:15
|
|||||||||||
| 422794 | ncm-dbt-02 | 585211 | 368 | 131 59 178 | +68.87 ± 16.32 | 1 11 87 85 0 | +145.84 ± 36.42 | ||||
cutechess-cli \
-rounds 266 \
-games 2 \
-concurrency 16 \
-srand 3109779668 \
-pgnout ncm-dbt-20230924-1804-009.pgn \
-openings \
file=UHO_4060_v2.epd \
format=epd \
order=random \
-repeat \
-resign \
movecount=3 \
score=600 \
-draw \
movenumber=34 \
movecount=8 \
score=5 \
-each \
tc=30+0.3 \
timemargin=10000 \
proto=uci \
option.Hash=128 \
option.Threads=8 \
-engine \
name=20230924-1804 \
cmd=docker \
arg=run \
arg=-i \
arg=--rm \
arg=--entrypoint=/engine \
arg=dev_build:22cdb6c1ea1f5ca429333bcbe26706c8b4dd38d7 \
-engine \
name=sf15 \
cmd=docker \
arg=run \
arg=-i \
arg=--rm \
arg=--entrypoint=/engine \
arg=stockfish:15
|
|||||||||||
| 422793 | ncm-dbt-01 | 583279 | 372 | 130 63 179 | +63.27 ± 16.16 | 0 15 89 82 0 | +131.03 ± 36.09 | ||||
cutechess-cli \
-rounds 266 \
-games 2 \
-concurrency 16 \
-srand 4189268187 \
-pgnout ncm-dbt-20230924-1804-008.pgn \
-openings \
file=UHO_4060_v2.epd \
format=epd \
order=random \
-repeat \
-resign \
movecount=3 \
score=600 \
-draw \
movenumber=34 \
movecount=8 \
score=5 \
-each \
tc=30+0.3 \
timemargin=10000 \
proto=uci \
option.Hash=128 \
option.Threads=8 \
-engine \
name=20230924-1804 \
cmd=docker \
arg=run \
arg=-i \
arg=--rm \
arg=--entrypoint=/engine \
arg=dev_build:22cdb6c1ea1f5ca429333bcbe26706c8b4dd38d7 \
-engine \
name=sf15 \
cmd=docker \
arg=run \
arg=-i \
arg=--rm \
arg=--entrypoint=/engine \
arg=stockfish:15
|
|||||||||||
| 422792 | ncm-dbt-03 | 582068 | 378 | 127 62 189 | +60.34 ± 16.69 | 0 19 86 84 0 | +124.56 ± 36.88 | ||||
cutechess-cli \
-rounds 266 \
-games 2 \
-concurrency 16 \
-srand 166849379 \
-pgnout ncm-dbt-20230924-1804-007.pgn \
-openings \
file=UHO_4060_v2.epd \
format=epd \
order=random \
-repeat \
-resign \
movecount=3 \
score=600 \
-draw \
movenumber=34 \
movecount=8 \
score=5 \
-each \
tc=30+0.3 \
timemargin=10000 \
proto=uci \
option.Hash=128 \
option.Threads=8 \
-engine \
name=20230924-1804 \
cmd=docker \
arg=run \
arg=-i \
arg=--rm \
arg=--entrypoint=/engine \
arg=dev_build:22cdb6c1ea1f5ca429333bcbe26706c8b4dd38d7 \
-engine \
name=sf15 \
cmd=docker \
arg=run \
arg=-i \
arg=--rm \
arg=--entrypoint=/engine \
arg=stockfish:15
|
|||||||||||
| 422791 | ncm-dbt-04 | 565548 | 386 | 131 70 185 | +55.37 ± 15.76 | 1 14 101 77 0 | +115.71 ± 33.54 | ||||
cutechess-cli \
-rounds 266 \
-games 2 \
-concurrency 16 \
-srand 1840078302 \
-pgnout ncm-dbt-20230924-1804-006.pgn \
-openings \
file=UHO_4060_v2.epd \
format=epd \
order=random \
-repeat \
-resign \
movecount=3 \
score=600 \
-draw \
movenumber=34 \
movecount=8 \
score=5 \
-each \
tc=30+0.3 \
timemargin=10000 \
proto=uci \
option.Hash=128 \
option.Threads=8 \
-engine \
name=20230924-1804 \
cmd=docker \
arg=run \
arg=-i \
arg=--rm \
arg=--entrypoint=/engine \
arg=dev_build:22cdb6c1ea1f5ca429333bcbe26706c8b4dd38d7 \
-engine \
name=sf15 \
cmd=docker \
arg=run \
arg=-i \
arg=--rm \
arg=--entrypoint=/engine \
arg=stockfish:15
|
|||||||||||
| 422790 | ncm-dbt-05 | 583824 | 500 | 170 88 242 | +57.5 ± 14.08 | 0 23 122 105 0 | +118.33 ± 30.8 | ↓ | |||
cutechess-cli \
-rounds 266 \
-games 2 \
-concurrency 16 \
-srand 405101584 \
-pgnout ncm-dbt-20230924-1804-005.pgn \
-openings \
file=UHO_4060_v2.epd \
format=epd \
order=random \
-repeat \
-resign \
movecount=3 \
score=600 \
-draw \
movenumber=34 \
movecount=8 \
score=5 \
-each \
tc=30+0.3 \
timemargin=10000 \
proto=uci \
option.Hash=128 \
option.Threads=8 \
-engine \
name=20230924-1804 \
cmd=docker \
arg=run \
arg=-i \
arg=--rm \
arg=--entrypoint=/engine \
arg=dev_build:22cdb6c1ea1f5ca429333bcbe26706c8b4dd38d7 \
-engine \
name=sf15 \
cmd=docker \
arg=run \
arg=-i \
arg=--rm \
arg=--entrypoint=/engine \
arg=stockfish:15
|
|||||||||||
| 422789 | ncm-dbt-02 | 584076 | 500 | 175 70 255 | +74.06 ± 14.03 | 0 16 115 117 2 | +152.18 ± 31.7 | ↓ | |||
cutechess-cli \
-rounds 266 \
-games 2 \
-concurrency 16 \
-srand 939670559 \
-pgnout ncm-dbt-20230924-1804-004.pgn \
-openings \
file=UHO_4060_v2.epd \
format=epd \
order=random \
-repeat \
-resign \
movecount=3 \
score=600 \
-draw \
movenumber=34 \
movecount=8 \
score=5 \
-each \
tc=30+0.3 \
timemargin=10000 \
proto=uci \
option.Hash=128 \
option.Threads=8 \
-engine \
name=20230924-1804 \
cmd=docker \
arg=run \
arg=-i \
arg=--rm \
arg=--entrypoint=/engine \
arg=dev_build:22cdb6c1ea1f5ca429333bcbe26706c8b4dd38d7 \
-engine \
name=sf15 \
cmd=docker \
arg=run \
arg=-i \
arg=--rm \
arg=--entrypoint=/engine \
arg=stockfish:15
|
|||||||||||
| 422788 | ncm-dbt-03 | 584369 | 500 | 172 92 236 | +56.07 ± 14.01 | 0 22 127 100 1 | +113.68 ± 30.06 | ↓ | |||
cutechess-cli \
-rounds 266 \
-games 2 \
-concurrency 16 \
-srand 1854853434 \
-pgnout ncm-dbt-20230924-1804-003.pgn \
-openings \
file=UHO_4060_v2.epd \
format=epd \
order=random \
-repeat \
-resign \
movecount=3 \
score=600 \
-draw \
movenumber=34 \
movecount=8 \
score=5 \
-each \
tc=30+0.3 \
timemargin=10000 \
proto=uci \
option.Hash=128 \
option.Threads=8 \
-engine \
name=20230924-1804 \
cmd=docker \
arg=run \
arg=-i \
arg=--rm \
arg=--entrypoint=/engine \
arg=dev_build:22cdb6c1ea1f5ca429333bcbe26706c8b4dd38d7 \
-engine \
name=sf15 \
cmd=docker \
arg=run \
arg=-i \
arg=--rm \
arg=--entrypoint=/engine \
arg=stockfish:15
|
|||||||||||
| 422787 | ncm-dbt-01 | 584706 | 500 | 180 100 220 | +56.07 ± 14.56 | 0 25 122 101 2 | +112.14 ± 30.83 | ↓ | |||
cutechess-cli \
-rounds 266 \
-games 2 \
-concurrency 16 \
-srand 3322260569 \
-pgnout ncm-dbt-20230924-1804-002.pgn \
-openings \
file=UHO_4060_v2.epd \
format=epd \
order=random \
-repeat \
-resign \
movecount=3 \
score=600 \
-draw \
movenumber=34 \
movecount=8 \
score=5 \
-each \
tc=30+0.3 \
timemargin=10000 \
proto=uci \
option.Hash=128 \
option.Threads=8 \
-engine \
name=20230924-1804 \
cmd=docker \
arg=run \
arg=-i \
arg=--rm \
arg=--entrypoint=/engine \
arg=dev_build:22cdb6c1ea1f5ca429333bcbe26706c8b4dd38d7 \
-engine \
name=sf15 \
cmd=docker \
arg=run \
arg=-i \
arg=--rm \
arg=--entrypoint=/engine \
arg=stockfish:15
|
|||||||||||
| 422786 | ncm-dbt-04 | 567878 | 500 | 184 87 229 | +68.27 ± 14.4 | 0 19 118 110 3 | +137.37 ± 31.31 | ↓ | |||
cutechess-cli \
-rounds 266 \
-games 2 \
-concurrency 16 \
-srand 3976992515 \
-pgnout ncm-dbt-20230924-1804-001.pgn \
-openings \
file=UHO_4060_v2.epd \
format=epd \
order=random \
-repeat \
-resign \
movecount=3 \
score=600 \
-draw \
movenumber=34 \
movecount=8 \
score=5 \
-each \
tc=30+0.3 \
timemargin=10000 \
proto=uci \
option.Hash=128 \
option.Threads=8 \
-engine \
name=20230924-1804 \
cmd=docker \
arg=run \
arg=-i \
arg=--rm \
arg=--entrypoint=/engine \
arg=dev_build:22cdb6c1ea1f5ca429333bcbe26706c8b4dd38d7 \
-engine \
name=sf15 \
cmd=docker \
arg=run \
arg=-i \
arg=--rm \
arg=--entrypoint=/engine \
arg=stockfish:15
|
|||||||||||
| Commit ID | 22cdb6c1ea1f5ca429333bcbe26706c8b4dd38d7 |
|---|---|
| Author | Joost VandeVondele |
| Date | 2023-09-24 18:04:42 UTC |
|
Explicitly invoke shell
in some cases the permission on the script might be incorrect (zip downloads?).
Explicitly invoke the shell
closes https://github.com/official-stockfish/Stockfish/pull/4803
No functional change
|
|