Goptuna Distributed Bayesian Optimization Framework at Go Conference 2019 Autumn

Go Conference 2019 Autumn
c-bata c_bata_
Distributed Bayesian Optimization Framework
Goptuna

CyberAgent AI Lab
Masashi SHIBATA
c-bata c_bata_
Python
go-prompt, kube-prompt

https://github.com/c-bata

2
innodb_buﬀer_pool_size
80%
FFM 0.02
0.00001
15

[Eric et al., 2010]
• :

• :

• :
 
( )

package main
import (
"log"
"math"
bayesopt "github.com/d4l3k/go-bayesopt"
)
var (
px1 = bayesopt.UniformParam{Name: "X1", Max: 10, Min: -10}
px2 = bayesopt.UniformParam{Name: "X2", Max: 10, Min: -10}
searchSpace = []bayesopt.Param{px1, px2}
)
func objective(params map[bayesopt.Param]float64) float64 {
x1 := params[px1]
x2 := params[px2]
return math.Pow(x1-1, 2) + math.Pow(x2-2, 2)
}
func main() {
o := bayesopt.New(
searchSpace,
bayesopt.WithRandomRounds(10),
bayesopt.WithRounds(100),
bayesopt.WithMinimize(true))
x, y, err := o.Optimize(objective)
if err != nil { ... }
log.Printf(...)
}

$ go get github.com/c-bata/goptuna

package main
import (
"log"
"math"
"github.com/c-bata/goptuna"
"github.com/c-bata/goptuna/tpe"
)
func objective(trial goptuna.Trial) (float64, error) {
x1, _ := trial.SuggestUniform("x1", -10, 10)
return math.Pow(x1-2, 2) + math.Pow(x2+5, 2), nil
}
func main() {
study, _ := goptuna.CreateStudy(
"goptuna-example",
goptuna.StudyOptionSampler(tpe.NewSampler()))
err = study.Optimize(objective, 100)
if err != nil { ... }
v, _ := study.GetBestValue()
params, _ := study.GetBestParams()
log.Printf(...)
}

package main
import (
"log"
"math"
)
}
func main() {
"goptuna-example",
if err != nil { ... }
log.Printf(...)
}
goptuna.Trial
float64 error

package main
import (
"log"
"math"
)
}
func main() {
"goptuna-example",
if err != nil { ... }
log.Printf(...)
}
(x1, x2) = (2, -5)

package main
import (
"log"
"math"
)
}
func main() {
"goptuna-example",
if err != nil { ... }
log.Printf(...)
}
SuggestUniform SuggestLogUniform
SuggestInt SuggestCategorical
x1 x2
[-10, 10]

package main
import (
"log"
"math"
)
}
func main() {
"goptuna-example",
if err != nil { ... }
log.Printf(...)
}
$ go run _examples/simple_tpe/main.go
2019/10/28 01:55:34 [INFO] Trial finished: trialID=0 state=Complete evaluation=47.674293
:
2019/10/28 01:55:34 Best evaluation=0.038327 (x1=2.181604, x2=-4.926880)

package main
import (
"github.com/c-bata/goptuna/rdb"
"github.com/jinzhu/gorm"
_ "github.com/jinzhu/gorm/dialects/sqlite"
)
func main() {
db, err := gorm.Open("sqlite3", "db.sqlite3")
if err != nil { ... }
defer db.Close()
db.DB().SetMaxOpenConns(1)
rdb.RunAutoMigrate(db)
storage := rdb.NewStorage(db)
study, _ := goptuna.LoadStudy(
"study-name",
goptuna.StudyOptionStorage(storage))
...
}

$ pip install optuna bokeh mysqlclient
$ optuna dashboard
> --storage mysql+mysqldb://
goptuna:password@127.0.0.1:3306/yourdb
> -—study yourstudy

import ...
func main() {
db, _ := gorm.Open(“mysql",
“user:pass@tcp(localhost:3306)/yourdb" +
”?parseTime=true”)
storage := rdb.NewStorage(db)
defer db.Close()
study, _ := goptuna.LoadStudy(
“study-name",
goptuna.StudyOptionStorage(storage))
...
}
$ goptuna create-study
> --storage mysql+mysqldb://
goptuna:password@127.0.0.1:3306/yourdb
> -—study yourstudy
goputna.LoadStudy study
: _examples/concurrency

• optuna.visualization.plot_contour(st
udy)
• optuna.visualization.
plot_intermediate_values(study)
• optuna.visualization.plot_optimizati
on_history(study)
• optuna.visualization.plot_parallel_c
oordinate(study)
• optuna.visualization.plot_slice(stud
y)

package main
import (
"log"
"math"
bo "github.com/d4l3k/go-bayesopt"
)
var (
px1 = bo.UniformParam{Name: "X1", Max: 10, Min: -10}
px2 = bo.UniformParam{Name: "X2", Max: 10, Min: -10}
searchSpace = []bo.Param{px1, px2}
)
func objective(params map[bo.Param]float64) float64 {
x1 := params[px1]
x2 := params[px2]
return math.Pow(x1-1, 2) + math.Pow(x2-2, 2)
}
func main() {
o := bo.New(
searchSpace,
bo.WithRandomRounds(10),
bo.WithRounds(100),
bo.WithMinimize(true))
x, y, err := o.Optimize(objective)
if err != nil { ... }
log.Printf(...)
}
package main
import (
"log"
"math"
)
}
func main() {
"goptuna-example",
if err != nil { ... }
log.Printf(...)
}
Deﬁne-and-Run interface
go-bayesopt
Deﬁne-by-Run interface
Goptuna

10 epochs
trial #1
trial #2
trial #3
trial #4
trial #5
trial #6
trial #7
trial #8
trial #9
30 epochs 90 epochs
[Jamieson and Talwalkar, 2016] [Li et al., 2018]

2,010
10,660
1 ISUCON Alibaba Cloud ecs.sn1ne.large (vCPUx2, Mem 4.0GiB) 3
2 ISUCON9 9560 9100

// Go application
goMySQLOpenConns, _ := trial.SuggestInt("mysql_client_open_conns", 1, 64)
goMySQLIdleConns, _ := trial.SuggestInt("mysql_client_idle_conns", 1, 64)
goMySQLMaxLifetime, _ := trial.SuggestInt("mysql_client_max_lifetime", 1, 128)
goHttpIdleConnsPerHost, _ := trial.SuggestInt("http_max_idle_conns_per_host", 1, 4096)
goHttpKeepAlive, _ := trial.SuggestInt("http_keep_alive_timeout", 1, 128)
campaign, _ := trial.SuggestInt("campaign", 0, 2)
// Nginx
nginxWorkerProcesses, _ := trial.SuggestInt("nginx_worker_processes", 1, 16)
nginxWorkerConns, _ := trial.SuggestInt("nginx_worker_connections", 1, 4096)
nginxKeepAliveTimeout, _ := trial.SuggestInt("nginx_keep_alive_timeout", 1, 100)
// MySQL
innoDBBufferPoolSize, _ := trial.SuggestInt("innodb_buffer_pool_size", 10, 3800)
innoDBFlushLogAtTRXCommit, _ := trial.SuggestCategorical(“innodb_flush_log_at_trx_commit",
[]string{"0", "1", "2"})
innodbFlushMethod, _ := trial.SuggestCategorical("innodb_flush_method", []string{
"fsync",
"littlesync",
"nosync",
"O_DIRECT",
"O_DIRECT_NO_FSYNC",
})
...
} https://github.com/c-bata/goptuna-isucon9q/blob/master/optimizer/main.go

// Go application
// Nginx
// MySQL
[]string{"0", "1", "2"})
"fsync",
"littlesync",
"nosync",
"O_DIRECT",
})
...
(database/sql).DB

// Go application
// Nginx
// MySQL
[]string{"0", "1", "2"})
"fsync",
"littlesync",
"nosync",
"O_DIRECT",
})
...
API  
(net/http).Client

// Go application
// Nginx
// MySQL
[]string{"0", "1", "2"})
"fsync",
"littlesync",
"nosync",
"O_DIRECT",
})
...
campaign:

// Go application
// Nginx
// MySQL
[]string{"0", "1", "2"})
"fsync",
"littlesync",
"nosync",
"O_DIRECT",
})
...
Nginx
suggest text/template
nginx.conf systemd

// Go application
// Nginx
// MySQL
[]string{"0", "1", "2"})
"fsync",
"littlesync",
"nosync",
"O_DIRECT",
})
...
MySQL
Nginx

Go MySQL client (net/http).Client Nginx MySQL
SetMaxOpen
Conns
SetMaxIdleC
onns
SetConnMax
Lifetime
DialContext.
Keepalive
MaxIdleConn
sPerHost
worker_proc
esses
worker_conn
ections
keep_alive_ti
meout
innodb_buffe
r_pool_size
innodb_flush
_log_at_trx_c
ommit
innodb_flush
_method
10660
11160

(+500)
200 37 53 77s 40s 709 11 367 74s 711M 0 fsync

Go MySQL client Nginx MySQL
SetMaxOpenCo
nns
SetMaxIdleConn
s
SetConnMaxLife
time
worker_connecti
ons
innodb_buffer_p
ool_size
innodb_flush_log
_at_trx_commit
innodb_log_buff
er_size
innodb_log_file_
size
2010
2310

(+300)
50 23 4 28s 5 210M 0 27M 341M
https://github.com/c-bata/goptuna-isucon9q/issues/2
https://github.com/c-bata/goptuna-isucon9q/issues/9

Optuna x.264
(by Takeru Ohta a.k.a. @sile)

x.264
https://gist.github.com/sile/8aa1ﬀ7808dd55298f51dd70c8b83092

• Goptuna

LibFFM [Juan et al., 2016]  
• Optuna

RocsDB by @sile 
• BoTorch/Ax (Facebook)

HHVM JIT Compiler

• [Eric et al., 2010] Eric Brochu, Vlad M. Cora, and Nando de Freitas. A tutorial on Bayesian optimization of expensive cost
functions, with application to active user modeling and hierarchical reinforcement learning. 2010. arXiv:1012.2599.

• [James et al., 2011] James S. Bergstra, Remi Bardenet, Yoshua Bengio, and Balázs Kégl. Algorithms for hyper-
parameter optimization. In Advances in Neural Information Processing Systems 25. 2011.

• [Akiba at el., 2019] Takuya Akiba, Shotaro Sano, Toshihiko Yanase, Takeru Ohta, Masanori Koyama. 2019. Optuna: A
Next-generation Hyperparameter Optimization Framework. In The 25th ACM SIGKDD Conference on Knowledge
Discovery and Data Mining (KDD ’19), August 4–8, 2019.

• [Jamieson and Talwalkar, 2016] K. Jamieson and A. Talwalkar. Non-stochastic best arm identiﬁcation and
hyperparameter optimization. In AISTATS, 2016.

• [Li et al., 2018] Liam Li, Kevin Jamieson, Afshin Rostamizadeh, Ekaterina Gonina, Moritz Hardt, Benjamin Recht, and
Ameet Talwalkar. Massively parallel hyperparameter tuning. arXiv preprint arXiv:1810.05934, 2018.

• [Eggensperger et al., 2013] Eggensperger, K., Feurer, M., Hutter, F., Bergstra, J., Snoek, J., Hoos, H., and Leyton-Brown,
K.: Towards an Empirical Foundation for Assessing Bayesian Optimization of Hyperparameters, in NeurIPS workshop on
Bayesian Optimization in Theory and Practice (2013).

• [Juan et al., 2016] Yu-Chin Juan, Yong Zhuang, Wei-Sheng Chin, and Chih-Jen Lin. Field-aware factorization machines
for CTR prediction. In RecSys, pages 43–50, 2016.
References

https://speakerdeck.com/nmasahiro/ji-jie-xue-xi-niokeru-haihaharametazui-shi-hua-falseli-lun-toshi-jian?slide=52

Goptuna Distributed Bayesian Optimization Framework at Go Conference 2019 Autumn

More Related Content

What's hot (20)

Similar to Goptuna Distributed Bayesian Optimization Framework at Go Conference 2019 Autumn (20)

More from Masashi Shibata (20)

Recently uploaded (20)

Goptuna Distributed Bayesian Optimization Framework at Go Conference 2019 Autumn