![$ http GET http://127.0.0.1:8080/api/v1/namespaces/default
{
"apiVersion": "v1",
"kind": "Namespace",
"metadata": {
"annotations": {
"meetup": “hallo"
},
"creationTimestamp": "2017-03-10T07:51:39Z",
"name": "default",
"resourceVersion": “73",
},
"spec": { "finalizers": ["kubernetes“] },
"status": { "phase": "Active“ }
}
$ http GET http://127.0.0.1:8080/api/v1/namespaces/default |
jq ".metadata.annotations["meetup"] = "$(date)"" |
http PUT http://127.0.0.1:8080/api/v1/namespaces/default
HTTP/1.1 200 OK
Content-Length: 341
Content-Type: application/json
Date: Fri, 10 Mar 2017 08:28:01 GMT](https://image.slidesharecdn.com/meetup-extendandbuildonkubernetes-170311003200/85/Extend-and-build-on-Kubernetes-6-320.jpg)
![$ while true; do
http GET http://127.0.0.1:8080/api/v1/namespaces/default |
jq ".metadata.annotations["meetup"] = "$(date)"" |
http --check-status PUT http://127.0.0.1:8080/api/v1/namespaces/default || break
done
HTTP/1.1 409 Conflict
Content-Length: 310
Content-Type: application/json
Date: Fri, 10 Mar 2017 08:27:58 GMT
{
"apiVersion": "v1",
"code": 409,
"details": {
"kind": "namespaces",
"name": "default“
},
"kind": "Status",
"message": "Operation cannot be fulfilled on namespaces "default": the object has been
modified; please apply your changes to the latest version and try again",
"metadata": {},
"reason": "Conflict",
"status": "Failure“](https://image.slidesharecdn.com/meetup-extendandbuildonkubernetes-170311003200/85/Extend-and-build-on-Kubernetes-7-320.jpg)
![$ http 127.0.0.1:8080/apis/
{
"groups": [{
"name": "batch",
"preferredVersion": {"groupVersion": "batch/v1", "version": "v1“},
"versions": [{"groupVersion": "batch/v1", "version": "v1"}]
}, ...]
}
$ http 127.0.0.1:8080/apis/batch/v1
{
"apiVersion": "v1",
"groupVersion": "batch/v1",
"kind": "APIResourceList",
"resources": [{
"kind": "Job",
"name": "jobs",
"namespaced": true,
"verbs": ["create", "delete", "deletecollection",
"get", "list", "patch", "update", "watch“
]
}, ...]
}
resource name ⇒ /apis/batch/v1/jobs](https://image.slidesharecdn.com/meetup-extendandbuildonkubernetes-170311003200/85/Extend-and-build-on-Kubernetes-16-320.jpg)
![$ while true; do
http GET http://127.0.0.1:8080/api/v1/namespaces/default |
jq ".metadata.annotations["meetup"] = "$(date)"" |
http --check-status PUT http://127.0.0.1:8080/api/v1/namespaces/default || break
done
⟲](https://image.slidesharecdn.com/meetup-extendandbuildonkubernetes-170311003200/85/Extend-and-build-on-Kubernetes-27-320.jpg)
Extend and build on Kubernetes
- 1. Extend and Build on Kubernetes Dr. Stefan Schimanski [email protected] @the1stein Kubernetes Meetup Frankfurt, Mar 10 2017
- 2. Nune Isabekyan, https://x-team.com/blog/introduction-kubernetes-architecture/
- 3. Disclaimer: Kubernetes happens to be able to launch pods. It‘s even quite good at it. We will not launch pods today.
- 5. $ kube-apiserver --secure-port 0 --etcd-servers http://127.0.0.1:2379 --service-cluster-ip-range 10.0.0.0/16 --storage-backend etcd2 --storage-media-type application/json $ etcd $ kubectl config set-cluster local --server=http://127.0.0.1:8080 $ kubectl config set-context local --cluster=local $ kubectl config use-context local $ kubectl get namespaces –v=7 $ kubectl get namespace default -o json $ kubectl annotate namespace default meetup=hello $ curl http://127.0.0.1:8080/api/v1/namespaces/default $ etcdctl get / --recursive $ etcdctl get /registry/namespaces/default $ etcdctl -o extended get /registry/namespaces/default
- 6. $ http GET http://127.0.0.1:8080/api/v1/namespaces/default { "apiVersion": "v1", "kind": "Namespace", "metadata": { "annotations": { "meetup": “hallo" }, "creationTimestamp": "2017-03-10T07:51:39Z", "name": "default", "resourceVersion": “73", }, "spec": { "finalizers": ["kubernetes“] }, "status": { "phase": "Active“ } } $ http GET http://127.0.0.1:8080/api/v1/namespaces/default | jq ".metadata.annotations["meetup"] = "$(date)"" | http PUT http://127.0.0.1:8080/api/v1/namespaces/default HTTP/1.1 200 OK Content-Length: 341 Content-Type: application/json Date: Fri, 10 Mar 2017 08:28:01 GMT
- 7. $ while true; do http GET http://127.0.0.1:8080/api/v1/namespaces/default | jq ".metadata.annotations["meetup"] = "$(date)"" | http --check-status PUT http://127.0.0.1:8080/api/v1/namespaces/default || break done HTTP/1.1 409 Conflict Content-Length: 310 Content-Type: application/json Date: Fri, 10 Mar 2017 08:27:58 GMT { "apiVersion": "v1", "code": 409, "details": { "kind": "namespaces", "name": "default“ }, "kind": "Status", "message": "Operation cannot be fulfilled on namespaces "default": the object has been modified; please apply your changes to the latest version and try again", "metadata": {}, "reason": "Conflict", "status": "Failure“
- 10. /apis/extensions/v1alpha1/jobs /apis/batch/v2alpha1/jobs /apis/batch/v1/jobs { “apiVersion“: “v1alpha1“, “kind“: “Job“, “metadata“: { “name“: “nightly“ }, “spec“: { ... } } { “apiVersion“: “v2alpha1“, “kind“: “Job“, “metadata“: { “name“: “nightly“ }, “spec“: { ... } } { “apiVersion“: “v1“, “kind“: “Job“, “metadata“: { “name“: “nightly“ }, “spec“: { ... } }
- 11. /apis/extensions/v1alpha1/jobs /apis/batch/v2alpha1/jobs /apis/batch/v1/jobs { “apiVersion“: “v1alpha1“, “kind“: “Job“, “metadata“: { “name“: “nightly“ }, “spec“: { ... } } { “apiVersion“: “v2alpha1“, “kind“: “Job“, “metadata“: { “name“: “nightly“ }, “spec“: { ... } } { “apiVersion“: “v1“, “kind“: “Job“, “metadata“: { “name“: “nightly“ }, “spec“: { ... } }
- 12. /apis/extensions/v1alpha1/jobs/nightly /apis/batch/v2alpha1/jobs/nightly /apis/batch/v1/jobs/nightly { “apiVersion“: “v1alpha1“, “kind“: “Job“, “metadata“: { “name“: “nightly“ }, “spec“: { ... } } { “apiVersion“: “v2alpha1“, “kind“: “Job“, “metadata“: { “name“: “nightly“ }, “spec“: { ... } } { “apiVersion“: “v1beta1“, “kind“: “Job“, “metadata“: { “name“: “nightly“ }, “spec“: { ... } } * I omitted the namespace in /apis/batch/v1/jobs/namespaces/default/nightly
- 13. { “apiVersion“: “v1alpha1“, “kind“: “Job“, “metadata“: { “name“: “nightly“ }, “spec“: { ... } } { “apiVersion“: “v2alpha1“, “kind“: “Job“, “metadata“: { “name“: “nightly“ }, “spec“: { ... } } { “apiVersion“: “v1 “, “kind“: “Job“, “metadata“: { “name“: “nightly“ }, “spec“: { ... } } etcd { “apiVersion“: “v1 “, “kind“: “Job“, “metadata“: { “name“: “nightly“ }, “spec“: { ... } } Protobuf or JSON storage version encoding
- 14. I want my own kinds and store them in the apiserver and use kubectl.
- 15. Discovery how kubectl knows which kinds/resources exist
- 16. $ http 127.0.0.1:8080/apis/ { "groups": [{ "name": "batch", "preferredVersion": {"groupVersion": "batch/v1", "version": "v1“}, "versions": [{"groupVersion": "batch/v1", "version": "v1"}] }, ...] } $ http 127.0.0.1:8080/apis/batch/v1 { "apiVersion": "v1", "groupVersion": "batch/v1", "kind": "APIResourceList", "resources": [{ "kind": "Job", "name": "jobs", "namespaced": true, "verbs": ["create", "delete", "deletecollection", "get", "list", "patch", "update", "watch“ ] }, ...] } resource name ⇒ /apis/batch/v1/jobs
- 18. Third Party Resources * they are marked as beta in the API, but became beta before we even had alpha.
- 19. apiVersion: extensions/v1beta1 kind: ThirdPartyResource metadata: name: databases.example.com description: "A specification of a SQL database.“ versions: - name: v1 $ kubectl create –f databases-tpr.yaml
- 20. apiVersion: extensions/v1beta1 kind: ThirdPartyResource metadata: name: databases.example.com description: "A specification of a SQL database.“ versions: - name: v1 $ kubectl create –f databases-tpr.yaml $ kubectl create –f wordpress-databases.yaml apiVersion: example.com/v1 kind: Databases metadata: name: wordpress spec: user: wp password: secret encoding: unicode
- 21. apiVersion: extensions/v1beta1 kind: ThirdPartyResource metadata: name: database.example.com description: "A specification of a SQL database.“ versions: - name: v1 $ kubectl create –f databases-tpr.yaml $ kubectl create –f wordpress-database.yaml apiVersion: example.com/v1 kind: Database metadata: name: wordpress spec: user: wp password: secret encoding: unicode
- 22. $ kubectl get databases –w --no-headers wordpress <none> {"apiVersion":"example.com/v1","kind":"Databases",... wordpress <none> {"apiVersion":"example.com/v1","kind":"Databases",...
- 25. ThirdPartyResources are limited • no version conversion • no defaulting • no validation • no subresources (scale, status) • no admission • alpha ⇒ API might change • but: demand is high, expect improvements in 1.7+ • Today‘s users of TPRs: https://gist.github.com/philips/a97a143546c87b86b870a82a753db14c
- 26. Controllers where is the business logic?
- 27. $ while true; do http GET http://127.0.0.1:8080/api/v1/namespaces/default | jq ".metadata.annotations["meetup"] = "$(date)"" | http --check-status PUT http://127.0.0.1:8080/api/v1/namespaces/default || break done ⟲
- 28. $ kubectl get namespaces -w --no-headers | while read NS STATUS TIME ; do # do whatever you like here, e.g. change the namespace echo "$NS changed“ done ⟲ $ curl -f 'http://127.0.0.1:8080/api/v1/namespaces?watch=true&resourceVersion=4711‘ {"type":"ADDED","object":{"kind":"Namespace","apiVersion":"v1","metadata":{"name ... {"type":“MODIFIED","object":{"kind":"Namespace","apiVersion":"v1","metadata":{"name ... {"type":“DELETED","object":{"kind":"Namespace","apiVersion":"v1","metadata":{"name ...
- 29. API Aggregation
- 30. Why • ThirdPartyResources are limited • no version conversion • no defaulting • no validation • no subresources (scale, status) • no admission • Some things need full power of Go • Service catalog • OpenShift PaaS • other powerful APIs ≫ Goal: allow powerful extensions without modifying Kubernetes itself
- 31. Alpha in v1.6: k8s.io/apiserver • generic apiserver library in Go • today used inside • kube-apiserver • federation apiserver • service catalog • allows creation of custom apiservers in a couple hundred lines of code • each custom apiserver is its own process, communicating via HTTPS • delegates authentication/authorization to kube-apiserver • uses etcd storage (possibly shared with kube)
- 32. Nune Isabekyan, https://x-team.com/blog/introduction-kubernetes-architecture/
- 33. kube-apiserver service catalog PaaS kube-aggregator
- 34. kube-apiserver service catalog PaaS kube-aggregator API API API API Pods Jobs ... announcement ... build test project
- 35. kube-apiserver kube-apiserver kube-apiserver federation apiserver API API API API Pods replicasets Pods replicasets Pods replicasets deployment service Europe US Asia Not this: federation controllers⟲ federated resources: availability zones + regions:
- 36. kube-apiserver service catalog PaaS kube-aggregator discoverydiscovery
- 37. kube-apiserver service catalog PaaS kube-aggregator GET GET /apis/servicecatalog/subscription/database-prod-wordpress
- 38. kube-apiserver service catalog PaaS kube-aggregator GET /apis/servicecatalog/subscription/database-prod-wordpress GET RBAC Namespace
- 39. kube-apiserver service catalog PaaS kube-aggregator GET Vision: $ helm install service-catalog $ kubectl create service-announcement .... GET /apis/servicecatalog/subscription/database-prod-wordpress
- 41. Status • will be part of Kubernetes 1.6 as an alpha • https://github.com/kubernetes/sample-apiserver • potentially kube-aggregator integrated into kube-apiserver in 1.7
- 42. Links • https://docs.google.com/document/d/1y16jKL2hMjQO0trYBJJSczPA Wj8vAgNFrdTZeCincmI/ Two Ways to Extend the K8s API - Add resources to a Kubernetes API with TPR or AA • https://github.com/kubernetes/community/blob/master/contributor s/design-proposals/aggregated-api-servers.md • https://gist.github.com/philips/a97a143546c87b86b870a82a753db1 4c - Kubernetes Third-Party Resource Users
- 44. Backup
- 45. Restful http API / /version /api /api/v1/pods /api/v1/pods/status /apis /apis/batch /apis/batch/v2alpha1 /apis/batch/v2alpha1/jobs /apis/batch/v2alpha1/cronjo /apis/batch/v1beta1 /apis/batch/v1beta1/jobs /apis/batch/v2alpha1/jobs Group Version ResourceHTTP paths: In Go: gvk := schema.GroupVersionKind{Group: “batch“, Version: “v2alpha1“, Kind: “Job“} obj := api.Scheme.New(gvk) codec := api.Codecs.LegacyCodec(gvk.GroupVersion()) codec.Decode(reqBody, gvk, obj) type Job struct { metav1.TypeMeta metav1.ObjectMeta Spec JobSpec Status JobStatus } pkg/apis/batch/v2alpha1/types.go type TypeMeta struct { Kind string APIVersion string } type ObjectMeta struct { Name string ... }
- 46. Restful http API / /version /api /api/v1/pods /api/v1/pods/status /apis /apis/batch /apis/batch/v2alpha1 /apis/batch/v2alpha1/jobs /apis/batch/v2alpha1/cronjo /apis/batch/v1beta1 /apis/batch/v1beta1/jobs MaxInFlightLimit TimeoutForNonLongRunningRequests Panic Recovery CORS Authentication Audit Impersonation Authorization k8s.io/apiserver/pkg/server. DefaultBuildHandlerChain „Filters“ k8s.io/apiserver/pkg/server/routes/index.go – / k8s.io/apiserver/pkg/server/routes/version.go – /version k8s.io/apiserver/pkg/server/routes/swagger.go – /swaggerapi k8s.io/apiserver/pkg/server/routes/openapi.go – /swagger.json „Routes“ mux k8s.io/apiserver/pkg/endpoints.APIGroupVersion.InstallREST AddSupportedResourcesWebService – /apis/batch/v2alpha1 k8s.io/apiserver/pkg/endpoints.APIInstaller.Install /apis/batch/v2alpha1/jobs /apis/batch/v2alpha1/cronjobs ... WithRequestInfo ctx.RequestInfo
- 47. Restful http API mux k8s.io/apiserver/pkg/endpoints.APIGroupVersion.InstallREST AddSupportedResourcesWebService – /apis/batch/v2alpha1 k8s.io/apiserver/pkg/endpoints.APIInstaller.Install /apis/batch/v2alpha1/jobs /apis/batch/v2alpha1/cronjobs ... pkg/apis/batch type Jobs struct pkg/apis/batch/v2alpha1 type Jobs struct api.Scheme k8s.io/apiserver pkg/api api.Scheme.Convert(&internalJob, &v2alohaJob) /apis/batch/v2alpha1/jobs GET PUT POST DELETE ... /status /scale /proxy ... subresources
- 48. Restful http API mux pkg/apis/batch type Jobs struct pkg/apis/batch/v2alp ha1 type Jobs struct api.Sche me api.Scheme.Convert(&job, &v1job) POST /apis/batch/v2alpha1/jobs k8s.io/apiserver pkg/endpoints/handlers.CreateNamedResource binary JSON payload Go struct v2alpha1.Job HTTP Request Go struct internal.Job Store k8s.io/apiserver pkg/registry/generic Storage k8s.io/apiserver pkg/storage/etcd3 ProtoBuf Job Go struct v2alpha1.Job etcd
- 49. type Scheme struct • AddKnownTypes(gv, obj Object) • Default(src Object) • Copy(src Object) Object • Convert(in, out interface{}) • New(gvk) Object ApiGroup pkg/apis/batch pkg/apis/batch/v1 pkg/apis/batch/v2alpha1 pkg/apis/batch/register.go pgk/apis/batch/install Group Version Kind Resource type Object interface • GetObjectKind() string client-go/pkg/api.Scheme client-go/pkg/api.Codecs Discovery type APIGroupList struct type APIVersions struct type APIResourceList struct GroupVersionKind „gvk“ GroupVersionResource Unversioned types Unstructured List Registry / Storage type Storage interface type Lister interface type Updater interface type Getter interface type Deleter interface .... deepcopy-gen conversion-gen defaulting-gen Code Generation type OwnerReference struct type ObjectReference struct type TypeMeta struct type ObjectMeta struct Meta api.Scheme api.Codecs api.Registry api.GroupFactoryRegistry Globals