diff --git a/Makefile b/Makefile index 5eff1eaf3792a..bd83e8bdd9e5f 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -# Copyright 2016 The Kubernetes Authors. +# Copyright 2019 The Kubernetes Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -54,7 +54,7 @@ unexport KOPS_BASE_URL KOPS_CLUSTER_NAME KOPS_RUN_OBSOLETE_VERSION KOPS_STATE_ST unexport SKIP_REGION_CHECK S3_ACCESS_KEY_ID S3_ENDPOINT S3_REGION S3_SECRET_ACCESS_KEY VSPHERE_USERNAME VSPHERE_PASSWORD # Keep in sync with upup/models/cloudup/resources/addons/dns-controller/ -DNS_CONTROLLER_TAG=1.11.0-alpha.1 +DNS_CONTROLLER_TAG=1.11.0 # Keep in sync with logic in get_workspace_status # TODO: just invoke tools/get_workspace_status.sh? @@ -486,6 +486,7 @@ dep-ensure: dep-prereqs rm -rf vendor/k8s.io/code-generator/cmd/set-gen/ rm -rf vendor/k8s.io/code-generator/cmd/go-to-protobuf/ rm -rf vendor/k8s.io/code-generator/cmd/import-boss/ + rm -rf vendor/github.com/docker/docker/contrib/ make bazel-gazelle @@ -772,8 +773,14 @@ bazel-version-dist: bazel-crossbuild-nodeup bazel-crossbuild-kops bazel-protokub bazel-upload: bazel-version-dist # Upload kops to S3 aws s3 sync --acl public-read ${BAZELUPLOAD}/ ${S3_BUCKET} -#----------------------------------------------------------- -# static html documentation +# prow-postsubmit is run by the prow postsubmit job +# It uploads a build to a staging directory, which in theory we can publish as a release +.PHONY: prow-postsubmit +prow-postsubmit: bazel-version-dist + ${UPLOAD} ${BAZELUPLOAD}/kops/${VERSION}/ ${UPLOAD_DEST}/${KOPS_RELEASE_VERSION}-${GITSHA}/ + +#----------------------------------------------------------- +# static html documentation .PHONY: live-docs live-docs: @@ -783,8 +790,9 @@ live-docs: build-docs: @docker run --rm -it -v ${PWD}:/docs aledbf/mkdocs:0.1 build +# Update machine_types.go .PHONY: update-machine-types -update-machine-types: #Update machine_types.go - go build -o hack/machine_types/machine_types ${KOPS_ROOT}/hack/machine_types/machine_types.go +update-machine-types: + go build -o hack/machine_types/machine_types ${KOPS_ROOT}/hack/machine_types/ hack/machine_types/machine_types --out upup/pkg/fi/cloudup/awsup/machine_types.go go fmt upup/pkg/fi/cloudup/awsup/machine_types.go diff --git a/README.md b/README.md index c9abc90a13281..2c6f73b498153 100644 --- a/README.md +++ b/README.md @@ -197,7 +197,7 @@ Our office hours call is recorded, but the tone tends to be casual. First-timers Office hours are designed for ALL of those contributing to kops or the community. Contributions are not limited to those who commit source code. There are so many important ways to be involved- - helping in the slack channels - triaging/writing issues - - thinking about the topics raised at office hours and forming and advocating for your good ideas forming opinions + - thinking about the topics raised at office hours and forming and advocating for your good ideas forming opinions - testing pre-(and official) releases Although not exhaustive, the above activities are extremely important to our continued success and are all worth contributions. If you want to talk about kops and you have doubt, just come. diff --git a/ROADMAP.md b/ROADMAP.md index 2cb6703e6df02..6189917c483eb 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -1,99 +1,51 @@ # ROADMAP -The kops 1.N.x version officially supports kubernetes 1.N.x and earlier. While kubernetes 1.99 will likely run with kops 1.98, -the configuration will probably not be correct (for example docker versions, CNI versions etc). +## VERSION SUPPORT +kops 1.N.x _officially_ supports Kubernetes 1.N.x and prior versions. We understand that those in the community run a wide selection of versions and we do our best to maintain backward compatibility as far as we can. -kops 1.N.0 is released when it is believed that kubernetes 1.N.x is stable, along with all the core addons (e.g. networking). -This can mean that kops can release months after the release of kubernetes. It's also not a deterministic release criteria, -particularly with some networking plugins that are supported by kops but themselves still under development. We discussed -this challenge in kops office hours in March 2018, and the consensus was that we want to keep this, but that we should release -alphas & betas much earlier so that users can try out new kubernetes versions on release day. +However, kops 1.N.x does NOT support Kubernetes 1.N+1.x. Sometimes you get lucky and kops 1.N will technically install a later version of Kubernetes, but we cannot guarantee or support this situation. As always, we recommend waiting for the official release of kops with minor version >= the version of Kubernetes you wish to install. Please see the [compatibility matrix](README.md#Compatibility_Matrix) for further questions. -For the next few releases this means that: +## RELEASE SCHEDULE +There is a natural lag between the release of Kubernetes and the corresponding version of kops that has full support for it. While the first patch versions of a minor Kubernetes release are burning in, the kops team races to incorporate all the updates needed to release. Once we have both some stability in the upstream version of Kubernetes AND full support in kops, we will cut a release that includes version specific configuration and a selection of add-ons to match. -* 1.9.0 release target April 7th -* 1.10 alpha.1 with release of kops 1.9.0 (April 7th) -* 1.10 release target April 28th -* 1.11 alpha.1 at release of kops 1.10 -* 1.11 beta.1 at release of k8s 1.11 -* 1.12 alpha.1 at release of kops 1.11 etc +In practice, sometimes this means that kops release lags the upstream release by 1 or more months. We sincerely try to avoid this scenario- we understand how important this project is and respect the need that teams have to maintain their clusters. +Our goal is to have an official kops release no later than a month after the corresponding Kubernetes version is released. Please help us achieve this timeline and meet our goals by jumping in and giving us a hand. We always need assistance closing issues, reviewing PRs, and contributing code! Stop by office hours if you're interested. -### _kops 1.10_ +A rough outline of the timeline/release cycle with respect to the Kubernetes release follows. We are revising the automation around the release process so that we can get alpha and beta releases out to the community and other developers much faster for testing and to get more eyes on open issues. -* Support for kubernetes 1.10 -* Full support for GCE -* Make the etcd-backup tool enabled-by-default, so everyone should have backups. -* Allow users to opt-in to the full etcd-manager. -* Make etcd3 the default for new clusters, now that we have an upgrade path. -* Beginning of separation of addon functionality -* Support for more clouds (Aliyun, DigitalOcean, OpenStack) +Example release timeline based on Kubernetes quarterly release cycle: +July 1: Kubernetes 1.W.0 is released. +July 7: kops 1.W.beta1 +July 21: kops 1.W.0 released +August 15: kops 1.W+1alpha1 +August 31: kops 1.W+1alpha2 +etc... +September 25: Kubernetes1.W+1.RC-X +Oct 1: Kubernetes 1.W+1.0 +Oct 7: kops 1.W+1beta1 +Oct 21: kops 1.W+1.0 -### _kops 1.11_ -* Make the etcd-manager the default, deprecate the protokube-integrated approach -* kops-server -* Machines API support (including bare-metal) +## UPCOMING RELEASES -# HISTORICAL +### kops 1.11 -# 1.9 +* Full support for Kubernetes 1.11 +* Alpha support for bundles (etcd-manager is the test case) +* etcd3 will be the default for newly created clusters. + - Existing clusters will continue to run etcd2 but will be prompted to upgrade to 3. The upgrade will become mandatory in 1.12. +* Default to Debian stretch images which increase support for newer instance types +* Improvements to the release process that will make it easier for community members to cut releases. -## Must-have features +### kops 1.12 +* Full support for Kubernetes 1.12 +* Improvements to etcd3 support -* Support for k8s 1.9 _done_ -* etcd backup support _done_ - -## Other features - -* Use NodeAuthorizer / bootstrap kubeconfigs [#3551](https://github.com/kubernetes/kops/issues/3551) _no progress; may be less important with machines API_ - -# 1.8 - -## Must-have features - -* Support for k8s 1.8 - -## Other features - -* Improved GCE support -* Support for API aggregation - -# 1.7 - -## Must-have features - -* Support for k8s 1.7 - -## Other features we are working on in the 1.7 timeframe - -* etcd controller to allow moving between versions -* kops server for better team scenarios -* support for bare-metal -* more gossip backends -* IAM integration -* more cloud providers -* promote GCE to stable -* RBAC policies for all components -* bringing rolling-update out of alpha - -## 1.6 - -### Must-have features - -* Support for k8s 1.6 _done_ -* RBAC enabled by default _yes, but we kept RBAC optional_ - -## Other features we are working on in the 1.6 timeframe - -* Support for GCE _alpha_ -* Support for Google's [Container Optimized OS](https://cloud.google.com/container-optimized-os) (formerly known as GCI) _alpha_ -* Some support for bare-metal _private branches, not merged_ -* Some support for more cloud providers _initial work on vsphere_ -* Some IAM integration _discussions, but no code_ -* Federation made easy _no progress_ -* Authentication made easy _no progress_ -* Integration with kubeadm _kops now uses kubeadm for some RBAC related functionality_ -* CloudFormation integration on AWS _beta_ +## UPCOMING FEATURES +NB: These are features that are in process and may be introduced behind flags or in alpha capacity but are not explicitly targeting specific releases. +* Documentation revamp that is closer to k8s.io: Stories and walkthroughs of common scenarios, restructure and update information +* Additional cloud provider support: spotinst, aliyun, azure...? +* Revisit recommended base cluster configurations to get them modernized. Update recommendations and defaults for instances, disks, etc, diff --git a/WORKSPACE b/WORKSPACE index d9ed5b1c97a57..871ff0ddf243f 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -1,16 +1,19 @@ +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") + #============================================================================= # Go rules http_archive( name = "io_bazel_rules_go", - url = "https://github.com/bazelbuild/rules_go/releases/download/0.14.1/rules_go-0.14.1.tar.gz", - sha256 = "ee0e3b346388c447f13009d789c8bf2d7bae4643ac70bd7997ded0ad09b2fff7", + url = "https://github.com/bazelbuild/rules_go/releases/download/0.15.11/rules_go-0.15.11.tar.gz", + sha256 = "7b7c74740e3a757204ddb93241ce728906af795d6c6aa0950e0e640716dc1e4a", ) http_archive( name = "bazel_gazelle", - url = "https://github.com/bazelbuild/bazel-gazelle/releases/download/0.14.0/bazel-gazelle-0.14.0.tar.gz", - sha256 = "c0a5739d12c6d05b6c1ad56f2200cb0b57c5a70e03ebd2f7b87ce88cabf09c7b", + url = "https://github.com/bazelbuild/bazel-gazelle/releases/download/0.15.0/bazel-gazelle-0.15.0.tar.gz", + sha256 = "6e875ab4b6bf64a38c352887760f21203ab054676d9c1b274963907e0768740d", ) load("@io_bazel_rules_go//go:def.bzl", "go_rules_dependencies", "go_register_toolchains") @@ -18,7 +21,7 @@ load("@io_bazel_rules_go//go:def.bzl", "go_rules_dependencies", "go_register_too go_rules_dependencies() go_register_toolchains( - go_version = "1.10.3", + go_version = "1.10.8", ) load("@bazel_gazelle//:deps.bzl", "gazelle_dependencies") @@ -31,7 +34,7 @@ gazelle_dependencies() git_repository( name = "io_bazel_rules_docker", remote = "https://github.com/bazelbuild/rules_docker.git", - tag = "v0.4.0", + tag = "v0.5.1", ) load( @@ -54,7 +57,7 @@ container_pull( git_repository( name = "distroless", remote = "https://github.com/googlecloudplatform/distroless.git", - commit = "886114394dfed219001ec3b068b139a3456e49d4", + commit = "3585653b2b0d33c3fb369b907ef68df8344fd2ad", ) load( diff --git a/addons/cluster-autoscaler/cluster-autoscaler.sh b/addons/cluster-autoscaler/cluster-autoscaler.sh index 9b021bb6df66d..ccacb58d0b107 100755 --- a/addons/cluster-autoscaler/cluster-autoscaler.sh +++ b/addons/cluster-autoscaler/cluster-autoscaler.sh @@ -14,6 +14,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +set -e + #Set all the variables in this section CLUSTER_NAME="myfirstcluster.k8s.local" CLOUD_PROVIDER=aws @@ -61,6 +63,7 @@ cat > asg-policy.json << EOF "Action": [ "autoscaling:DescribeAutoScalingGroups", "autoscaling:DescribeAutoScalingInstances", + "autoscaling:DescribeLaunchConfigurations", "autoscaling:DescribeTags", "autoscaling:SetDesiredCapacity", "autoscaling:TerminateInstanceInAutoScalingGroup" diff --git a/addons/cluster-autoscaler/v1.10.0.yaml b/addons/cluster-autoscaler/v1.10.0.yaml index c0317a5200dda..1aba3610583ff 100644 --- a/addons/cluster-autoscaler/v1.10.0.yaml +++ b/addons/cluster-autoscaler/v1.10.0.yaml @@ -155,7 +155,7 @@ spec: - --stderrthreshold=info - --cloud-provider={{CLOUD_PROVIDER}} - --skip-nodes-with-local-storage=false - - --nodes={{MIN_NODES}:{{MAX_NODES}}:{{GROUP_NAME}} + - --nodes={{MIN_NODES}}:{{MAX_NODES}}:{{GROUP_NAME}} env: - name: AWS_REGION value: {{AWS_REGION}} diff --git a/addons/metrics-server/README.md b/addons/metrics-server/README.md index f7d3c841be5e4..ec5fceaa3799e 100644 --- a/addons/metrics-server/README.md +++ b/addons/metrics-server/README.md @@ -1,9 +1,27 @@ # Kubernetes Metrics Server +## User guide + +You can find the user guide in +[the official Kubernetes documentation](https://kubernetes.io/docs/tasks/debug-application-cluster/core-metrics-pipeline/). + +## Design + +The detailed design of the project can be found in the following docs: + +- [Metrics API](https://github.com/kubernetes/community/blob/master/contributors/design-proposals/instrumentation/resource-metrics-api.md) +- [Metrics Server](https://github.com/kubernetes/community/blob/master/contributors/design-proposals/instrumentation/metrics-server.md) + +For the broader view of monitoring in Kubernetes take a look into +[Monitoring architecture](https://github.com/kubernetes/community/blob/master/contributors/design-proposals/instrumentation/monitoring_architecture.md) + +## Deployment + Compatibility matrix: Metrics Server | Metrics API group/version | Supported Kubernetes version ---------------|---------------------------|----------------------------- +0.3.x | `metrics.k8s.io/v1beta1` | 1.8+ 0.2.x | `metrics.k8s.io/v1beta1` | 1.8+ 0.1.x | `metrics/v1alpha1` | 1.7 @@ -18,3 +36,44 @@ $ kubectl apply -f https://raw.githubusercontent.com/kubernetes/kops/master/addo # Kubernetes 1.8+ $ kubectl apply -f https://raw.githubusercontent.com/kubernetes/kops/master/addons/metrics-server/v1.8.x.yaml ``` + +## Flags + +Metrics Server supports all the standard Kubernetes API server flags, as +well as the standard Kubernetes `glog` logging flags. The most +commonly-used ones are: + +- `--logtostderr`: log to standard error instead of files in the + container. You generally want this on. + +- `--v=`: set log verbosity. It's generally a good idea to run a log + level 1 or 2 unless you're encountering errors. At log level 10, large + amounts of diagnostic information will be reported, include API request + and response bodies, and raw metric results from Kubelet. + +- `--secure-port=`: set the secure port. If you're not running as + root, you'll want to set this to something other than the default (port + 443). + +- `--tls-cert-file`, `--tls-private-key-file`: the serving certificate and + key files. If not specified, self-signed certificates will be + generated, but it's recommended that you use non-self-signed + certificates in production. + +Additionally, Metrics Server defines a number of flags for configuring its +behavior: + +- `--metric-resolution=`: the interval at which metrics will be + scraped from Kubelets (defaults to 60s). + +- `--kubelet-insecure-tls`: skip verifying Kubelet CA certificates. Not + recommended for production usage, but can be useful in test clusters + with self-signed Kubelet serving certificates. + +- `--kubelet-port`: the port to use to connect to the Kubelet (defaults to + the default secure Kubelet port, 10250). + +- `--kubelet-preferred-address-types`: the order in which to consider + different Kubelet node address types when connecting to Kubelet. + Functions similarly to the flag of the same name on the API server. +s \ No newline at end of file diff --git a/addons/metrics-server/v1.8.x.yaml b/addons/metrics-server/v1.8.x.yaml index aef36387e7e5b..412de71c4915c 100644 --- a/addons/metrics-server/v1.8.x.yaml +++ b/addons/metrics-server/v1.8.x.yaml @@ -57,6 +57,19 @@ rules: - list - watch --- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: system:aggregated-metrics-reader + labels: + rbac.authorization.k8s.io/aggregate-to-view: "true" + rbac.authorization.k8s.io/aggregate-to-edit: "true" + rbac.authorization.k8s.io/aggregate-to-admin: "true" +rules: + - apiGroups: ["metrics.k8s.io"] + resources: ["pods"] + verbs: ["get", "list", "watch"] +--- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: @@ -99,7 +112,7 @@ spec: protocol: TCP targetPort: 443 --- -apiVersion: apps/v1beta2 +apiVersion: extensions/v1beta1 kind: Deployment metadata: name: metrics-server @@ -117,17 +130,14 @@ spec: k8s-app: metrics-server spec: serviceAccountName: metrics-server + volumes: + # mount in tmp so we can safely use from-scratch images and/or read-only containers + - name: tmp-dir + emptyDir: {} containers: - - name: metrics-server - image: gcr.io/google_containers/metrics-server-amd64:v0.2.1 - imagePullPolicy: Always - command: - - /metrics-server - - --source=kubernetes.summary_api:'' - resources: - requests: - cpu: 20m - memory: 200Mi - limits: - cpu: 100m - memory: 500Mi + - name: metrics-server + image: gcr.io/google_containers/metrics-server-amd64:v0.3.1 + imagePullPolicy: Always + volumeMounts: + - name: tmp-dir + mountPath: /tmp diff --git a/addons/monitoring-standalone/addon.yaml b/addons/monitoring-standalone/addon.yaml index f5ce000ded8ee..af0995839a284 100644 --- a/addons/monitoring-standalone/addon.yaml +++ b/addons/monitoring-standalone/addon.yaml @@ -19,8 +19,12 @@ spec: selector: k8s-addon: monitoring-standalone.addons.k8s.io manifest: v1.6.0.yaml - kubernetesVersion: ">=1.6.0" -- version: 1.11.0 + - version: 1.7.0 + selector: + k8s-addon: monitoring-standalone.addons.k8s.io + manifest: v1.7.0.yaml + kubernetesVersion: ">=1.7.0" + - version: 1.11.0 selector: k8s-addon: monitoring-standalone.addons.k8s.io manifest: v1.11.0.yaml diff --git a/addons/prometheus-operator/v0.19.0.yaml b/addons/prometheus-operator/v0.19.0.yaml index 8fec572a48086..b6dfb1290a53a 100644 --- a/addons/prometheus-operator/v0.19.0.yaml +++ b/addons/prometheus-operator/v0.19.0.yaml @@ -1378,7 +1378,7 @@ spec: imagePullSecrets: description: An optional list of references to secrets in the same namespace to use for pulling prometheus and alertmanager images from registries - see http://kubernetes.io/docs/user-guide/images#specifying-imagepullsecrets-on-a-pod + see https://kubernetes.io/docs/concepts/containers/images/#specifying-imagepullsecrets-on-a-pod items: description: LocalObjectReference contains enough information to let you locate the referenced object inside the same namespace. @@ -1410,7 +1410,7 @@ spec: description: 'Annotations is an unstructured key value map stored with a resource that may be set by external tools to store and retrieve arbitrary metadata. They are not queryable and should - be preserved when modifying objects. More info: http://kubernetes.io/docs/user-guide/annotations' + be preserved when modifying objects. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/' type: object clusterName: description: The name of the cluster which the object belongs to. @@ -1550,7 +1550,7 @@ spec: uid: description: 'UID of the resource. (when there is a single resource which can be described). More info: - http://kubernetes.io/docs/user-guide/identifiers#uids' + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' type: string kind: description: 'Kind is a string value representing the REST @@ -1608,20 +1608,20 @@ spec: description: 'Map of string keys and values that can be used to organize and categorize (scope and select) objects. May match selectors of replication controllers and services. More info: - http://kubernetes.io/docs/user-guide/labels' + https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/' type: object name: description: 'Name must be unique within a namespace. Is required when creating resources, although some resources may allow a client to request the generation of an appropriate name automatically. Name is primarily intended for creation idempotence and configuration - definition. Cannot be updated. More info: http://kubernetes.io/docs/user-guide/identifiers#names' + definition. Cannot be updated. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' type: string namespace: description: |- Namespace defines the space within each name must be unique. An empty namespace is equivalent to the "default" namespace, but "default" is the canonical representation. Not all objects are required to be scoped to a namespace - the value of this field for those objects will be empty. - Must be a DNS_LABEL. Cannot be updated. More info: http://kubernetes.io/docs/user-guide/namespaces + Must be a DNS_LABEL. Cannot be updated. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ type: string ownerReferences: description: List of objects depended by this object. If ALL objects @@ -1652,10 +1652,10 @@ spec: description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' type: string name: - description: 'Name of the referent. More info: http://kubernetes.io/docs/user-guide/identifiers#names' + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' type: string uid: - description: 'UID of the referent. More info: http://kubernetes.io/docs/user-guide/identifiers#uids' + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' type: string required: - apiVersion @@ -1877,7 +1877,7 @@ spec: stored with a resource that may be set by external tools to store and retrieve arbitrary metadata. They are not queryable and should be preserved when modifying objects. - More info: http://kubernetes.io/docs/user-guide/annotations' + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/' type: object clusterName: description: The name of the cluster which the object belongs @@ -2087,7 +2087,7 @@ spec: description: 'Map of string keys and values that can be used to organize and categorize (scope and select) objects. May match selectors of replication controllers and services. - More info: http://kubernetes.io/docs/user-guide/labels' + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/' type: object name: description: 'Name must be unique within a namespace. Is @@ -2095,13 +2095,13 @@ spec: may allow a client to request the generation of an appropriate name automatically. Name is primarily intended for creation idempotence and configuration definition. Cannot be updated. - More info: http://kubernetes.io/docs/user-guide/identifiers#names' + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' type: string namespace: description: |- Namespace defines the space within each name must be unique. An empty namespace is equivalent to the "default" namespace, but "default" is the canonical representation. Not all objects are required to be scoped to a namespace - the value of this field for those objects will be empty. - Must be a DNS_LABEL. Cannot be updated. More info: http://kubernetes.io/docs/user-guide/namespaces + Must be a DNS_LABEL. Cannot be updated. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ type: string ownerReferences: description: List of objects depended by this object. If @@ -2135,10 +2135,10 @@ spec: description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' type: string name: - description: 'Name of the referent. More info: http://kubernetes.io/docs/user-guide/identifiers#names' + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' type: string uid: - description: 'UID of the referent. More info: http://kubernetes.io/docs/user-guide/identifiers#uids' + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' type: string required: - apiVersion @@ -2160,7 +2160,7 @@ spec: description: |- UID is the unique in time and space value for this object. It is typically generated by the server on successful creation of a resource and is not allowed to change on PUT operations. - Populated by the system. Read-only. More info: http://kubernetes.io/docs/user-guide/identifiers#uids + Populated by the system. Read-only. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids type: string spec: description: PersistentVolumeClaimSpec describes the common @@ -3724,7 +3724,7 @@ spec: imagePullSecrets: description: An optional list of references to secrets in the same namespace to use for pulling prometheus and alertmanager images from registries - see http://kubernetes.io/docs/user-guide/images#specifying-imagepullsecrets-on-a-pod + see https://kubernetes.io/docs/concepts/containers/images/#specifying-imagepullsecrets-on-a-pod items: description: LocalObjectReference contains enough information to let you locate the referenced object inside the same namespace. @@ -3755,7 +3755,7 @@ spec: description: 'Annotations is an unstructured key value map stored with a resource that may be set by external tools to store and retrieve arbitrary metadata. They are not queryable and should - be preserved when modifying objects. More info: http://kubernetes.io/docs/user-guide/annotations' + be preserved when modifying objects. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/' type: object clusterName: description: The name of the cluster which the object belongs to. @@ -3895,7 +3895,7 @@ spec: uid: description: 'UID of the resource. (when there is a single resource which can be described). More info: - http://kubernetes.io/docs/user-guide/identifiers#uids' + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' type: string kind: description: 'Kind is a string value representing the REST @@ -3953,20 +3953,20 @@ spec: description: 'Map of string keys and values that can be used to organize and categorize (scope and select) objects. May match selectors of replication controllers and services. More info: - http://kubernetes.io/docs/user-guide/labels' + https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/' type: object name: description: 'Name must be unique within a namespace. Is required when creating resources, although some resources may allow a client to request the generation of an appropriate name automatically. Name is primarily intended for creation idempotence and configuration - definition. Cannot be updated. More info: http://kubernetes.io/docs/user-guide/identifiers#names' + definition. Cannot be updated. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' type: string namespace: description: |- Namespace defines the space within each name must be unique. An empty namespace is equivalent to the "default" namespace, but "default" is the canonical representation. Not all objects are required to be scoped to a namespace - the value of this field for those objects will be empty. - Must be a DNS_LABEL. Cannot be updated. More info: http://kubernetes.io/docs/user-guide/namespaces + Must be a DNS_LABEL. Cannot be updated. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ type: string ownerReferences: description: List of objects depended by this object. If ALL objects @@ -3997,10 +3997,10 @@ spec: description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' type: string name: - description: 'Name of the referent. More info: http://kubernetes.io/docs/user-guide/identifiers#names' + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' type: string uid: - description: 'UID of the referent. More info: http://kubernetes.io/docs/user-guide/identifiers#uids' + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' type: string required: - apiVersion @@ -4022,7 +4022,7 @@ spec: description: |- UID is the unique in time and space value for this object. It is typically generated by the server on successful creation of a resource and is not allowed to change on PUT operations. - Populated by the system. Read-only. More info: http://kubernetes.io/docs/user-guide/identifiers#uids + Populated by the system. Read-only. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids type: string remoteRead: description: If specified, the remote_read spec. This is an experimental @@ -4566,7 +4566,7 @@ spec: stored with a resource that may be set by external tools to store and retrieve arbitrary metadata. They are not queryable and should be preserved when modifying objects. - More info: http://kubernetes.io/docs/user-guide/annotations' + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/' type: object clusterName: description: The name of the cluster which the object belongs @@ -4713,7 +4713,7 @@ spec: uid: description: 'UID of the resource. (when there is a single resource which can be described). - More info: http://kubernetes.io/docs/user-guide/identifiers#uids' + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' type: string kind: description: 'Kind is a string value representing @@ -4776,7 +4776,7 @@ spec: description: 'Map of string keys and values that can be used to organize and categorize (scope and select) objects. May match selectors of replication controllers and services. - More info: http://kubernetes.io/docs/user-guide/labels' + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/' type: object name: description: 'Name must be unique within a namespace. Is @@ -4784,13 +4784,13 @@ spec: may allow a client to request the generation of an appropriate name automatically. Name is primarily intended for creation idempotence and configuration definition. Cannot be updated. - More info: http://kubernetes.io/docs/user-guide/identifiers#names' + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' type: string namespace: description: |- Namespace defines the space within each name must be unique. An empty namespace is equivalent to the "default" namespace, but "default" is the canonical representation. Not all objects are required to be scoped to a namespace - the value of this field for those objects will be empty. - Must be a DNS_LABEL. Cannot be updated. More info: http://kubernetes.io/docs/user-guide/namespaces + Must be a DNS_LABEL. Cannot be updated. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ type: string ownerReferences: description: List of objects depended by this object. If @@ -4824,10 +4824,10 @@ spec: description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' type: string name: - description: 'Name of the referent. More info: http://kubernetes.io/docs/user-guide/identifiers#names' + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' type: string uid: - description: 'UID of the referent. More info: http://kubernetes.io/docs/user-guide/identifiers#uids' + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' type: string required: - apiVersion @@ -4849,7 +4849,7 @@ spec: description: |- UID is the unique in time and space value for this object. It is typically generated by the server on successful creation of a resource and is not allowed to change on PUT operations. - Populated by the system. Read-only. More info: http://kubernetes.io/docs/user-guide/identifiers#uids + Populated by the system. Read-only. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids type: string spec: description: PersistentVolumeClaimSpec describes the common diff --git a/channels/pkg/channels/addons_test.go b/channels/pkg/channels/addons_test.go index 6c03ed9b119aa..5b6ea934dc585 100644 --- a/channels/pkg/channels/addons_test.go +++ b/channels/pkg/channels/addons_test.go @@ -83,7 +83,7 @@ func Test_Replacement(t *testing.T) { New *ChannelVersion Replaces bool }{ - // With no id, update iff newer semver + // With no id, update if and only if newer semver { Old: &ChannelVersion{Version: s("1.0.0"), Id: ""}, New: &ChannelVersion{Version: s("1.0.0"), Id: ""}, diff --git a/channels/pkg/channels/channel_version.go b/channels/pkg/channels/channel_version.go index ab6a53e3e60aa..57ff01e205a5b 100644 --- a/channels/pkg/channels/channel_version.go +++ b/channels/pkg/channels/channel_version.go @@ -120,9 +120,8 @@ func (c *ChannelVersion) replaces(existing *ChannelVersion) bool { // Same version; check ids if c.Id == existing.Id { return false - } else { - glog.V(4).Infof("Channels had same version %q but different ids (%q vs %q); will replace", *c.Version, c.Id, existing.Id) } + glog.V(4).Infof("Channels had same version %q but different ids (%q vs %q); will replace", *c.Version, c.Id, existing.Id) } } diff --git a/cmd/kops/BUILD.bazel b/cmd/kops/BUILD.bazel index 4a6c0bb91f05c..76f7f26d7912f 100644 --- a/cmd/kops/BUILD.bazel +++ b/cmd/kops/BUILD.bazel @@ -76,8 +76,10 @@ go_library( "//pkg/featureflag:go_default_library", "//pkg/formatter:go_default_library", "//pkg/instancegroups:go_default_library", + "//pkg/k8sversion:go_default_library", "//pkg/kopscodecs:go_default_library", "//pkg/kubeconfig:go_default_library", + "//pkg/model/components:go_default_library", "//pkg/pki:go_default_library", "//pkg/pretty:go_default_library", "//pkg/resources:go_default_library", diff --git a/cmd/kops/create.go b/cmd/kops/create.go index 901abd931144a..4d31b2f14ef0b 100644 --- a/cmd/kops/create.go +++ b/cmd/kops/create.go @@ -20,7 +20,6 @@ import ( "bytes" "fmt" "io" - "os" "github.com/golang/glog" "github.com/spf13/cobra" @@ -167,10 +166,9 @@ func RunCreate(f *util.Factory, out io.Writer, c *CreateOptions) error { return fmt.Errorf("cluster %q already exists", v.ObjectMeta.Name) } return fmt.Errorf("error creating cluster: %v", err) - } else { - fmt.Fprintf(&sb, "Created cluster/%s\n", v.ObjectMeta.Name) - //cSpec = true } + fmt.Fprintf(&sb, "Created cluster/%s\n", v.ObjectMeta.Name) + //cSpec = true case *kopsapi.InstanceGroup: clusterName = v.ObjectMeta.Labels[kopsapi.LabelClusterName] @@ -219,9 +217,8 @@ func RunCreate(f *util.Factory, out io.Writer, c *CreateOptions) error { err = sshCredentialStore.AddSSHPublicKey("admin", sshKeyArr) if err != nil { return err - } else { - fmt.Fprintf(&sb, "Added ssh credential\n") } + fmt.Fprintf(&sb, "Added ssh credential\n") default: glog.V(2).Infof("Type of object was %T", v) @@ -245,15 +242,3 @@ func RunCreate(f *util.Factory, out io.Writer, c *CreateOptions) error { } return nil } - -// ConsumeStdin reads all the bytes available from stdin -func ConsumeStdin() ([]byte, error) { - file := os.Stdin - buf := new(bytes.Buffer) - _, err := buf.ReadFrom(file) - if err != nil { - return nil, fmt.Errorf("error reading stdin: %v", err) - } - - return buf.Bytes(), nil -} diff --git a/cmd/kops/create_cluster.go b/cmd/kops/create_cluster.go index 66184b1ac27e6..d8fca3c2ed5fc 100644 --- a/cmd/kops/create_cluster.go +++ b/cmd/kops/create_cluster.go @@ -33,6 +33,7 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/validation/field" "k8s.io/kops" "k8s.io/kops/cmd/kops/util" api "k8s.io/kops/pkg/apis/kops" @@ -43,6 +44,8 @@ import ( "k8s.io/kops/pkg/commands" "k8s.io/kops/pkg/dns" "k8s.io/kops/pkg/featureflag" + "k8s.io/kops/pkg/k8sversion" + "k8s.io/kops/pkg/model/components" "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/cloudup" "k8s.io/kops/upup/pkg/fi/cloudup/aliup" @@ -80,6 +83,7 @@ type CreateClusterOptions struct { VPCID string SubnetIDs []string UtilitySubnetIDs []string + DisableSubnetTags bool NetworkCIDR string DNSZone string AdminAccess []string @@ -294,6 +298,7 @@ func NewCmdCreateCluster(f *util.Factory, out io.Writer) *cobra.Command { cmd.Flags().StringSliceVar(&options.SubnetIDs, "subnets", options.SubnetIDs, "Set to use shared subnets") cmd.Flags().StringSliceVar(&options.UtilitySubnetIDs, "utility-subnets", options.UtilitySubnetIDs, "Set to use shared utility subnets") cmd.Flags().StringVar(&options.NetworkCIDR, "network-cidr", options.NetworkCIDR, "Set to override the default network CIDR") + cmd.Flags().BoolVar(&options.DisableSubnetTags, "disable-subnet-tags", options.DisableSubnetTags, "Set to disable automatic subnet tagging") cmd.Flags().Int32Var(&options.MasterCount, "master-count", options.MasterCount, "Set the number of masters. Defaults to one master per master-zone") cmd.Flags().Int32Var(&options.NodeCount, "node-count", options.NodeCount, "Set the number of nodes") @@ -496,9 +501,8 @@ func RunCreateCluster(f *util.Factory, out io.Writer, c *CreateClusterOptions) e if cluster.Spec.CloudProvider == "" { if allZones.Len() == 0 { return fmt.Errorf("must specify --zones or --cloud") - } else { - return fmt.Errorf("unable to infer CloudProvider from Zones (is there a typo in --zones?)") } + return fmt.Errorf("unable to infer CloudProvider from Zones (is there a typo in --zones?)") } } @@ -918,7 +922,17 @@ func RunCreateCluster(f *util.Factory, out io.Writer, c *CreateClusterOptions) e Backend: "udp", } case "calico": - cluster.Spec.Networking.Calico = &api.CalicoNetworkingSpec{} + cluster.Spec.Networking.Calico = &api.CalicoNetworkingSpec{ + MajorVersion: "v3", + } + // Validate to check if etcd clusters have an acceptable version + if errList := validation.ValidateEtcdVersionForCalicoV3(cluster.Spec.EtcdClusters[0], cluster.Spec.Networking.Calico.MajorVersion, field.NewPath("Calico")); len(errList) != 0 { + + // This is not a special version but simply of the 3 series + for _, etcd := range cluster.Spec.EtcdClusters { + etcd.Version = components.DefaultEtcd3Version_1_11 + } + } case "canal": cluster.Spec.Networking.Canal = &api.CanalNetworkingSpec{} case "kube-router": @@ -929,6 +943,8 @@ func RunCreateCluster(f *util.Factory, out io.Writer, c *CreateClusterOptions) e cluster.Spec.Networking.AmazonVPC = &api.AmazonVPCNetworkingSpec{} case "cilium": cluster.Spec.Networking.Cilium = &api.CiliumNetworkingSpec{} + case "lyftvpc": + cluster.Spec.Networking.LyftVPC = &api.LyftVPCNetworkingSpec{} default: return fmt.Errorf("unknown networking mode %q", c.Networking) } @@ -946,6 +962,8 @@ func RunCreateCluster(f *util.Factory, out io.Writer, c *CreateClusterOptions) e c.Topology = api.TopologyPublic } + cluster.Spec.DisableSubnetTags = c.DisableSubnetTags + switch c.Topology { case api.TopologyPublic: cluster.Spec.Topology = &api.TopologySpec{ @@ -964,7 +982,7 @@ func RunCreateCluster(f *util.Factory, out io.Writer, c *CreateClusterOptions) e case api.TopologyPrivate: if !supportsPrivateTopology(cluster.Spec.Networking) { - return fmt.Errorf("Invalid networking option %s. Currently only '--networking kopeio-vxlan (or kopeio)', '--networking weave', '--networking flannel', '--networking calico', '--networking canal', '--networking kube-router', '--networking romana', '--networking amazon-vpc-routed-eni' are supported for private topologies", c.Networking) + return fmt.Errorf("Invalid networking option %s. Currently only '--networking kopeio-vxlan (or kopeio)', '--networking weave', '--networking flannel', '--networking calico', '--networking canal', '--networking kube-router', '--networking romana', '--networking amazon-vpc-routed-eni' '--networking lyftvpc' are supported for private topologies", c.Networking) } cluster.Spec.Topology = &api.TopologySpec{ Masters: api.TopologyPrivate, @@ -1044,6 +1062,22 @@ func RunCreateCluster(f *util.Factory, out io.Writer, c *CreateClusterOptions) e cluster.Spec.MasterPublicName = c.MasterPublicName } + kv, err := k8sversion.Parse(cluster.Spec.KubernetesVersion) + if err != nil { + return fmt.Errorf("failed to parse kubernetes version: %s", err.Error()) + } + + // check if we should set anonymousAuth to false on k8s versions >=1.11 + if kv.IsGTE("1.11") { + if cluster.Spec.Kubelet == nil { + cluster.Spec.Kubelet = &api.KubeletConfigSpec{} + } + + if cluster.Spec.Kubelet.AnonymousAuth == nil { + cluster.Spec.Kubelet.AnonymousAuth = fi.Bool(false) + } + } + // Populate the API access, so that it can be discoverable // TODO: This is the same code as in defaults - try to dedup? if cluster.Spec.API == nil { @@ -1273,7 +1307,7 @@ func RunCreateCluster(f *util.Factory, out io.Writer, c *CreateClusterOptions) e func supportsPrivateTopology(n *api.NetworkingSpec) bool { - if n.CNI != nil || n.Kopeio != nil || n.Weave != nil || n.Flannel != nil || n.Calico != nil || n.Canal != nil || n.Kuberouter != nil || n.Romana != nil || n.AmazonVPC != nil || n.Cilium != nil { + if n.CNI != nil || n.Kopeio != nil || n.Weave != nil || n.Flannel != nil || n.Calico != nil || n.Canal != nil || n.Kuberouter != nil || n.Romana != nil || n.AmazonVPC != nil || n.Cilium != nil || n.LyftVPC != nil { return true } return false @@ -1368,11 +1402,9 @@ func loadSSHPublicKeys(sshPublicKey string) (map[string][]byte, error) { authorized, err := ioutil.ReadFile(sshPublicKey) if err != nil { return nil, err - } else { - sshPublicKeys[fi.SecretNameSSHPrimary] = authorized - - glog.Infof("Using SSH public key: %v\n", sshPublicKey) } + sshPublicKeys[fi.SecretNameSSHPrimary] = authorized + glog.Infof("Using SSH public key: %v\n", sshPublicKey) } return sshPublicKeys, nil } diff --git a/cmd/kops/create_cluster_test.go b/cmd/kops/create_cluster_test.go index d94eb1e7119f9..483b61303f4a4 100644 --- a/cmd/kops/create_cluster_test.go +++ b/cmd/kops/create_cluster_test.go @@ -36,9 +36,8 @@ func checkParse(t *testing.T, s string, expect map[string]string, shouldErr bool if err != nil { if shouldErr { return - } else { - t.Errorf(err.Error()) } + t.Errorf(err.Error()) } for k, v := range expect { diff --git a/cmd/kops/create_secret_dockerconfig.go b/cmd/kops/create_secret_dockerconfig.go index bb80029b1017d..da51dc0ffc00e 100644 --- a/cmd/kops/create_secret_dockerconfig.go +++ b/cmd/kops/create_secret_dockerconfig.go @@ -40,6 +40,9 @@ var ( # Create a new docker config. kops create secret dockerconfig -f /path/to/docker/config.json \ --name k8s-cluster.example.com --state s3://example.com + # Create a docker config via stdin. + generate-docker-config.sh | kops create secret dockerconfig -f - \ + --name k8s-cluster.example.com --state s3://example.com # Replace an existing docker config secret. kops create secret dockerconfig -f /path/to/docker/config.json --force \ --name k8s-cluster.example.com --state s3://example.com @@ -110,10 +113,17 @@ func RunCreateSecretDockerConfig(f *util.Factory, out io.Writer, options *Create if err != nil { return err } - - data, err := ioutil.ReadFile(options.DockerConfigPath) - if err != nil { - return fmt.Errorf("error reading docker config %v: %v", options.DockerConfigPath, err) + var data []byte + if options.DockerConfigPath == "-" { + data, err = ConsumeStdin() + if err != nil { + return fmt.Errorf("error reading docker config from stdin: %v", err) + } + } else { + data, err = ioutil.ReadFile(options.DockerConfigPath) + if err != nil { + return fmt.Errorf("error reading docker config %v: %v", options.DockerConfigPath, err) + } } var parsedData map[string]interface{} diff --git a/cmd/kops/create_secret_encryptionconfig.go b/cmd/kops/create_secret_encryptionconfig.go index 0c4794e88167e..420889ce74550 100644 --- a/cmd/kops/create_secret_encryptionconfig.go +++ b/cmd/kops/create_secret_encryptionconfig.go @@ -40,6 +40,9 @@ var ( # Create a new encryption config. kops create secret encryptionconfig -f config.yaml \ --name k8s-cluster.example.com --state s3://example.com + # Create a new encryption config via stdin. + generate-encryption-config.sh | kops create secret encryptionconfig -f - \ + --name k8s-cluster.example.com --state s3://example.com # Replace an existing encryption config secret. kops create secret encryptionconfig -f config.yaml --force \ --name k8s-cluster.example.com --state s3://example.com @@ -111,10 +114,17 @@ func RunCreateSecretEncryptionConfig(f *util.Factory, out io.Writer, options *Cr if err != nil { return err } - - data, err := ioutil.ReadFile(options.EncryptionConfigPath) - if err != nil { - return fmt.Errorf("error reading encryption config %v: %v", options.EncryptionConfigPath, err) + var data []byte + if options.EncryptionConfigPath == "-" { + data, err = ConsumeStdin() + if err != nil { + return fmt.Errorf("error reading encryption config from stdin: %v", err) + } + } else { + data, err = ioutil.ReadFile(options.EncryptionConfigPath) + if err != nil { + return fmt.Errorf("error reading encryption config %v: %v", options.EncryptionConfigPath, err) + } } var parsedData map[string]interface{} diff --git a/cmd/kops/create_secret_weave_encryptionconfig.go b/cmd/kops/create_secret_weave_encryptionconfig.go index 2e6a27d8ae0f8..499cd645da93e 100644 --- a/cmd/kops/create_secret_weave_encryptionconfig.go +++ b/cmd/kops/create_secret_weave_encryptionconfig.go @@ -45,6 +45,9 @@ var ( # Install a specific weave password. kops create secret weavepassword -f /path/to/weavepassword \ --name k8s-cluster.example.com --state s3://example.com + # Install a specific weave password via stdin. + kops create secret weavepassword -f - \ + --name k8s-cluster.example.com --state s3://example.com # Replace an existing weavepassword secret. kops create secret weavepassword -f /path/to/weavepassword --force \ --name k8s-cluster.example.com --state s3://example.com @@ -112,9 +115,18 @@ func RunCreateSecretWeaveEncryptionConfig(f *util.Factory, options *CreateSecret } if options.WeavePasswordFilePath != "" { - data, err := ioutil.ReadFile(options.WeavePasswordFilePath) - if err != nil { - return fmt.Errorf("error reading weave password file %v: %v", options.WeavePasswordFilePath, err) + var data []byte + if options.WeavePasswordFilePath == "-" { + data, err = ConsumeStdin() + if err != nil { + return fmt.Errorf("error reading weave password file from stdin: %v", err) + } + } else { + data, err = ioutil.ReadFile(options.WeavePasswordFilePath) + if err != nil { + return fmt.Errorf("error reading weave password file %v: %v", options.WeavePasswordFilePath, err) + } + } secret.Data = data diff --git a/cmd/kops/delete.go b/cmd/kops/delete.go index 264532c068b63..940e02e1385ec 100644 --- a/cmd/kops/delete.go +++ b/cmd/kops/delete.go @@ -51,7 +51,10 @@ var ( deleteExample = templates.Examples(i18n.T(` # Delete a cluster using a manifest file kops delete -f my-cluster.yaml - + + # Delete a cluster using a pasted manifest file from stdin. + pbpaste | kops delete -f - + # Delete a cluster in AWS. kops delete cluster --name=k8s.example.com --state=s3://kops-state-1234 @@ -101,9 +104,18 @@ func RunDelete(factory *util.Factory, out io.Writer, d *DeleteOptions) error { deletedClusters := sets.NewString() for _, f := range d.Filenames { - contents, err := vfs.Context.ReadFile(f) - if err != nil { - return fmt.Errorf("error reading file %q: %v", f, err) + var contents []byte + var err error + if f == "-" { + contents, err = ConsumeStdin() + if err != nil { + return fmt.Errorf("error reading from stdin: %v", err) + } + } else { + contents, err = vfs.Context.ReadFile(f) + if err != nil { + return fmt.Errorf("error reading file %q: %v", f, err) + } } sections := bytes.Split(contents, []byte("\n---\n")) diff --git a/cmd/kops/replace.go b/cmd/kops/replace.go index 86375e34428ed..7d502583c8aa3 100644 --- a/cmd/kops/replace.go +++ b/cmd/kops/replace.go @@ -165,9 +165,8 @@ func RunReplace(f *util.Factory, cmd *cobra.Command, out io.Writer, c *replaceOp if err != nil { if errors.IsNotFound(err) { return fmt.Errorf("cluster %q not found", clusterName) - } else { - return fmt.Errorf("error fetching cluster %q: %v", clusterName, err) } + return fmt.Errorf("error fetching cluster %q: %v", clusterName, err) } // check if the instancegroup exists already igName := v.ObjectMeta.Name diff --git a/cmd/kops/rollingupdatecluster.go b/cmd/kops/rollingupdatecluster.go index dc5e9d0a1f701..e77eace623311 100644 --- a/cmd/kops/rollingupdatecluster.go +++ b/cmd/kops/rollingupdatecluster.go @@ -177,6 +177,7 @@ func NewCmdRollingUpdateCluster(f *util.Factory, out io.Writer) *cobra.Command { cmd.Flags().BoolVar(&options.Force, "force", options.Force, "Force rolling update, even if no changes") cmd.Flags().BoolVar(&options.CloudOnly, "cloudonly", options.CloudOnly, "Perform rolling update without confirming progress with k8s") + cmd.Flags().DurationVar(&options.ValidationTimeout, "validation-timeout", options.ValidationTimeout, "Maximum time to wait for a cluster to validate") cmd.Flags().DurationVar(&options.MasterInterval, "master-interval", options.MasterInterval, "Time to wait between restarting masters") cmd.Flags().DurationVar(&options.NodeInterval, "node-interval", options.NodeInterval, "Time to wait between restarting nodes") cmd.Flags().DurationVar(&options.BastionInterval, "bastion-interval", options.BastionInterval, "Time to wait between restarting bastions") diff --git a/cmd/kops/root.go b/cmd/kops/root.go index b9fc48526c082..87e02b923f121 100644 --- a/cmd/kops/root.go +++ b/cmd/kops/root.go @@ -17,6 +17,7 @@ limitations under the License. package main import ( + "bytes" goflag "flag" "fmt" "io" @@ -220,9 +221,8 @@ func (c *RootCmd) ProcessArgs(args []string) error { if len(args) == 1 { return fmt.Errorf("Cannot specify cluster via --name and positional argument") - } else { - return fmt.Errorf("expected a single to be passed as an argument") } + return fmt.Errorf("expected a single to be passed as an argument") } func (c *RootCmd) ClusterName() string { @@ -326,3 +326,14 @@ func GetCluster(factory Factory, clusterName string) (*kopsapi.Cluster, error) { } return cluster, nil } + +// ConsumeStdin reads all the bytes available from stdin +func ConsumeStdin() ([]byte, error) { + file := os.Stdin + buf := new(bytes.Buffer) + _, err := buf.ReadFrom(file) + if err != nil { + return nil, fmt.Errorf("error reading stdin: %v", err) + } + return buf.Bytes(), nil +} diff --git a/cmd/kops/toolbox_template.go b/cmd/kops/toolbox_template.go index bcc493daf0151..e6a9ece790235 100644 --- a/cmd/kops/toolbox_template.go +++ b/cmd/kops/toolbox_template.go @@ -210,7 +210,7 @@ func runToolBoxTemplate(f *util.Factory, out io.Writer, options *toolboxTemplate return nil } -// newTemplateContext is responsible for loadding the --values and build a context for the template +// newTemplateContext is responsible for loading the --values and build a context for the template func newTemplateContext(files []string, values []string, stringValues []string) (map[string]interface{}, error) { context := make(map[string]interface{}, 0) diff --git a/cmd/kops/util/factory.go b/cmd/kops/util/factory.go index e368f57fd64fe..75d9210222ef7 100644 --- a/cmd/kops/util/factory.go +++ b/cmd/kops/util/factory.go @@ -53,12 +53,12 @@ func NewFactory(options *FactoryOptions) *Factory { const ( STATE_ERROR = `Please set the --state flag or export KOPS_STATE_STORE. -A valid value follows the format s3://. -A s3 bucket is required to store cluster state information.` +For example, a valid value follows the format s3://. +You can find the supported stores in https://github.com/kubernetes/kops/blob/master/docs/state.md.` - INVALID_STATE_ERROR = `Unable to read state store s3 bucket. -Please use a valid s3 bucket uri when setting --state or KOPS_STATE_STORE env var. -A valid value follows the format s3://. + INVALID_STATE_ERROR = `Unable to read state store. +Please use a valid state store when setting --state or KOPS_STATE_STORE env var. +For example, a valid value follows the format s3://. Trailing slash will be trimmed.` ) diff --git a/dns-controller/README.md b/dns-controller/README.md index ba977385d53f7..bce977c1fbed6 100644 --- a/dns-controller/README.md +++ b/dns-controller/README.md @@ -7,7 +7,7 @@ we have an `etcd` cluster and an `apiserver`. It also sets up DNS records for the `etcd` nodes (this is a much simpler problem, because we have a 1:1 mapping from an `etcd` node to a DNS name.) -However, none of the nodes can reach the api server to register. Nor +However, none of the nodes can reach the API server to register. Nor can end-users reach the API. In future we might expose the API server as a normal service via `Type=LoadBalancer` or via a normal Ingress, but for now we just expose it via DNS. diff --git a/dns-controller/pkg/dns/dnscache.go b/dns-controller/pkg/dns/dnscache.go index 1817673d5bb0f..5bc61ab9c3ac5 100644 --- a/dns-controller/pkg/dns/dnscache.go +++ b/dns-controller/pkg/dns/dnscache.go @@ -68,9 +68,8 @@ func (d *dnsCache) ListZones(validity time.Duration) ([]dnsprovider.Zone, error) if d.cachedZones != nil { if (d.cachedZonesTimestamp + validity.Nanoseconds()) > now { return d.cachedZones, nil - } else { - glog.V(2).Infof("querying all DNS zones (cache expired)") } + glog.V(2).Infof("querying all DNS zones (cache expired)") } else { glog.V(2).Infof("querying all DNS zones (no cached results)") } diff --git a/docker/Dockerfile b/docker/Dockerfile index ab0faf9104f4e..6d9d0b79b03f0 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,4 +1,4 @@ -# Copyright 2017 The Kubernetes Authors. +# Copyright 2019 The Kubernetes Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,9 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM alpine:3.6 +FROM alpine:3.8 -ARG GO_VERSION=1.8.1 +ARG GO_VERSION=1.10.8 # KOPS_GITISH: Modify to build at an explicit tag/gitish ARG KOPS_GITISH=release diff --git a/docs/addons.md b/docs/addons.md index 304a4ae393ddd..fb3634e734b62 100644 --- a/docs/addons.md +++ b/docs/addons.md @@ -8,7 +8,74 @@ Addons in Kubernetes are traditionally done by copying files to `/etc/kubernetes doesn't really make sense in HA master configurations. We also have kubectl available, and addons are just a thin wrapper over calling kubectl. -This document describes how to install some common addons. +The command `kops create cluster` does not support specifying addons to be added to the cluster when it is created. Instead they can be added after cluster creation using kubectl. Alternatively when creating a cluster from a yaml manifest, addons can be specified using `spec.addons`. +```yaml +spec: + addons: + - kubernetes-dashboard + - s3://kops-addons/addon.yaml +``` + +This document describes how to install some common addons and how to create your own custom ones. + +### Custom addons + +The docs about the [addon manager](addon_manager.md) describe in more detail how to define a addon resource with regards to versioning. +Here is a minimal example of an addon manifest that would install two different addons. + +```yaml +kind: Addons +metadata: + name: example +spec: + addons: + - name: foo.addons.org.io + version: 0.0.1 + selector: + k8s-addon: foo.addons.org.io + manifest: foo.addons.org.io/v0.0.1.yaml + - name: bar.addons.org.io + version: 0.0.1 + selector: + k8s-addon: bar.addons.org.io + manifest: bar.addons.org.io/v0.0.1.yaml +``` + +In this this example the folder structure should look like this; + +``` +addon.yaml + foo.addons.org.io + v0.0.1.yaml + bar.addons.org.io + v0.0.1.yaml +``` + +The yaml files in the foo/bar folders can be any kubernetes resource. Typically this file structure would be pushed to S3 or another of the supported backends and then referenced as above in `spec.addons`. In order for master nodes to be able to access the S3 bucket containing the addon manifests, one might have to add additional iam policies to the master nodes using `spec.additionalPolicies`, like so; +```yaml +spec: + additionalPolicies: + master: | + [ + { + "Effect": "Allow", + "Action": [ + "s3:GetObject" + ], + "Resource": ["arn:aws:s3:::kops-addons/*"] + }, + { + "Effect": "Allow", + "Action": [ + "s3:GetBucketLocation", + "s3:ListBucket" + ], + "Resource": ["arn:aws:s3:::kops-addons"] + } + ] +``` +The masters will poll for changes changes in the bucket and keep the addons up to date. + ### Dashboard diff --git a/docs/api-server/README.md b/docs/api-server/README.md index 5d7d16e69016f..b34f9dd4dcdcf 100644 --- a/docs/api-server/README.md +++ b/docs/api-server/README.md @@ -14,7 +14,7 @@ Build the kops API server container, and push the image up to your registry. kops-server-push ``` -# Deploy the kops API server to a cluster +# Deploying the kops API server to a cluster From the kops directory run the following `helm` command. More information on `helm` can be found [here](https://github.com/kubernetes/helm) diff --git a/docs/apireference/build/tabvisibility.js b/docs/apireference/build/tabvisibility.js index 48c0df7fe2684..9589fac92c5af 100644 --- a/docs/apireference/build/tabvisibility.js +++ b/docs/apireference/build/tabvisibility.js @@ -18,10 +18,10 @@ $(document).ready(function() { }); } - function setDefautTab() { + function setDefaultTab() { $(codeTabs[0]).addClass('tab-selected'); $('.' + codeTabs[0].id).addClass('active'); } - setDefautTab(); -}); \ No newline at end of file + setDefaultTab(); +}); diff --git a/docs/apireference/openapi-spec/swagger.json b/docs/apireference/openapi-spec/swagger.json index d0bff1397fd22..4cb6a2ee95010 100644 --- a/docs/apireference/openapi-spec/swagger.json +++ b/docs/apireference/openapi-spec/swagger.json @@ -3,6 +3,7 @@ "/apis", "/apis/kops", "/apis/kops/v1alpha2", + "/metrics", "/version" ] } \ No newline at end of file diff --git a/docs/calico-v3.md b/docs/calico-v3.md new file mode 100644 index 0000000000000..7e85183ec4323 --- /dev/null +++ b/docs/calico-v3.md @@ -0,0 +1,96 @@ +# Calico Version 3 +In early 2018 Version 3 of Calico was released, it included a reworked data +model and with that a switch from the etcd v2 to v3 API. This document covers +the requirements, upgrade process, and configuration to install +Calico Version 3. + +## Requirements +- The main requirement needed for Calico Version 3 is the etcd v3 API available + with etcd server version 3. +- Another requirement is for the Kubernetes version to be a minimum of v1.7.0. + +### etcd +Due to the etcd v3 API being a requirement of Calico Version 3 +(when using etcd as the datastore) not all Kops installations will be +upgradable to Calico V3. Installations using etcd v2 (or earlier) will need +to remain on Calico V2 or update to etcdv3. + +## Configuration of a new cluster +To ensure a new cluster will have Calico Version 3 installed the following +two configurations options should be set: +- `spec.etcdClusters.etcdMembers[0].Version` (Main cluster) should be + set to a Version of etcd greater than 3.x or the default version + needs to be greater than 3.x. +- The Networking config must have the Calico MajorVersion set to `v3` like + the following: + ``` + spec: + networking: + calico: + majorVersion: v3 + ``` + +Both of the above two settings can be set by doing a `kops edit cluster ...` +before bringing the cluster up for the first time. + +With the above two settings your Kops deployed cluster will be running with +Calico Version 3. + +### Create cluster networking flag + +When enabling Calico with the `--networking calico` flag, etcd will be set to +a v3 version. Feel free to change to a different v3 version of etcd. + +## Upgrading an existing cluster +Assuming your cluster meets the requirements it is possible to upgrade +your Calico Kops cluster. + +A few notes about the upgrade: +- During the first portion of the migration, while the calico-kube-controllers + pod is running its Init, no new policies will be applied though already + applied policy will be active. +- During the migration no new pods will be scheduled as adding new workloads + to Calico is blocked. Once the calico-complete-upgrade job has completed + pods will once again be schedulable. +- The upgrade process that has been automated in kops can be found in + [the Upgrading Calico docs](https://docs.projectcalico.org/v3.1/getting-started/kubernetes/upgrade/upgrade). + +Perform the upgrade with the following steps: + +1. First you must ensure that you are running Calico V2.6.5+. With the + latest Kops (greater than 1.9) ensuring your cluster is updated can be + done by doing a `kops update` on the cluster. +1. Verify your Calico data will migrate successfully by installing and + configuring the + [calico-upgrade command](https://docs.projectcalico.org/v3.1/getting-started/kubernetes/upgrade/setup) + and then run `calico-upgrade dry-run` and verify it reports that the + migration can be completed successfully. +1. Set `majorVersion` field as below by editing + your cluster configuration with `kops edit cluster`. + ``` + spec: + networking: + calico: + majorVersion: v3 + ``` +1. Update your cluster with `kops update` like you would normally update. +1. Monitor the progress of the migration by using + `kubectl get pods -n kube-system` and checking the status of the following pods: + - calico-node pods should restart one at a time and all becoming Running + - calico-kube-controllers pod will restart and after the first calico-node + pod starts running it will start running + - calico-complete-upgrade pod will be Completed after all the calico-node + pods start running + If any of the above fail by entering a crash loop you should investigate + by checking the logs with `kubectl -n kube-system logs `. +1. Once the calico-node and calico-kube-controllers are running and the + calico-complete-upgrade pod has completed the migration has finished + successfully. + +### Recovering from a partial migration + +The InitContainer of the first calico-node pod that starts will perform the +datastore migration necessary for upgrading from Calico v2 to Calico v3, if +this InitContainer is killed or restarted when the new datastore is being +populated it will be necessary to manually remove the Calico data in the +etcd v3 API before the migration will be successful. diff --git a/docs/cli/kops_create_cluster.md b/docs/cli/kops_create_cluster.md index d93793a5776a3..48d067bd2c36d 100644 --- a/docs/cli/kops_create_cluster.md +++ b/docs/cli/kops_create_cluster.md @@ -73,6 +73,7 @@ kops create cluster [flags] --channel string Channel for default versions and configuration to use (default "stable") --cloud string Cloud provider to use - gce, aws, vsphere --cloud-labels string A list of KV pairs used to tag all instance groups in AWS (eg "Owner=John Doe,Team=Some Team"). + --disable-subnet-tags Set to disable automatic subnet tagging --dns string DNS hosted zone to use: public|private. (default "Public") --dns-zone string DNS hosted zone to use (defaults to longest matching zone) --dry-run If true, only print the object that would be sent, without sending it. This flag can be used to create a cluster YAML or JSON manifest. diff --git a/docs/cli/kops_create_secret_dockerconfig.md b/docs/cli/kops_create_secret_dockerconfig.md index 7a3212379c14b..3f0d0d8a3b074 100644 --- a/docs/cli/kops_create_secret_dockerconfig.md +++ b/docs/cli/kops_create_secret_dockerconfig.md @@ -19,6 +19,9 @@ kops create secret dockerconfig [flags] # Create a new docker config. kops create secret dockerconfig -f /path/to/docker/config.json \ --name k8s-cluster.example.com --state s3://example.com + # Create a docker config via stdin. + generate-docker-config.sh | kops create secret dockerconfig -f - \ + --name k8s-cluster.example.com --state s3://example.com # Replace an existing docker config secret. kops create secret dockerconfig -f /path/to/docker/config.json --force \ --name k8s-cluster.example.com --state s3://example.com diff --git a/docs/cli/kops_create_secret_encryptionconfig.md b/docs/cli/kops_create_secret_encryptionconfig.md index 2cdbebf7f584c..d77dce56eb17a 100644 --- a/docs/cli/kops_create_secret_encryptionconfig.md +++ b/docs/cli/kops_create_secret_encryptionconfig.md @@ -19,6 +19,9 @@ kops create secret encryptionconfig [flags] # Create a new encryption config. kops create secret encryptionconfig -f config.yaml \ --name k8s-cluster.example.com --state s3://example.com + # Create a new encryption config via stdin. + generate-encryption-config.sh | kops create secret encryptionconfig -f - \ + --name k8s-cluster.example.com --state s3://example.com # Replace an existing encryption config secret. kops create secret encryptionconfig -f config.yaml --force \ --name k8s-cluster.example.com --state s3://example.com diff --git a/docs/cli/kops_create_secret_weavepassword.md b/docs/cli/kops_create_secret_weavepassword.md index 9a4923e1c8e05..c85df46c5a7a6 100644 --- a/docs/cli/kops_create_secret_weavepassword.md +++ b/docs/cli/kops_create_secret_weavepassword.md @@ -26,6 +26,9 @@ kops create secret weavepassword [flags] # Install a specific weave password. kops create secret weavepassword -f /path/to/weavepassword \ --name k8s-cluster.example.com --state s3://example.com + # Install a specific weave password via stdin. + kops create secret weavepassword -f - \ + --name k8s-cluster.example.com --state s3://example.com # Replace an existing weavepassword secret. kops create secret weavepassword -f /path/to/weavepassword --force \ --name k8s-cluster.example.com --state s3://example.com diff --git a/docs/cli/kops_delete.md b/docs/cli/kops_delete.md index 4a084e525083c..e6ab52868a27a 100644 --- a/docs/cli/kops_delete.md +++ b/docs/cli/kops_delete.md @@ -19,6 +19,9 @@ kops delete -f FILENAME [--yes] [flags] # Delete a cluster using a manifest file kops delete -f my-cluster.yaml + # Delete a cluster using a pasted manifest file from stdin. + pbpaste | kops delete -f - + # Delete a cluster in AWS. kops delete cluster --name=k8s.example.com --state=s3://kops-state-1234 diff --git a/docs/cli/kops_rolling-update_cluster.md b/docs/cli/kops_rolling-update_cluster.md index 5259afd7e4e43..817912c6f2f15 100644 --- a/docs/cli/kops_rolling-update_cluster.md +++ b/docs/cli/kops_rolling-update_cluster.md @@ -78,6 +78,7 @@ kops rolling-update cluster [flags] -i, --interactive Prompt to continue after each instance is updated --master-interval duration Time to wait between restarting masters (default 5m0s) --node-interval duration Time to wait between restarting nodes (default 4m0s) + --validation-timeout duration Maximum time to wait for a cluster to validate (default 5m0s) -y, --yes Perform rolling update immediately, without --yes rolling-update executes a dry-run ``` diff --git a/docs/cluster_spec.md b/docs/cluster_spec.md index 0175b4c1b0712..eac0d166d208c 100644 --- a/docs/cluster_spec.md +++ b/docs/cluster_spec.md @@ -85,19 +85,23 @@ etcdClusters: > __Note:__ The images for etcd that kops uses are from the Google Cloud Repository. Google doesn't release every version of etcd to the gcr. Check that the version of etcd you want to use is available [at the gcr](https://console.cloud.google.com/gcr/images/google-containers/GLOBAL/etcd?gcrImageListsize=50) before using it in your cluster spec. -By default, the Volumes created for the etcd clusters are 20GB each. They can be adjusted via the `volumeSize` parameter. +By default, the Volumes created for the etcd clusters are `gp2` and 20GB each. The volume size, type and Iops( for `io1`) can be configured via their parameters. Conversion between `gp2` and `io1` is not supported, nor are size changes. ```yaml etcdClusters: - etcdMembers: - instanceGroup: master-us-east-1a name: a - volumeSize: 5 + volumeType: gp2 + volumeSize: 20 name: main - etcdMembers: - instanceGroup: master-us-east-1a name: a - volumeSize: 5 + volumeType: io1 + # WARNING: bear in mind that the Iops to volume size ratio has a maximum of 50 on AWS! + volumeIops: 100 + volumeSize: 21 name: events ``` @@ -184,6 +188,14 @@ spec: ``` +#### profiling flag + +```yaml +spec: + kubeAPIServer: + profiling: true +``` + #### audit logging Read more about this here: https://kubernetes.io/docs/admin/audit @@ -264,6 +276,16 @@ spec: serviceNodePortRange: 30000-33000 ``` +#### Disable Basic Auth + +This will disable the passing of the `--basic-auth-file` flag. + +```yaml +spec: + kubeAPIServer: + disableBasicAuth: true +``` + #### targetRamMb Memory limit for apiserver in MB (used to configure sizes of caches, etc.) @@ -274,6 +296,26 @@ spec: targetRamMb: 4096 ``` +#### Disable Basic Auth + +This will disable the passing of the `--basic-auth-file` flag. + +```yaml +spec: + kubeAPIServer: + disableBasicAuth: true +``` + +#### Disable Token Auth + +This will disable the passing of the `--token-auth-file` flag. + +```yaml +spec: + kubeAPIServer: + disableTokenAuth: true +``` + ### externalDns This block contains configuration options for your `external-DNS` provider. @@ -336,6 +378,7 @@ This block contains configurations for `kube-scheduler`. See https://kubernetes spec: kubeScheduler: usePolicyConfigMap: true + profiling: true ``` Will make kube-scheduler use the scheduler policy from configmap "scheduler-policy" in namespace kube-system. @@ -371,6 +414,7 @@ spec: horizontalPodAutoscalerSyncPeriod: 15s horizontalPodAutoscalerDownscaleDelay: 5m0s horizontalPodAutoscalerUpscaleDelay: 3m0s + profiling: true ``` For more details on `horizontalPodAutoscaler` flags see the [official HPA docs](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/) and the [Kops guides on how to set it up](horizontal_pod_autoscaling.md). diff --git a/docs/etcd/manager.md b/docs/etcd/manager.md index d24f9e35b84da..00ac322deb048 100644 --- a/docs/etcd/manager.md +++ b/docs/etcd/manager.md @@ -41,7 +41,7 @@ it won't change the configuration: ```bash # Enable etcd-manager -kops set cluster cluster.spec.etcdClusters[*].manager.image=kopeio/etcd-manager:latest +kops set cluster cluster.spec.etcdClusters[*].provider=Manager kops update cluster --yes kops rolling-update cluster --yes diff --git a/docs/install.md b/docs/install.md index 446766bf43207..98d9450116f8d 100644 --- a/docs/install.md +++ b/docs/install.md @@ -25,7 +25,7 @@ You can also [install from source](development/building.md). From Github: ```bash -wget -O kops https://github.com/kubernetes/kops/releases/download/$(curl -s https://api.github.com/repos/kubernetes/kops/releases/latest | grep tag_name | cut -d '"' -f 4)/kops-linux-amd64 +curl -Lo kops https://github.com/kubernetes/kops/releases/download/$(curl -s https://api.github.com/repos/kubernetes/kops/releases/latest | grep tag_name | cut -d '"' -f 4)/kops-linux-amd64 chmod +x ./kops sudo mv ./kops /usr/local/bin/ ``` @@ -58,7 +58,7 @@ sudo mv ./kubectl /usr/local/bin/kubectl From the [official kubernetes kubectl release](https://kubernetes.io/docs/tasks/tools/install-kubectl/): ``` -wget -O kubectl https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl +curl -Lo kubectl https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl chmod +x ./kubectl sudo mv ./kubectl /usr/local/bin/kubectl ``` diff --git a/docs/networking.md b/docs/networking.md index 3c53fc9c5c111..62fc8e490e7ff 100644 --- a/docs/networking.md +++ b/docs/networking.md @@ -34,7 +34,7 @@ has built in support for CNI networking components. Several different CNI providers are currently built into kops: -* [Calico](http://docs.projectcalico.org/v2.0/getting-started/kubernetes/installation/hosted/) +* [Calico](https://docs.projectcalico.org/v3.1/getting-started/kubernetes/installation/calico#installing-with-the-etcd-datastore) * [Canal (Flannel + Calico)](https://github.com/projectcalico/canal) * [flannel](https://github.com/coreos/flannel) - use `--networking flannel-vxlan` (recommended) or `--networking flannel-udp` (legacy). `--networking flannel` now selects `flannel-vxlan`. * [kopeio-vxlan](https://github.com/kopeio/networking) @@ -122,7 +122,7 @@ $ kops create secret weavepassword -f password $ kops update cluster ``` -Since unencrypted nodes will not be able to connect to nodes configured with encryption enabled, this configuration cannot be changed easily without downtime. +Since unencrypted nodes will not be able to connect to nodes configured with encryption enabled, this configuration cannot be changed easily without downtime. ### Calico Example for CNI and Network Policy @@ -158,10 +158,10 @@ Reference: [Calico 2.1 Release Notes](https://www.projectcalico.org/project-cali Note that Calico by default, routes between nodes within a subnet are distributed using a full node-to-node BGP mesh. Each node automatically sets up a BGP peering with every other node within the same L2 network. This full node-to-node mesh per L2 network has its scaling challenges for larger scale deployments. -BGP route reflectors can be used as a replacement to a full mesh, and is useful for scaling up a cluster. +BGP route reflectors can be used as a replacement to a full mesh, and is useful for scaling up a cluster. BGP route reflectors are recommended once the number of nodes goes above ~50-100. The setup of BGP route reflectors is currently out of the scope of kops. -Read more here: [BGP route reflectors](http://docs.projectcalico.org/latest/usage/routereflector/calico-routereflector) +Read more here: [BGP route reflectors](https://docs.projectcalico.org/latest/usage/routereflector) To enable this mode in a cluster, with Calico as the CNI and Network Policy provider, you must edit the cluster after the previous `kops create ...` command. @@ -170,7 +170,8 @@ To enable this mode in a cluster, with Calico as the CNI and Network Policy prov ``` networking: - calico: {} + calico: + majorVersion: v3 ``` You will need to change that block, and add an additional field, to look like this: @@ -178,6 +179,7 @@ You will need to change that block, and add an additional field, to look like th ``` networking: calico: + majorVersion: v3 crossSubnet: true ``` @@ -194,6 +196,8 @@ Only the masters have the IAM policy (`ec2:*`) to allow k8s-ec2-srcdst to execut For Calico specific documentation please visit the [Calico Docs](http://docs.projectcalico.org/latest/getting-started/kubernetes/). +For details on upgrading a Calico v2 deployment see [Calico Version 3](calico-v3.md). + #### Getting help with Calico For help with Calico or to report any issues: diff --git a/docs/node_resource_handling.md b/docs/node_resource_handling.md index 3d0769a2079c2..0e55ebadc3f5d 100644 --- a/docs/node_resource_handling.md +++ b/docs/node_resource_handling.md @@ -128,4 +128,4 @@ But, it seems fitting to recommend the following: [5]: https://cloud.google.com/container-engine/ [6]: https://github.com/kubernetes/kops [7]: http://node-perf-dash.k8s.io/#/builds -[8]: http://blog.kubernetes.io/2016/11/visualize-kubelet-performance-with-node-dashboard.html +[8]: http://kubernetes.io/blog/2016/11/visualize-kubelet-performance-with-node-dashboard.html diff --git a/docs/philosophy.md b/docs/philosophy.md index 80aa97316909c..2f5b3cd2f9830 100644 --- a/docs/philosophy.md +++ b/docs/philosophy.md @@ -32,9 +32,7 @@ There are two primary types: ## State Store -The API objects are currently stored in an abstraction called a "state store", and currently the only implemented -storage is an S3 bucket. The storage of files in the S3 bucket is an implementation detail. Expect more state -stores soon. For example, it might be convenient to put the InstanceGroup into the kubernetes API itself. +The API objects are currently stored in an abstraction called a "state store". [state.md](/docs/state.md) has more detail. ## Configuration inference @@ -53,4 +51,4 @@ single source of truth and it is practical to implement alternatives to nodeup & such as kubelet might read their configuration directly from the state store in future, eliminating the need to have a management process that copies values around. -Currently the 'completed' cluster specification is stored in the state store in a file called `cluster.spec` \ No newline at end of file +Currently the 'completed' cluster specification is stored in the state store in a file called `cluster.spec` diff --git a/docs/run_in_existing_vpc.md b/docs/run_in_existing_vpc.md index 4b2dcac3921c7..a4daacdc4ca76 100644 --- a/docs/run_in_existing_vpc.md +++ b/docs/run_in_existing_vpc.md @@ -142,17 +142,11 @@ spec: kops update cluster ${CLUSTER_NAME} --yes ``` - **If you run in AWS private topology with shared subnets, and you would like Kubernetes to provision resources in these shared subnets, you must create tags on them.** - - **This is important, for example, if your `utility` subnets are shared, you will not be able to launch any services that create Elastic Load Balancers (ELBs).** - - **Prior to kops 1.8 `KubernetesCluster` tag was used for this. This lead to several problems if there were more than one Kubernetes Cluster in a subnet.** - - **After you upgraded to kops 1.8 remove `KubernetesCluster` Tag from subnets otherwise `kubernetes.io/cluster/` won't have any effect!** +### Subnet Tags - **These are currently needed Tags on shared resources:** + By default, kops will tag your existing subnets with the standard tags: - Public Subnets: + Public/Utility Subnets: ``` "kubernetes.io/cluster/" = "shared" "kubernetes.io/role/elb" = "1" @@ -165,7 +159,12 @@ spec: "kubernetes.io/role/internal-elb" = "1" "SubnetType" = "Private" ``` - + + These tags are important, for example, your services will be unable to create public or private Elastic Load Balancers (ELBs) if the respective `elb` or `internal-elb` tags are missing. + + If you would like to manage these tags externally then specify `--disable-subnet-tags` during your cluster creation. This will prevent kops from tagging existing subnets and allow some custom control, such as separate subnets for internal ELBs. + + Prior to kops 1.8 `KubernetesCluster` tag was used instead of `kubernetes.io/cluster/`. This lead to several problems if there were more than one Kubernetes Cluster in a subnet. After you upgraded to kops 1.8 ensure the `KubernetesCluster` Tag is removed from subnets otherwise `kubernetes.io/cluster/` won't have any effect! ### Shared NAT Egress diff --git a/docs/single-to-multi-master.md b/docs/single-to-multi-master.md index 0c7add57b87ff..5b5202a66f8a6 100644 --- a/docs/single-to-multi-master.md +++ b/docs/single-to-multi-master.md @@ -292,7 +292,7 @@ and not during a future upgrade or, worse, during a master failure. In case you failed to upgrade to multi-master you will need to restore from the backup you have taken previously. -Take extra care becase kops will not start etcd and etcd-events with the same ID on an/or for example but will mix them (ex: etcd-b and etcd-events-c on & etcd-c and etcd-events-b on ); this can be double checked in Route53 where kops will create DNS records for your services. +Take extra care because kops will not start etcd and etcd-events with the same ID on an/or for example but will mix them (ex: etcd-b and etcd-events-c on & etcd-c and etcd-events-b on ); this can be double checked in Route53 where kops will create DNS records for your services. If your 2nd spinned master failed and cluster becomes inconsistent edit the corresponding kops master instancegroup and switch ``MinSize`` and ``MaxSize`` to "0" and run an update on your cluster. diff --git a/docs/upgrade_from_kubeup.md b/docs/upgrade_from_kubeup.md index 71c9abf24273f..3038fc2c0fa29 100644 --- a/docs/upgrade_from_kubeup.md +++ b/docs/upgrade_from_kubeup.md @@ -185,7 +185,8 @@ kubectl delete deployment -lk8s-app=heapster --namespace=kube-system ## Delete remaining resources of the old cluster `kops delete cluster ${OLD_NAME}` -> ``` +> +``` TYPE NAME ID autoscaling-config kubernetes-minion-group-us-west-2a kubernetes-minion-group-us-west-2a autoscaling-group kubernetes-minion kubernetes-minion-group-us-west-2a diff --git a/hack/.packages b/hack/.packages index 7f5367ee6f373..2c78c321c9a53 100644 --- a/hack/.packages +++ b/hack/.packages @@ -130,6 +130,7 @@ k8s.io/kops/pkg/tokens k8s.io/kops/pkg/try k8s.io/kops/pkg/urls k8s.io/kops/pkg/util/stringorslice +k8s.io/kops/pkg/util/subnet k8s.io/kops/pkg/util/templater k8s.io/kops/pkg/validation k8s.io/kops/pkg/values diff --git a/hack/boilerplate/boilerplate.py b/hack/boilerplate/boilerplate.py index 21da02db9022c..8adc5adfa4481 100755 --- a/hack/boilerplate/boilerplate.py +++ b/hack/boilerplate/boilerplate.py @@ -176,7 +176,7 @@ def get_regexs(): # Search for "YEAR" which exists in the boilerplate, but shouldn't in the real thing regexs["year"] = re.compile( 'YEAR' ) # dates can be 2014, 2015, 2016, or 2017; company holder names can be anything - regexs["date"] = re.compile( '(2014|2015|2016|2017|2018)' ) + regexs["date"] = re.compile( '(2014|2015|2016|2017|2018|2019|2020|2021|2022)' ) # strip // +build \n\n build constraints regexs["go_build_constraints"] = re.compile(r"^(// \+build.*\n)+\n", re.MULTILINE) # strip #!.* from shell scripts diff --git a/hack/machine_types/BUILD.bazel b/hack/machine_types/BUILD.bazel index 8c93f7bceb825..d99c3354bbba4 100644 --- a/hack/machine_types/BUILD.bazel +++ b/hack/machine_types/BUILD.bazel @@ -2,7 +2,10 @@ load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") go_library( name = "go_default_library", - srcs = ["machine_types.go"], + srcs = [ + "machine_types.go", + "vpc_ip_resource_limit.go", + ], importpath = "k8s.io/kops/hack/machine_types", visibility = ["//visibility:private"], deps = [ diff --git a/hack/machine_types/machine_types.go b/hack/machine_types/machine_types.go index a9a6ee7810cb9..dc09d40714a50 100644 --- a/hack/machine_types/machine_types.go +++ b/hack/machine_types/machine_types.go @@ -152,6 +152,7 @@ func run() error { } } + seen := map[string]bool{} for _, item := range prices { for k, v := range item { if k == "product" { @@ -161,6 +162,11 @@ func run() error { attributes[k] = v.(string) } + if _, ok := seen[attributes["instanceType"]]; ok { + continue + } + seen[attributes["instanceType"]] = true + machine := awsup.AWSMachineTypeInfo{ Name: attributes["instanceType"], Cores: stringToInt(attributes["vcpu"]), @@ -189,10 +195,19 @@ func run() error { if attributes["ecu"] == "Variable" { machine.Burstable = true machine.ECU = t2CreditsPerHour[machine.Name] // This is actually credits * ECUs, but we'll add that later + } else if attributes["ecu"] == "NA" { + machine.ECU = 0 } else { machine.ECU = stringToFloat32(attributes["ecu"]) } + if enis, enisOK := InstanceENIsAvailable[attributes["instanceType"]]; enisOK { + machine.InstanceENIs = enis + } + if ipsPerENI, ipsOK := InstanceIPsAvailable[attributes["instanceType"]]; ipsOK { + machine.InstanceIPsPerENI = int(ipsPerENI) + } + machines = append(machines, machine) family := strings.Split(attributes["instanceType"], ".")[0] @@ -230,7 +245,14 @@ func run() error { for _, f := range sortedFamilies { output = output + fmt.Sprintf("\n// %s family", f) + previousMachine := "" for _, m := range machines { + // Ignore duplicates + if m.Name == previousMachine { + continue + } + previousMachine = m.Name + if family := strings.Split(m.Name, ".")[0]; family == f { var ecu string if m.Burstable { @@ -245,7 +267,9 @@ func run() error { MemoryGB: %v, ECU: %v, Cores: %v, - `, m.Name, m.MemoryGB, ecu, m.Cores) + InstanceENIs: %v, + InstanceIPsPerENI: %v, + `, m.Name, m.MemoryGB, ecu, m.Cores, m.InstanceENIs, m.InstanceIPsPerENI) output = output + body // Avoid awkward []int(nil) syntax diff --git a/hack/machine_types/vpc_ip_resource_limit.go b/hack/machine_types/vpc_ip_resource_limit.go new file mode 100644 index 0000000000000..e31372252e131 --- /dev/null +++ b/hack/machine_types/vpc_ip_resource_limit.go @@ -0,0 +1,388 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Copyright 2017-2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. + +package main + +// InstanceENIsAvailable contains a mapping of instance types to the number of ENIs available which is described at +// https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-eni.html#AvailableIpPerENI +var InstanceENIsAvailable = map[string]int{ + "a1.medium": 2, + "a1.large": 3, + "a1.xlarge": 4, + "a1.2xlarge": 4, + "a1.4xlarge": 8, + "c1.medium": 2, + "c1.xlarge": 4, + "c3.large": 3, + "c3.xlarge": 4, + "c3.2xlarge": 4, + "c3.4xlarge": 8, + "c3.8xlarge": 8, + "c4.large": 3, + "c4.xlarge": 4, + "c4.2xlarge": 4, + "c4.4xlarge": 8, + "c4.8xlarge": 8, + "c5.large": 3, + "c5.xlarge": 4, + "c5.2xlarge": 4, + "c5.4xlarge": 8, + "c5.9xlarge": 8, + "c5.18xlarge": 15, + "c5d.large": 3, + "c5d.xlarge": 4, + "c5d.2xlarge": 4, + "c5d.4xlarge": 8, + "c5d.9xlarge": 8, + "c5d.18xlarge": 15, + "c5n.large": 3, + "c5n.xlarge": 4, + "c5n.2xlarge": 4, + "c5n.4xlarge": 8, + "c5n.9xlarge": 8, + "c5n.18xlarge": 15, + "cc2.8xlarge": 8, + "cr1.8xlarge": 8, + "d2.xlarge": 4, + "d2.2xlarge": 4, + "d2.4xlarge": 8, + "d2.8xlarge": 8, + "f1.2xlarge": 4, + "f1.4xlarge": 8, + "f1.16xlarge": 8, + "g2.2xlarge": 4, + "g2.8xlarge": 8, + "g3s.xlarge": 4, + "g3.4xlarge": 8, + "g3.8xlarge": 8, + "g3.16xlarge": 15, + "h1.2xlarge": 4, + "h1.4xlarge": 8, + "h1.8xlarge": 8, + "h1.16xlarge": 15, + "hs1.8xlarge": 8, + "i2.xlarge": 4, + "i2.2xlarge": 4, + "i2.4xlarge": 8, + "i2.8xlarge": 8, + "i3.large": 3, + "i3.xlarge": 4, + "i3.2xlarge": 4, + "i3.4xlarge": 8, + "i3.8xlarge": 8, + "i3.16xlarge": 15, + "i3.metal": 15, + "m1.small": 2, + "m1.medium": 2, + "m1.large": 3, + "m1.xlarge": 4, + "m2.xlarge": 4, + "m2.2xlarge": 4, + "m2.4xlarge": 8, + "m3.medium": 2, + "m3.large": 3, + "m3.xlarge": 4, + "m3.2xlarge": 4, + "m4.large": 2, + "m4.xlarge": 4, + "m4.2xlarge": 4, + "m4.4xlarge": 8, + "m4.10xlarge": 8, + "m4.16xlarge": 8, + "m5.large": 3, + "m5.xlarge": 4, + "m5.2xlarge": 4, + "m5.4xlarge": 8, + "m5.12xlarge": 8, + "m5.24xlarge": 15, + "m5.metal": 15, + "m5a.large": 3, + "m5a.xlarge": 4, + "m5a.2xlarge": 4, + "m5a.4xlarge": 8, + "m5a.12xlarge": 8, + "m5a.24xlarge": 15, + "m5d.large": 3, + "m5d.xlarge": 4, + "m5d.2xlarge": 4, + "m5d.4xlarge": 8, + "m5d.12xlarge": 8, + "m5d.24xlarge": 15, + "m5d.metal": 15, + "p2.xlarge": 4, + "p2.8xlarge": 8, + "p2.16xlarge": 8, + "p3.2xlarge": 4, + "p3.8xlarge": 8, + "p3.16xlarge": 8, + "p3dn.24xlarge": 15, + "r3.large": 3, + "r3.xlarge": 4, + "r3.2xlarge": 4, + "r3.4xlarge": 8, + "r3.8xlarge": 8, + "r4.large": 3, + "r4.xlarge": 4, + "r4.2xlarge": 4, + "r4.4xlarge": 8, + "r4.8xlarge": 8, + "r4.16xlarge": 15, + "r5.large": 3, + "r5.xlarge": 4, + "r5.2xlarge": 4, + "r5.4xlarge": 8, + "r5.12xlarge": 8, + "r5.24xlarge": 15, + "r5.metal": 15, + "r5a.large": 3, + "r5a.xlarge": 4, + "r5a.2xlarge": 4, + "r5a.4xlarge": 8, + "r5a.12xlarge": 8, + "r5a.24xlarge": 15, + "r5d.large": 3, + "r5d.xlarge": 4, + "r5d.2xlarge": 4, + "r5d.4xlarge": 8, + "r5d.12xlarge": 8, + "r5d.24xlarge": 15, + "r5d.metal": 15, + "t1.micro": 2, + "t2.nano": 2, + "t2.micro": 2, + "t2.small": 3, + "t2.medium": 3, + "t2.large": 3, + "t2.xlarge": 3, + "t2.2xlarge": 3, + "t3.nano": 2, + "t3.micro": 2, + "t3.small": 3, + "t3.medium": 3, + "t3.large": 3, + "t3.xlarge": 4, + "t3.2xlarge": 4, + "u-6tb1.metal": 5, + "u-9tb1.metal": 5, + "u-12tb1.metal": 5, + "x1.16xlarge": 8, + "x1.32xlarge": 8, + "x1e.xlarge": 3, + "x1e.2xlarge": 4, + "x1e.4xlarge": 4, + "x1e.8xlarge": 4, + "x1e.16xlarge": 8, + "x1e.32xlarge": 8, + "z1d.large": 3, + "z1d.xlarge": 4, + "z1d.2xlarge": 4, + "z1d.3xlarge": 8, + "z1d.6xlarge": 8, + "z1d.12xlarge": 15, + "z1d.metal": 15, +} + +// InstanceIPsAvailable contains a mapping of instance types to the number of IPs per ENI +// https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-eni.html#AvailableIpPerENI +var InstanceIPsAvailable = map[string]int64{ + "a1.medium": 4, + "a1.large": 10, + "a1.xlarge": 15, + "a1.2xlarge": 15, + "a1.4xlarge": 30, + "c1.medium": 6, + "c1.xlarge": 15, + "c3.large": 10, + "c3.xlarge": 15, + "c3.2xlarge": 15, + "c3.4xlarge": 30, + "c3.8xlarge": 30, + "c4.large": 10, + "c4.xlarge": 15, + "c4.2xlarge": 15, + "c4.4xlarge": 30, + "c4.8xlarge": 30, + "c5.large": 10, + "c5.xlarge": 15, + "c5.2xlarge": 15, + "c5.4xlarge": 30, + "c5.9xlarge": 30, + "c5.18xlarge": 50, + "c5d.large": 10, + "c5d.xlarge": 15, + "c5d.2xlarge": 15, + "c5d.4xlarge": 30, + "c5d.9xlarge": 30, + "c5d.18xlarge": 50, + "c5n.large": 10, + "c5n.xlarge": 15, + "c5n.2xlarge": 15, + "c5n.4xlarge": 30, + "c5n.9xlarge": 30, + "c5n.18xlarge": 50, + "cc2.8xlarge": 30, + "cr1.8xlarge": 30, + "d2.xlarge": 15, + "d2.2xlarge": 15, + "d2.4xlarge": 30, + "d2.8xlarge": 30, + "f1.2xlarge": 15, + "f1.4xlarge": 30, + "f1.16xlarge": 50, + "g2.2xlarge": 15, + "g2.8xlarge": 30, + "g3s.xlarge": 15, + "g3.4xlarge": 30, + "g3.8xlarge": 30, + "g3.16xlarge": 50, + "h1.2xlarge": 15, + "h1.4xlarge": 30, + "h1.8xlarge": 30, + "h1.16xlarge": 50, + "hs1.8xlarge": 30, + "i2.xlarge": 15, + "i2.2xlarge": 15, + "i2.4xlarge": 30, + "i2.8xlarge": 30, + "i3.large": 10, + "i3.xlarge": 15, + "i3.2xlarge": 15, + "i3.4xlarge": 30, + "i3.8xlarge": 30, + "i3.16xlarge": 50, + "i3.metal": 50, + "m1.small": 4, + "m1.medium": 6, + "m1.large": 10, + "m1.xlarge": 15, + "m2.xlarge": 15, + "m2.2xlarge": 30, + "m2.4xlarge": 30, + "m3.medium": 6, + "m3.large": 10, + "m3.xlarge": 15, + "m3.2xlarge": 30, + "m4.large": 10, + "m4.xlarge": 15, + "m4.2xlarge": 15, + "m4.4xlarge": 30, + "m4.10xlarge": 30, + "m4.16xlarge": 30, + "m5.large": 10, + "m5.xlarge": 15, + "m5.2xlarge": 15, + "m5.4xlarge": 30, + "m5.12xlarge": 30, + "m5.24xlarge": 50, + "m5.metal": 50, + "m5a.large": 10, + "m5a.xlarge": 15, + "m5a.2xlarge": 15, + "m5a.4xlarge": 30, + "m5a.12xlarge": 30, + "m5a.24xlarge": 50, + "m5d.large": 10, + "m5d.xlarge": 15, + "m5d.2xlarge": 15, + "m5d.4xlarge": 30, + "m5d.12xlarge": 30, + "m5d.24xlarge": 50, + "m5d.metal": 50, + "p2.xlarge": 15, + "p2.8xlarge": 30, + "p2.16xlarge": 30, + "p3.2xlarge": 15, + "p3.8xlarge": 30, + "p3.16xlarge": 30, + "p3dn.24xlarge": 50, + "r3.large": 10, + "r3.xlarge": 15, + "r3.2xlarge": 15, + "r3.4xlarge": 30, + "r3.8xlarge": 30, + "r4.large": 10, + "r4.xlarge": 15, + "r4.2xlarge": 15, + "r4.4xlarge": 30, + "r4.8xlarge": 30, + "r4.16xlarge": 50, + "r5.large": 10, + "r5.xlarge": 15, + "r5.2xlarge": 15, + "r5.4xlarge": 30, + "r5.12xlarge": 30, + "r5.24xlarge": 50, + "r5.metal": 50, + "r5a.large": 10, + "r5a.xlarge": 15, + "r5a.2xlarge": 15, + "r5a.4xlarge": 30, + "r5a.12xlarge": 30, + "r5a.24xlarge": 50, + "r5d.large": 10, + "r5d.xlarge": 15, + "r5d.2xlarge": 15, + "r5d.4xlarge": 30, + "r5d.12xlarge": 30, + "r5d.24xlarge": 50, + "r5d.metal": 50, + "t1.micro": 2, + "t2.nano": 2, + "t2.micro": 2, + "t2.small": 4, + "t2.medium": 6, + "t2.large": 12, + "t2.xlarge": 15, + "t2.2xlarge": 15, + "t3.nano": 2, + "t3.micro": 2, + "t3.small": 4, + "t3.medium": 6, + "t3.large": 12, + "t3.xlarge": 15, + "t3.2xlarge": 15, + "u-6tb1.metal": 30, + "u-9tb1.metal": 30, + "u-12tb1.metal": 30, + "x1.16xlarge": 30, + "x1.32xlarge": 30, + "x1e.xlarge": 10, + "x1e.2xlarge": 15, + "x1e.4xlarge": 15, + "x1e.8xlarge": 15, + "x1e.16xlarge": 30, + "x1e.32xlarge": 30, + "z1d.large": 10, + "z1d.xlarge": 15, + "z1d.2xlarge": 15, + "z1d.3xlarge": 30, + "z1d.6xlarge": 30, + "z1d.12xlarge": 50, + "z1d.metal": 50, +} diff --git a/hooks/nvidia-bootstrap/README.md b/hooks/nvidia-bootstrap/README.md index 04c73afa8aa23..7d3df1a49f156 100644 --- a/hooks/nvidia-bootstrap/README.md +++ b/hooks/nvidia-bootstrap/README.md @@ -1,6 +1,8 @@ ## NVIDIA Driver Installation -Using this hook indicates that you agree to the [license](http://www.nvidia.com/content/DriverDownload-March2009/licence.php?lang=us) +The source code within this directory is provided under the [Apache License, version 2.0](https://www.apache.org/licenses/LICENSE-2.0). + +Note that the NVIDIA software installed by this hook's container may be subject to [NVIDIA's own license terms](http://www.nvidia.com/content/DriverDownload-March2009/licence.php?lang=us). This is an experimental hook for installing the nvidia drivers as part of the kops boot process. diff --git a/hooks/nvidia-device-plugin/Makefile b/hooks/nvidia-device-plugin/Makefile new file mode 100644 index 0000000000000..cb19c580e01a8 --- /dev/null +++ b/hooks/nvidia-device-plugin/Makefile @@ -0,0 +1,31 @@ +# Copyright 2017 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +.DEFAULT_GOAL=help + +all: image + +.PHONY: image push + +image: ## Build the image + docker build -t dcwangmit01/nvidia-device-plugin:0.1.0 -f image/Dockerfile image/ + +push: ## Push the image + docker push dcwangmit01/nvidia-device-plugin:0.1.0 + +help: ## Print list of Makefile targets + @# Taken from https://github.com/spf13/hugo/blob/master/Makefile + @grep -E '^[a-zA-Z0-9_-]+:.*?## .*$$' $(MAKEFILE_LIST) | \ + cut -d ":" -f1- | \ + awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' diff --git a/hooks/nvidia-device-plugin/README.md b/hooks/nvidia-device-plugin/README.md new file mode 100644 index 0000000000000..6cf68895cab16 --- /dev/null +++ b/hooks/nvidia-device-plugin/README.md @@ -0,0 +1,214 @@ +# NVIDIA GPU Driver and DevicePlugin Installation + +## Summary + +This kops hook container may be used to enable nodes with GPUs to work with +Kubernetes. It is targeted specifically for AWS GPU [instance +types](https://aws.amazon.com/ec2/instance-types/). + +It installs the following from web sources. + +1. [Nvidia Device Drivers](http://www.nvidia.com/Download/index.aspx) +2. [Cuda Libraries v9.1](https://developer.nvidia.com/cuda-downloads) +3. [nvidia-docker](https://github.com/NVIDIA/nvidia-docker) +4. [docker-ce](https://www.docker.com/community-edition) + +Using this hook indicates that you agree to the Nvidia +[licenses](http://www.nvidia.com/content/DriverDownload-March2009/licence.php?lang=us). + +## How it works + +* This kops hook container runs on a kubernetes node upon every boot. +* It installs onto the host system a systemd oneshot service unit + `nvidia-device-plugin.service` along with setup scripts. +* The systemd unit `nvidia-device-plugin.service` runs and executes the setup + scripts in the host directory `/nvidia-device-plugin`. +* The scripts install the Nvidia device drivers, Cuda libs, Nvidia docker along + with the matching version of docker-ce. +* The scheduling of work in a separate systemd unit outside of this kops hook + is required because it is not possible to upgrade docker-ce on the host from + within a docker container. + +## Prerequisites + +Although this hook *may* work among many combinatorial versions of software and +images, it has only been tested with the following: + +* kops: **1.9** +* kubernetes: 1.10, **1.11** +* OS Image: **`kope.io/k8s-1.10-debian-stretch-amd64-hvm-ebs-2018-05-27`** + * This is most certainly not the default image for kops. The OS image must + be explicitly overridden in the cluster or instancegroup spec. + * Debian stretch is needed because `nvidia-docker` requires a newer version + of `docker-ce >= 18.0`, which is not available in the Debian jessie package + repository. In addition, the Debian jessie kernel was compiled with gcc-7, + while the system packages install gcc-4, thus making the nvidia driver + compilation fail. +* cloud: **AWS** + * This hook will only work on AWS at this moment. + * This is due to the fact that it uses an AWS discovery mechanism to + determine node instancetype, and subsequently install the correct drivers and + configure the optimal settings for the GPU chipsets. + +### Test Matrix + +This kops hook was developed against the following version combinations. + +| Kops Version | Kubernetes Version | GPU Mode | OS Image | +| ------------- | ------------------ | ------------ | -------- | +| 1.10-beta.1 | 1.10 | deviceplugin | kope.io/k8s-1.10-debian-stretch-amd64-hvm-ebs-2018-05-27 +| 1.9.1 | 1.11 | deviceplugin | kope.io/k8s-1.10-debian-stretch-amd64-hvm-ebs-2018-05-27 +| 1.9.1 | 1.10 | legacy | kope.io/k8s-1.10-debian-stretch-amd64-hvm-ebs-2018-05-27 + +## Using this DevicePlugin + +### Create a Cluster with GPU Nodes + +```bash +kops create cluster gpu.example.com \ + --zones us-east-1c \ + --node-size p2.xlarge \ + --node-count 1 \ + --image kope.io/k8s-1.10-debian-stretch-amd64-hvm-ebs-2018-05-27 \ + --kubernetes-version 1.11.0 +``` + +### Enable the Kops Installation Hook and DevicePlugins + +This should be safe to do for all machines, because the hook auto-detects if +the machine is an AWS GPU instancetype and will NO-OP otherwise. Choose +between the DevicePlugin GPU Mode or Legacy Accelerators GPU Mode. + +#### (Preferred) DevicePlugin GPU Mode + +This mode is: + +* Required for kubernetes >= 1.11.0 +* Optional for 1.8.0 =< kubernetes <= 1.11.0 + +For Kubernetes >= 1.11.0 or clusters supporting DevicePlugins + +```yaml +# > kops edit instancegroup nodes + +spec: + image: kope.io/k8s-1.10-debian-stretch-amd64-hvm-ebs-2018-05-27 + hooks: + - execContainer: + image: dcwangmit01/nvidia-device-plugin:0.1.0 + +### The settings below are only necessary for kubernetes <= 1.11.0, where +### deviceplugins are not enabled by default. +# kubelet: +# featureGates: +# # Enable DevicePlugins +# DevicePlugins: "true" +# # Disable Accelerators (may interfere with DevicePlugins) +# Accelerators: "false" +``` + +#### (Deprecated) Legacy Accelerators GPU Mode + +The legacy `accelerator` +GPU mode is equivalent to the original [GPU hook](/docs/gpu.md). +Accelerators are deprecated in `Kubernetes >= 1.11.0`. + +```yaml +# > kops edit instancegroup nodes + +spec: + image: kope.io/k8s-1.10-debian-stretch-amd64-hvm-ebs-2018-05-27 + hooks: + - execContainer: + image: dcwangmit01/nvidia-device-plugin:0.1.0 + environment: + NVIDIA_DEVICE_PLUGIN_MODE: legacy + kubelet: + featureGates: + # Disable DevicePlugins (may interfere with DevicePlugins) + DevicePlugins: "false" + # Enable Accelerators + Accelerators: "true" +``` + +### Update the cluster + +```bash +kops update cluster gpu.example.com --yes +kops rolling-update cluster gpu.example.com --yes +``` + +### Deploy the Daemonset for the Nvidia DevicePlugin + +Only for DevicePlugin GPU Mode, load the deviceplugin daemonset for your +specific environment. This is not required for the Legacy Accelerators GPU +Mode. + +```bash +# For kubernetes 1.10 +kubectl create -f https://raw.githubusercontent.com/NVIDIA/k8s-device-plugin/v1.10/nvidia-device-plugin.yml + +# For kubernetes 1.11 +kubectl create -f https://raw.githubusercontent.com/NVIDIA/k8s-device-plugin/v1.11/nvidia-device-plugin.yml + +# (Optional) Set permissive toleration to allow daemonset to run anywhere. +# By default this is permissive in case you have tainted your GPU nodes. +kubectl patch daemonset nvidia-device-plugin-daemonset --namespace kube-system \ + -p '{ "spec": { "template": { "spec": { "tolerations": [ { "operator": "Exists" } ] } } } }' +``` + +### Validate that GPUs are Working + +#### Deploy a Test Pod + +```bash +cat << EOF | kubectl create -f - +apiVersion: v1 +kind: Pod +metadata: + name: tf-gpu +spec: + containers: + - name: gpu + image: tensorflow/tensorflow:1.9.0-gpu + imagePullPolicy: IfNotPresent + resources: + limits: + memory: 1024Mi + # ^ Set memory in case default limits are set low + nvidia.com/gpu: 1 # requesting 1 GPUs + # ^ For Legacy Accelerators mode this key must be renamed + # 'alpha.kubernetes.io/nvidia-gpu' + tolerations: + # This toleration will allow the gpu hook to run anywhere + # By default this is permissive in case you have tainted your GPU nodes. + - operator: "Exists" + # ^ If you have a specific taint to target, comment out the above and modify + # the example below + +### Example tolerations +# - key: "dedicated" +# operator: "Equal" +# value: "gpu" +# effect: "NoExecute" +EOF +``` + +#### Validate that GPUs are working + +```bash +# Check that nodes are detected to have GPUs +kubectl describe nodes|grep -E 'gpu:\s.*[1-9]' + +# Check the logs of the Tensorflow Container to ensure that it ran +kubectl logs tf-gpu + +# Show GPU info from within the pod +# Only works in DevicePlugin mode +kubectl exec -it tf-gpu nvidia-smi + +# Show Tensorflow detects GPUs from within the pod. +# Only works in DevicePlugin mode +kubectl exec -it tf-gpu -- \ + python -c 'from tensorflow.python.client import device_lib; print(device_lib.list_local_devices())' +``` diff --git a/hooks/nvidia-device-plugin/image/Dockerfile b/hooks/nvidia-device-plugin/image/Dockerfile new file mode 100644 index 0000000000000..5cfe7857db60f --- /dev/null +++ b/hooks/nvidia-device-plugin/image/Dockerfile @@ -0,0 +1,27 @@ +# Copyright 2017 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM debian:stretch-slim +# ^ Cannot be Alpine since it does not support systemctl +# ^ Systemctl is used to control systemd resources of the host + +RUN apt-get update && \ + apt-get install -y -q --no-install-recommends systemd && \ + apt-get -y clean && \ + apt-get -y autoremove + +COPY run.sh /run.sh +COPY files/* /nvidia-device-plugin/ + +CMD [ "/bin/bash", "/run.sh" ] diff --git a/hooks/nvidia-device-plugin/image/files/01-aws-nvidia-driver.sh b/hooks/nvidia-device-plugin/image/files/01-aws-nvidia-driver.sh new file mode 100755 index 0000000000000..82b3c4b111234 --- /dev/null +++ b/hooks/nvidia-device-plugin/image/files/01-aws-nvidia-driver.sh @@ -0,0 +1,218 @@ +# Copyright 2017 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/bin/bash +set -euo pipefail +set -x + +################################################# +# Settings + +# A place on the host machine to cache 1.6GB+ downloads in-between reboots. + +CACHE_DIR=/nvidia-device-plugin + +# AWS Instance Types to Nvidia Card Mapping (cut and pasted from AWS docs) +# Load the correct driver for the correct instance type +# Instances Product Type Product Series Product +# G2 GRID GRID Series GRID K520 (deprecated) +# G3 Tesla M-Series M-60 +# P2 Tesla K-Series K-80 +# P3 Tesla V-Series V100 +# http://www.nvidia.com/Download/index.aspx +declare -A class_to_driver_file +class_to_driver_file=( \ + ["g2"]="http://us.download.nvidia.com/XFree86/Linux-x86_64/367.124/NVIDIA-Linux-x86_64-367.124.run" \ + ["g3"]="http://us.download.nvidia.com/tesla/390.46/NVIDIA-Linux-x86_64-390.46.run" \ + ["p2"]="http://us.download.nvidia.com/tesla/390.46/NVIDIA-Linux-x86_64-390.46.run" \ + ["p3"]="http://us.download.nvidia.com/tesla/390.46/NVIDIA-Linux-x86_64-390.46.run" \ +) +declare -A class_to_driver_checksum +class_to_driver_checksum=( \ + ["g2"]="77f37939efeea4b6505842bed50445971992e303" \ + ["g3"]="57569ecb6f6d839ecc77fa10a2c573cc069990cc" \ + ["p2"]="57569ecb6f6d839ecc77fa10a2c573cc069990cc" \ + ["p3"]="57569ecb6f6d839ecc77fa10a2c573cc069990cc" \ +) + +# CUDA Files that need to be installed ~1.4GB +# First one is main installation +# Subsequent files are patches which need to be applied in order +# Order in the arrays below matters +# https://developer.nvidia.com/cuda-downloads +cuda_files=( \ + "https://developer.nvidia.com/compute/cuda/9.1/Prod/local_installers/cuda_9.1.85_387.26_linux" \ + "https://developer.nvidia.com/compute/cuda/9.1/Prod/patches/1/cuda_9.1.85.1_linux" \ + "https://developer.nvidia.com/compute/cuda/9.1/Prod/patches/2/cuda_9.1.85.2_linux" \ + "https://developer.nvidia.com/compute/cuda/9.1/Prod/patches/3/cuda_9.1.85.3_linux" \ +) +cuda_files_checksums=( \ + "1540658f4fe657dddd8b0899555b7468727d4aa8" \ + "7ec6970ecd81163b0d02ef30d35599e7fd6e97d8" \ + "cfa3b029b58fc117d8ce510a70efc848924dd565" \ + "6269a2c5784b08997edb97ea0020fb4e6c8769ed" \ +) + +containsElement () { for e in "${@:2}"; do [[ "$e" = "$1" ]] && return 0; done; return 1; } + +################################################# +# Ensure that we are on a proper AWS GPU Instance + +apt-get -y update +apt-get -y --no-upgrade install curl jq + +AWS_INSTANCE_TYPE=$(curl -m 2 -fsSL http://169.254.169.254/latest/dynamic/instance-identity/document | jq -r ".instanceType" || true) # eg: p2.micro +AWS_INSTANCE_CLASS=$(echo $AWS_INSTANCE_TYPE | cut -d . -f 1 || true) # e.g. p2 + +if [[ -z $AWS_INSTANCE_TYPE ]] || [[ -z $AWS_INSTANCE_CLASS ]]; then + echo "This machine is not an AWS instance" + echo " Exiting without installing GPU drivers" + exit 1 +fi + +classnames=${!class_to_driver_file[@]} # e.g. [ "g2", "g3", "p2", "p3" ] +if ! containsElement $AWS_INSTANCE_CLASS $classnames; then + echo "This machine is an AWS instance, but not a GPU instance" + echo " Exiting without installing GPU drivers" + exit 1 +fi + +echo "Identified machine as AWS_INSTANCE_TYPE[$AWS_INSTANCE_TYPE] AWS_INSTANCE_CLASS[$AWS_INSTANCE_CLASS]" + +################################################# +# Install dependencies + +# Install GCC and linux headers on the host machine +# The NVIDIA driver build must be compiled with the same version of GCC as +# the kernel. In addition, linux-headers are machine image specific. +# Install with --no-upgrade so that the c-libs are not upgraded, possibly +# breaking programs and requiring restart +apt-get -y update +apt-get -y --no-upgrade install gcc libc-dev linux-headers-$(uname -r) +apt-get -y clean +apt-get -y autoremove + +################################################# +# Unload open-source nouveau driver if it exists +# The nvidia drivers won't install otherwise +# "g3" instances in particular have this module auto-loaded +modprobe -r nouveau || true + +################################################# +# Download and install the Nvidia drivers and cuda libraries + +# Create list of URLs and Checksums by merging driver item with array of cuda files +downloads=(${class_to_driver_file[$AWS_INSTANCE_CLASS]} ${cuda_files[@]}) +checksums=(${class_to_driver_checksum[$AWS_INSTANCE_CLASS]} ${cuda_files_checksums[@]}) + +# Download, verify, and execute each file +length=${#downloads[@]} +for (( i=0; i<${length}; i++ )); do + download=${downloads[$i]} + checksum=${checksums[$i]} + filename=$(basename $download) + filepath="${CACHE_DIR}/${filename}" + filepath_installed="${CACHE_DIR}/${filename}.installed" + + echo "Checking for file at $filepath" + if [[ ! -f $filepath ]] || ! (echo "$checksum $filepath" | sha1sum -c - 2>&1 >/dev/null); then + echo "Downloading $download" + curl -L $download > $filepath + chmod a+x $filepath + fi + + echo "Verifying sha1sum of file at $filepath" + if ! (echo "$checksum $filepath" | sha1sum -c -); then + echo "Failed to verify sha1sum for file at $filepath" + exit 1 + fi + + # Install the Nvidia driver and cuda libs + if [[ -f $filepath_installed ]]; then + echo "Detected prior install of file $filename on host" + else + echo "Installing file $filename on host" + if [[ $download =~ .*NVIDIA.* ]]; then + # Install the nvidia package + $filepath --accept-license --silent + touch $filepath_installed # Mark successful installation + elif [[ $download =~ .*local_installers.*cuda.* ]]; then + # Install the primary cuda library + $filepath --toolkit --silent --verbose + touch $filepath_installed # Mark successful installation + elif [[ $download =~ .*patches.*cuda.* ]]; then + # Install an update to the primary cuda library + $filepath --accept-eula --silent + touch $filepath_installed # Mark successful installation + else + echo "Unable to handle file $filepath" + exit 1 + fi + fi +done + +################################################# +# Output GPU info for debugging +nvidia-smi --list-gpus + +################################################# +# Configure and Optimize Nvidia cards now that things are installed +# AWS Optimizization Doc +# https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/optimize_gpu.html +# Nvidia Doc +# http://developer.download.nvidia.com/compute/DCGM/docs/nvidia-smi-367.38.pdf + +# Common configurations +nvidia-smi -pm 1 +nvidia-smi --auto-boost-default=0 +nvidia-smi --auto-boost-permission=0 + +# Custom configurations per class of nvidia video card +case "$AWS_INSTANCE_CLASS" in +"g2" | "g3") + nvidia-smi -ac 2505,1177 + ;; +"p2") + nvidia-smi -ac 2505,875 + nvidia-smi -acp 0 + ;; +"p3") + nvidia-smi -ac 877,1530 + nvidia-smi -acp 0 + ;; +*) + ;; +esac + +################################################# +# Load the Kernel Module + +if ! /sbin/modprobe nvidia-uvm; then + echo "Unable to modprobe nvidia-uvm" + exit 1 +fi + +# Ensure that the device node exists +if ! test -e /dev/nvidia-uvm; then + # Find out the major device number used by the nvidia-uvm driver + D=`grep nvidia-uvm /proc/devices | awk '{print $1}'` + mknod -m 666 /dev/nvidia-uvm c $D 0 +fi + +########################################################### +# Restart Kubelet +# Only necessary in the case of Accelerators (not Device Plugins) + +echo "Restarting Kubelet" +systemctl restart kubelet.service diff --git a/hooks/nvidia-device-plugin/image/files/02-nvidia-docker.sh b/hooks/nvidia-device-plugin/image/files/02-nvidia-docker.sh new file mode 100755 index 0000000000000..e5d13a5fe833b --- /dev/null +++ b/hooks/nvidia-device-plugin/image/files/02-nvidia-docker.sh @@ -0,0 +1,80 @@ +# Copyright 2017 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/bin/bash +set -euo pipefail +set -x + +################################################# +# Install nvidia-docker2 + +# This section is somewhat adapted from README at: +# https://github.com/NVIDIA/nvidia-docker + +####################################### +# Cleanup old nvidia-docker + +# If you have nvidia-docker 1.0 installed: we need to remove it and all existing GPU containers +docker volume ls -q -f driver=nvidia-docker | xargs -r -I{} -n1 docker ps -q -a -f volume={} | xargs -r docker rm -f + +# Remove the old nvidia-docker if it exists +apt-get purge -y nvidia-docker || true + +####################################### +# Add package repositories + +# Add the package repository for docker-ce +curl -fsSL https://download.docker.com/linux/debian/gpg | \ + apt-key add - +echo 'deb [arch=amd64] https://download.docker.com/linux/debian stretch stable' | \ + tee /etc/apt/sources.list.d/docker-ce.list + +# Add the package repository for nvidia-docker +curl -s -L https://nvidia.github.io/nvidia-docker/gpgkey | \ + apt-key add - +distribution=$(. /etc/os-release;echo $ID$VERSION_ID) +curl -s -L https://nvidia.github.io/nvidia-docker/$distribution/nvidia-docker.list | \ + tee /etc/apt/sources.list.d/nvidia-docker.list + +# Override the default runtime with the one from nvidia +# Also explicity set the storage-driver to the prior 'overlay' +cat << 'EOF' > /etc/docker/daemon.json +{ + "default-runtime": "nvidia", + "runtimes": { + "nvidia": { + "path": "/usr/bin/nvidia-container-runtime", + "runtimeArgs": [] + } + }, + "storage-driver": "overlay" +} +EOF + +# Install nvidia-docker2 and reload the Docker daemon configuration +# Note that the nvidia-docker version must match the docker-ce version +# --force-confold prevents prompt for replacement of daemon.json +apt-get -y update +apt-get install -y --allow-downgrades -o Dpkg::Options::="--force-confold" \ + nvidia-docker2 \ + nvidia-container-runtime \ + docker-ce + +# Disable a few things that break docker-ce/gpu support upon reboot: +# Upon boot, the kops-configuration.service systemd unit sets up and starts +# the cloud-init.service which runs nodeup which forces docker-ce to a +# specific version that is a downgrade and incompatible with nvidia-docker2. +# Permanently disable these systemd units via masking. +systemctl mask cloud-init.service +systemctl mask kops-configuration.service diff --git a/hooks/nvidia-device-plugin/image/files/nvidia-device-plugin.service b/hooks/nvidia-device-plugin/image/files/nvidia-device-plugin.service new file mode 100644 index 0000000000000..660d7e48c5ed4 --- /dev/null +++ b/hooks/nvidia-device-plugin/image/files/nvidia-device-plugin.service @@ -0,0 +1,10 @@ +[Unit] +Description=Prepare AWS GPU instances for Nvidia Kubernetes Device Plugin +After=cloud-config.target cloud-init.target kops-configuration.service apt-daily-upgrade.timer install-xfs.service + +[Service] +Type=oneshot +ExecStart=/bin/bash -c "/nvidia-device-plugin/nvidia-device-plugin.sh" + +[Install] +WantedBy=multi-user.target diff --git a/hooks/nvidia-device-plugin/image/files/nvidia-device-plugin.sh b/hooks/nvidia-device-plugin/image/files/nvidia-device-plugin.sh new file mode 100755 index 0000000000000..7590502dac058 --- /dev/null +++ b/hooks/nvidia-device-plugin/image/files/nvidia-device-plugin.sh @@ -0,0 +1,52 @@ +# Copyright 2017 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/bin/bash +set -euo pipefail +set -x + +CACHE_DIR=/nvidia-device-plugin + +# Load Passthrough enviroment variables from the original kops hook +source $CACHE_DIR/environment + +# Support both deviceplugin and legacy (accelerator) GPU modes. +# Default to 'deviceplugin' if env var is unset. +if [[ ! -v NVIDIA_DEVICE_PLUGIN_MODE ]]; then + NVIDIA_DEVICE_PLUGIN_MODE='deviceplugin' + echo "Defaulting to NVIDIA_DEVICE_PLUGIN_MODE='deviceplugin'" +fi + +# Figure out which scripts should run +scripts=() +case "$NVIDIA_DEVICE_PLUGIN_MODE" in + legacy) + scripts+=("$CACHE_DIR/01-aws-nvidia-driver.sh") + ;; + deviceplugin) + scripts+=("$CACHE_DIR/01-aws-nvidia-driver.sh") + scripts+=("$CACHE_DIR/02-nvidia-docker.sh") + ;; + *) + echo "Invalid NVIDIA_DEVICE_PLUGIN_MODE=$NVIDIA_DEVICE_PLUGIN_MODE" + echo " Valid values are 'deviceplugin' or 'legacy'" + exit 1 +esac + +# Run the scripts +for script in "${scripts[@]}"; do + echo "########## Starting $script ##########" + $script 2>&1 | tee -a $CACHE_DIR/install.log + echo "########## Finished $script ##########" +done diff --git a/hooks/nvidia-device-plugin/image/run.sh b/hooks/nvidia-device-plugin/image/run.sh new file mode 100755 index 0000000000000..ea997becf8ce4 --- /dev/null +++ b/hooks/nvidia-device-plugin/image/run.sh @@ -0,0 +1,37 @@ +# Copyright 2017 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/bin/bash +set -euo pipefail +set -x + +# Copy the setup scripts to the host +# The kops hook automatically mounts the host root filesystem into the +# container /rootfs +mkdir -p /rootfs/nvidia-device-plugin +cp -r /nvidia-device-plugin/* /rootfs/nvidia-device-plugin + +# Setup the host systemd to run the systemd unit that runs setup scripts +ln -sf /nvidia-device-plugin/nvidia-device-plugin.service /rootfs/etc/systemd/system/nvidia-device-plugin.service + +# Save the environment to be passed on to the systemd unit +(env | grep NVIDIA_DEVICE_PLUGIN > /rootfs/nvidia-device-plugin/environment) || true + +# Kickoff host systemd unit that runs the setup scripts +# 'systemctl' within this docker container uses the mounted /run/systemd/* +# volume from the host to control systemd on the host. +systemctl daemon-reload +systemctl start --no-block nvidia-device-plugin.service + +exit 0 diff --git a/images/utils-builder/Dockerfile b/images/utils-builder/Dockerfile index f04918c62f090..e278d928985f1 100644 --- a/images/utils-builder/Dockerfile +++ b/images/utils-builder/Dockerfile @@ -18,6 +18,8 @@ FROM k8s.gcr.io/debian-base-amd64:0.2 RUN echo "deb-src http://security.debian.org/ jessie/updates main" >> /etc/apt/sources.list RUN echo "deb-src http://ftp.us.debian.org/debian/ jessie main" >> /etc/apt/sources.list +RUN printf "deb http://archive.debian.org/debian/ jessie main\ndeb-src http://archive.debian.org/debian/ jessie main\ndeb http://security.debian.org jessie/updates main\ndeb-src http://security.debian.org jessie/updates main" > /etc/apt/sources.list + RUN apt-get update && apt-get install --yes dpkg-dev bash \ && apt-get build-dep --yes socat conntrack \ && apt-get clean diff --git a/kube-discovery/cmd/kube-discovery/main.go b/kube-discovery/cmd/kube-discovery/main.go index ff01bc4ab5ecf..cae7009180ac6 100644 --- a/kube-discovery/cmd/kube-discovery/main.go +++ b/kube-discovery/cmd/kube-discovery/main.go @@ -146,9 +146,8 @@ func (c *DiscoveryController) runOnce() error { // TODO: Verify resolved records against certificates? if err := hosts.UpdateHostsFileWithRecords(hostsPath, addrToHosts); err != nil { return fmt.Errorf("error updating hosts file: %v", err) - } else { - glog.Infof("updated %s", hostsPath) } + glog.Infof("updated %s", hostsPath) return nil } diff --git a/node-authorizer/pkg/server/middleware.go b/node-authorizer/pkg/server/middleware.go index 91401da7b3eb6..22b8e790fc531 100644 --- a/node-authorizer/pkg/server/middleware.go +++ b/node-authorizer/pkg/server/middleware.go @@ -17,6 +17,7 @@ limitations under the License. package server import ( + "fmt" "net/http" "k8s.io/kops/node-authorizer/pkg/utils" @@ -24,6 +25,22 @@ import ( "go.uber.org/zap" ) +// recovery is responsible for ensuring we don't exit on a panic +func recovery(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + defer func() { + if err := recover(); err != nil { + w.WriteHeader(http.StatusInternalServerError) + + utils.Logger.Error("failed to handle request, threw exception", + zap.String("error", fmt.Sprintf("%v", err))) + } + }() + + next.ServeHTTP(w, req) + }) +} + // authorized is responsible for validating the client certificate func authorized(next http.HandlerFunc, commonName string, requireAuth bool) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { diff --git a/node-authorizer/pkg/server/server.go b/node-authorizer/pkg/server/server.go index 78f9d2acb37ad..d3d22294460b6 100644 --- a/node-authorizer/pkg/server/server.go +++ b/node-authorizer/pkg/server/server.go @@ -58,14 +58,14 @@ func New(config *Config, authorizer Authorizer) (*NodeAuthorizer, error) { zap.String("listen", config.Listen), zap.String("version", Version)) - if err := config.IsValid(); err != nil { - return nil, fmt.Errorf("configuration error: %s", err) - } - if authorizer == nil { return nil, errors.New("no authorizer") } + if err := config.IsValid(); err != nil { + return nil, fmt.Errorf("configuration error: %s", err) + } + return &NodeAuthorizer{ authorizer: authorizer, config: config, @@ -109,7 +109,7 @@ func (n *NodeAuthorizer) Run() error { r.Handle("/authorize/{name}", authorized(n.authorizeHandler, n.config.ClientCommonName, n.useMutualTLS())).Methods(http.MethodPost) r.Handle("/metrics", prometheus.Handler()).Methods(http.MethodGet) r.HandleFunc("/health", n.healthHandler).Methods(http.MethodGet) - server.Handler = r + server.Handler = recovery(r) // @step: wait for either an error or a termination signal errs := make(chan error, 2) diff --git a/node-authorizer/pkg/server/types.go b/node-authorizer/pkg/server/types.go index f31b70929a34b..fd012801dcac1 100644 --- a/node-authorizer/pkg/server/types.go +++ b/node-authorizer/pkg/server/types.go @@ -68,7 +68,7 @@ type Config struct { Features []string // EnableVerbose indicate verbose logging EnableVerbose bool - // ClientCommonName is the common name on the client certiicate if mutual tls is enabled + // ClientCommonName is the common name on the client certificate if mutual tls is enabled ClientCommonName string // ClusterName is the name of the kubernetes cluster ClusterName string @@ -168,7 +168,7 @@ type Authorizer interface { // Verifier is the client side of authorizer type Verifier interface { - // VerifyIdentity is responisible for constructing the parameters for a request + // VerifyIdentity is responsible for constructing the parameters for a request VerifyIdentity(context.Context) ([]byte, error) } diff --git a/node-authorizer/pkg/utils/logger.go b/node-authorizer/pkg/utils/logger.go index 97f025d3b7e65..2c18fc0b0f13f 100644 --- a/node-authorizer/pkg/utils/logger.go +++ b/node-authorizer/pkg/utils/logger.go @@ -21,7 +21,7 @@ import ( ) var ( - // Logger is the defaut logger + // Logger is the default logger Logger *zap.Logger ) diff --git a/nodeup/pkg/model/BUILD.bazel b/nodeup/pkg/model/BUILD.bazel index fe83f434227d8..e35410b3487a0 100644 --- a/nodeup/pkg/model/BUILD.bazel +++ b/nodeup/pkg/model/BUILD.bazel @@ -10,6 +10,7 @@ go_library( "directories.go", "docker.go", "etcd.go", + "etcd_manager_tls.go", "etcd_tls.go", "file_assets.go", "firewall.go", @@ -51,6 +52,7 @@ go_library( "//pkg/tokens:go_default_library", "//pkg/try:go_default_library", "//upup/pkg/fi:go_default_library", + "//upup/pkg/fi/cloudup/awsup:go_default_library", "//upup/pkg/fi/nodeup/nodetasks:go_default_library", "//util/pkg/exec:go_default_library", "//util/pkg/reflectutils:go_default_library", diff --git a/nodeup/pkg/model/context.go b/nodeup/pkg/model/context.go index 2d80c0cabf586..36bbab548eb20 100644 --- a/nodeup/pkg/model/context.go +++ b/nodeup/pkg/model/context.go @@ -288,7 +288,7 @@ func (c *NodeupModelContext) UseNodeAuthorizer() bool { // UsesSecondaryIP checks if the CNI in use attaches secondary interfaces to the host. func (c *NodeupModelContext) UsesSecondaryIP() bool { - if (c.Cluster.Spec.Networking.CNI != nil && c.Cluster.Spec.Networking.CNI.UsesSecondaryIP) || c.Cluster.Spec.Networking.AmazonVPC != nil { + if (c.Cluster.Spec.Networking.CNI != nil && c.Cluster.Spec.Networking.CNI.UsesSecondaryIP) || c.Cluster.Spec.Networking.AmazonVPC != nil || c.Cluster.Spec.Networking.LyftVPC != nil { return true } @@ -371,8 +371,13 @@ func (c *NodeupModelContext) BuildCertificateTask(ctx *fi.ModelBuilderContext, n return err } + p := filename + if !filepath.IsAbs(p) { + p = filepath.Join(c.PathSrvKubernetes(), filename) + } + ctx.AddTask(&nodetasks.File{ - Path: filepath.Join(c.PathSrvKubernetes(), filename), + Path: p, Contents: fi.NewStringResource(serialized), Type: nodetasks.FileType_File, Mode: s("0600"), @@ -397,8 +402,13 @@ func (c *NodeupModelContext) BuildPrivateKeyTask(ctx *fi.ModelBuilderContext, na return err } + p := filename + if !filepath.IsAbs(p) { + p = filepath.Join(c.PathSrvKubernetes(), filename) + } + ctx.AddTask(&nodetasks.File{ - Path: filepath.Join(c.PathSrvKubernetes(), filename), + Path: p, Contents: fi.NewStringResource(serialized), Type: nodetasks.FileType_File, Mode: s("0600"), diff --git a/nodeup/pkg/model/docker.go b/nodeup/pkg/model/docker.go index b8ed571a72000..d5357e519c823 100644 --- a/nodeup/pkg/model/docker.go +++ b/nodeup/pkg/model/docker.go @@ -43,10 +43,16 @@ type DockerBuilder struct { var _ fi.ModelBuilder = &DockerBuilder{} type dockerVersion struct { - Name string + Name string + + // Version is the version of docker, as specified in the kops Version string - Source string - Hash string + + // Source is the url where the package/tarfile can be found + Source string + + // Hash is the sha1 hash of the file + Hash string DockerVersion string Distros []distros.Distribution @@ -55,6 +61,9 @@ type dockerVersion struct { // PlainBinary indicates that the Source is not an OS, but a "bare" tar.gz PlainBinary bool + + // MarkImmutable is a list of files on which we should perform a `chattr +i ` + MarkImmutable []string } // DefaultDockerVersion is the (legacy) docker version we use if one is not specified in the manifest. @@ -379,6 +388,7 @@ var dockerVersions = []dockerVersion{ Source: "http://download.docker.com/linux/debian/dists/stretch/pool/stable/amd64/docker-ce_17.03.2~ce-0~debian-stretch_amd64.deb", Hash: "36773361cf44817371770cb4e6e6823590d10297", Dependencies: []string{"bridge-utils", "libapparmor1", "libltdl7", "perl"}, + MarkImmutable: []string{"/usr/bin/docker-runc"}, }, // 17.03.2 - Jessie @@ -391,6 +401,7 @@ var dockerVersions = []dockerVersion{ Source: "http://download.docker.com/linux/debian/dists/jessie/pool/stable/amd64/docker-ce_17.03.2~ce-0~debian-jessie_amd64.deb", Hash: "a7ac54aaa7d33122ca5f7a2df817cbefb5cdbfc7", Dependencies: []string{"bridge-utils", "libapparmor1", "libltdl7", "perl"}, + MarkImmutable: []string{"/usr/bin/docker-runc"}, }, // 17.03.2 - Jessie on ARM @@ -403,6 +414,7 @@ var dockerVersions = []dockerVersion{ Source: "http://download.docker.com/linux/debian/dists/jessie/pool/stable/armhf/docker-ce_17.03.2~ce-0~debian-jessie_armhf.deb", Hash: "71e425b83ce0ef49d6298d61e61c4efbc76b9c65", Dependencies: []string{"bridge-utils", "libapparmor1", "libltdl7", "perl"}, + MarkImmutable: []string{"/usr/bin/docker-runc"}, }, // 17.03.2 - Xenial @@ -415,6 +427,7 @@ var dockerVersions = []dockerVersion{ Source: "http://download.docker.com/linux/ubuntu/dists/xenial/pool/stable/amd64/docker-ce_17.03.2~ce-0~ubuntu-xenial_amd64.deb", Hash: "4dcee1a05ec592e8a76e53e5b464ea43085a2849", Dependencies: []string{"bridge-utils", "iptables", "libapparmor1", "libltdl7", "perl"}, + MarkImmutable: []string{"/usr/bin/docker-runc"}, }, // 17.03.2 - Ubuntu Bionic via binary download (no packages available) @@ -426,6 +439,7 @@ var dockerVersions = []dockerVersion{ Source: "http://download.docker.com/linux/static/stable/x86_64/docker-17.03.2-ce.tgz", Hash: "141716ae046016a1792ce232a0f4c8eed7fe37d1", Dependencies: []string{"bridge-utils", "iptables", "libapparmor1", "libltdl7", "perl"}, + MarkImmutable: []string{"/usr/bin/docker-runc"}, }, // 17.03.2 - Centos / Rhel7 (two packages) @@ -438,6 +452,7 @@ var dockerVersions = []dockerVersion{ Source: "https://download.docker.com/linux/centos/7/x86_64/stable/Packages/docker-ce-17.03.2.ce-1.el7.centos.x86_64.rpm", Hash: "494ca888f5b1553f93b9d9a5dad4a67f76cf9eb5", Dependencies: []string{"libtool-ltdl", "libseccomp", "libcgroup"}, + MarkImmutable: []string{"/usr/bin/docker-runc"}, }, { DockerVersion: "17.03.2", @@ -501,7 +516,31 @@ var dockerVersions = []dockerVersion{ //Recommends: aufs-tools, ca-certificates, cgroupfs-mount | cgroup-lite, git, xz-utils, apparmor }, - // 17.09.0 - Centos / Rhel7 + // 18.06.2 - Xenial + { + DockerVersion: "18.06.2", + Name: "docker-ce", + Distros: []distros.Distribution{distros.DistributionXenial}, + Architectures: []Architecture{ArchitectureAmd64}, + Version: "18.06.2~ce~3-0~ubuntu", + Source: "https://download.docker.com/linux/ubuntu/dists/xenial/pool/stable/amd64/docker-ce_18.06.2~ce~3-0~ubuntu_amd64.deb", + Hash: "03e5eaae9c84b144e1140d9b418e43fce0311892", + Dependencies: []string{"bridge-utils", "iptables", "libapparmor1", "libltdl7", "perl"}, + //Depends: iptables, init-system-helpers, lsb-base, libapparmor1, libc6, libdevmapper1.02.1, libltdl7, libeseccomp2, libsystemd0 + //Recommends: aufs-tools, ca-certificates, cgroupfs-mount | cgroup-lite, git, xz-utils, apparmor + }, + + // 17.09.0 - Centos / Rhel7 (two packages) + { + DockerVersion: "17.09.0", + Name: "container-selinux", + Distros: []distros.Distribution{distros.DistributionRhel7, distros.DistributionCentos7}, + Architectures: []Architecture{ArchitectureAmd64}, + Version: "2.68", + Source: "http://mirror.centos.org/centos/7/extras/x86_64/Packages/container-selinux-2.68-1.el7.noarch.rpm", + Hash: "d9f87f7f4f2e8e611f556d873a17b8c0c580fec0", + Dependencies: []string{"policycoreutils-python"}, + }, { DockerVersion: "17.09.0", Name: "docker-ce", @@ -527,6 +566,20 @@ var dockerVersions = []dockerVersion{ //Recommends: aufs-tools, ca-certificates, cgroupfs-mount | cgroup-lite, git, xz-utils, apparmor }, + // 18.06.2 - Bionic + { + DockerVersion: "18.06.2", + Name: "docker-ce", + Distros: []distros.Distribution{distros.DistributionBionic}, + Architectures: []Architecture{ArchitectureAmd64}, + Version: "18.06.2~ce~3-0~ubuntu", + Source: "https://download.docker.com/linux/ubuntu/dists/bionic/pool/stable/amd64/docker-ce_18.06.2~ce~3-0~ubuntu_amd64.deb", + Hash: "9607c67644e3e1ad9661267c99499004f2e84e05", + Dependencies: []string{"bridge-utils", "iptables", "libapparmor1", "libltdl7", "perl"}, + //Depends: iptables, init-system-helpers, lsb-base, libapparmor1, libc6, libdevmapper1.02.1, libltdl7, libeseccomp2, libsystemd0 + //Recommends: aufs-tools, ca-certificates, cgroupfs-mount | cgroup-lite, git, xz-utils, apparmor + }, + // 18.06.1 - Debian Stretch { @@ -539,6 +592,144 @@ var dockerVersions = []dockerVersion{ Hash: "18473b80e61b6d4eb8b52d87313abd71261287e5", Dependencies: []string{"bridge-utils", "libapparmor1", "libltdl7", "perl"}, }, + + // 18.06.2 - Debian Stretch + { + + DockerVersion: "18.06.2", + Name: "docker-ce", + Distros: []distros.Distribution{distros.DistributionDebian9}, + Architectures: []Architecture{ArchitectureAmd64}, + Version: "18.06.2~ce~3-0~debian", + Source: "https://download.docker.com/linux/debian/dists/stretch/pool/stable/amd64/docker-ce_18.06.2~ce~3-0~debian_amd64.deb", + Hash: "aad1efd2c90725034e996c6a368ccc2bf41ca5b8", + Dependencies: []string{"bridge-utils", "libapparmor1", "libltdl7", "perl"}, + }, + + // 18.06.2 - Jessie + { + DockerVersion: "18.06.2", + Name: "docker-ce", + Distros: []distros.Distribution{distros.DistributionJessie}, + Architectures: []Architecture{ArchitectureAmd64}, + Version: "ce_18.06.2~ce~3-0~debian", + Source: "https://download.docker.com/linux/debian/dists/jessie/pool/stable/amd64/docker-ce_18.06.2~ce~3-0~debian_amd64.deb", + Hash: "1a2500311230aff37aa81dd1292a88302fb0a2e1", + Dependencies: []string{"bridge-utils", "libapparmor1", "libltdl7", "perl"}, + }, + + // 18.06.1 - CentOS / Rhel7 (two packages) + { + DockerVersion: "18.06.1", + Name: "container-selinux", + Distros: []distros.Distribution{distros.DistributionRhel7, distros.DistributionCentos7}, + Architectures: []Architecture{ArchitectureAmd64}, + Version: "2.68", + Source: "http://mirror.centos.org/centos/7/extras/x86_64/Packages/container-selinux-2.68-1.el7.noarch.rpm", + Hash: "d9f87f7f4f2e8e611f556d873a17b8c0c580fec0", + Dependencies: []string{"policycoreutils-python"}, + }, + { + DockerVersion: "18.06.1", + Name: "docker-ce", + Distros: []distros.Distribution{distros.DistributionRhel7, distros.DistributionCentos7}, + Architectures: []Architecture{ArchitectureAmd64}, + Version: "18.06.1.ce", + Source: "https://download.docker.com/linux/centos/7/x86_64/stable/Packages/docker-ce-18.06.1.ce-3.el7.x86_64.rpm", + Hash: "0a1325e570c5e54111a79623c9fd0c0c714d3a11", + Dependencies: []string{"libtool-ltdl", "libseccomp", "libcgroup"}, + }, + + // 18.06.2 - CentOS / Rhel7 (two packages) + { + DockerVersion: "18.06.2", + Name: "container-selinux", + Distros: []distros.Distribution{distros.DistributionRhel7, distros.DistributionCentos7}, + Architectures: []Architecture{ArchitectureAmd64}, + Version: "2.68", + Source: "http://mirror.centos.org/centos/7/extras/x86_64/Packages/container-selinux-2.68-1.el7.noarch.rpm", + Hash: "d9f87f7f4f2e8e611f556d873a17b8c0c580fec0", + Dependencies: []string{"policycoreutils-python"}, + }, + { + DockerVersion: "18.06.2", + Name: "docker-ce", + Distros: []distros.Distribution{distros.DistributionRhel7, distros.DistributionCentos7}, + Architectures: []Architecture{ArchitectureAmd64}, + Version: "18.06.2.ce", + Source: "https://download.docker.com/linux/centos/7/x86_64/stable/Packages/docker-ce-18.06.2.ce-3.el7.x86_64.rpm", + Hash: "456eb7c5bfb37fac342e9ade21b602c076c5b367", + Dependencies: []string{"libtool-ltdl", "libseccomp", "libcgroup"}, + }, + + // 18.06.3 (contains fix for CVE-2019-5736) + + // 18.06.3 - Bionic + { + DockerVersion: "18.06.3", + Name: "docker-ce", + Distros: []distros.Distribution{distros.DistributionBionic}, + Architectures: []Architecture{ArchitectureAmd64}, + Version: "18.06.3~ce~3-0~ubuntu", + Source: "https://download.docker.com/linux/ubuntu/dists/bionic/pool/stable/amd64/docker-ce_18.06.3~ce~3-0~ubuntu_amd64.deb", + Hash: "b396678a8b70f0503a7b944fa6e3297ab27b345b", + Dependencies: []string{"bridge-utils", "iptables", "libapparmor1", "libltdl7", "perl"}, + //Depends: iptables, init-system-helpers, lsb-base, libapparmor1, libc6, libdevmapper1.02.1, libltdl7, libeseccomp2, libsystemd0 + //Recommends: aufs-tools, ca-certificates, cgroupfs-mount | cgroup-lite, git, xz-utils, apparmor + }, + + // 18.06.3 - Debian Stretch + { + + DockerVersion: "18.06.3", + Name: "docker-ce", + Distros: []distros.Distribution{distros.DistributionDebian9}, + Architectures: []Architecture{ArchitectureAmd64}, + Version: "18.06.3~ce~3-0~debian", + Source: "https://download.docker.com/linux/debian/dists/stretch/pool/stable/amd64/docker-ce_18.06.3~ce~3-0~debian_amd64.deb", + Hash: "93b5a055a39462867d79109b00db1367e3d9e32f", + Dependencies: []string{"bridge-utils", "libapparmor1", "libltdl7", "perl"}, + }, + + // 18.06.3 - Jessie + { + DockerVersion: "18.06.3", + Name: "docker-ce", + Distros: []distros.Distribution{distros.DistributionJessie}, + Architectures: []Architecture{ArchitectureAmd64}, + Version: "ce_18.06.3~ce~3-0~debian", + Source: "https://download.docker.com/linux/debian/dists/jessie/pool/stable/amd64/docker-ce_18.06.3~ce~3-0~debian_amd64.deb", + Hash: "058bcd4b055560866b8cad978c7aa224694602da", + Dependencies: []string{"bridge-utils", "libapparmor1", "libltdl7", "perl"}, + }, + + // 18.06.3 - CentOS / Rhel7 (two packages) + { + DockerVersion: "18.06.3", + Name: "container-selinux", + Distros: []distros.Distribution{distros.DistributionRhel7, distros.DistributionCentos7}, + Architectures: []Architecture{ArchitectureAmd64}, + Version: "2.68", + Source: "http://mirror.centos.org/centos/7/extras/x86_64/Packages/container-selinux-2.68-1.el7.noarch.rpm", + Hash: "d9f87f7f4f2e8e611f556d873a17b8c0c580fec0", + Dependencies: []string{"policycoreutils-python"}, + }, + { + DockerVersion: "18.06.3", + Name: "docker-ce", + Distros: []distros.Distribution{distros.DistributionRhel7, distros.DistributionCentos7}, + Architectures: []Architecture{ArchitectureAmd64}, + Version: "18.06.3.ce", + Source: "https://download.docker.com/linux/centos/7/x86_64/stable/Packages/docker-ce-18.06.3.ce-3.el7.x86_64.rpm", + Hash: "5369602f88406d4fb9159dc1d3fd44e76fb4cab8", + Dependencies: []string{"libtool-ltdl", "libseccomp", "libcgroup"}, + }, + + // TIP: When adding the next version, copy the previous + // version, string replace the version, run `VERIFY_HASHES=1 + // go test ./nodeup/pkg/model` (you might want to temporarily + // comment out older versions on a slower connection), and + // then validate the dependencies etc } func (d *dockerVersion) matches(arch Architecture, dockerVersion string, distro distros.Distribution) bool { @@ -568,6 +759,18 @@ func (d *dockerVersion) matches(arch Architecture, dockerVersion string, distro return true } +func (b *DockerBuilder) dockerVersion() string { + dockerVersion := "" + if b.Cluster.Spec.Docker != nil { + dockerVersion = fi.StringValue(b.Cluster.Spec.Docker.Version) + } + if dockerVersion == "" { + dockerVersion = DefaultDockerVersion + glog.Warningf("DockerVersion not specified; using default %q", dockerVersion) + } + return dockerVersion +} + // Build is responsible for configuring the docker daemon func (b *DockerBuilder) Build(c *fi.ModelBuilderContext) error { @@ -598,14 +801,7 @@ func (b *DockerBuilder) Build(c *fi.ModelBuilderContext) error { c.AddTask(t) } - dockerVersion := "" - if b.Cluster.Spec.Docker != nil { - dockerVersion = fi.StringValue(b.Cluster.Spec.Docker.Version) - } - if dockerVersion == "" { - dockerVersion = DefaultDockerVersion - glog.Warningf("DockerVersion not specified; using default %q", dockerVersion) - } + dockerVersion := b.dockerVersion() // Add packages { @@ -618,19 +814,21 @@ func (b *DockerBuilder) Build(c *fi.ModelBuilderContext) error { count++ + var packageTask fi.Task if dv.PlainBinary { - c.AddTask(&nodetasks.Archive{ + packageTask = &nodetasks.Archive{ Name: "docker", Source: dv.Source, Hash: dv.Hash, TargetDir: "/usr/bin/", StripComponents: 1, - }) + } + c.AddTask(packageTask) c.AddTask(b.buildDockerGroup()) c.AddTask(b.buildSystemdSocket()) } else { - c.AddTask(&nodetasks.Package{ + packageTask = &nodetasks.Package{ Name: dv.Name, Version: s(dv.Version), Source: s(dv.Source), @@ -638,6 +836,16 @@ func (b *DockerBuilder) Build(c *fi.ModelBuilderContext) error { // TODO: PreventStart is now unused? PreventStart: fi.Bool(true), + } + c.AddTask(packageTask) + } + + // As a mitigation for CVE-2019-5736 (possibly a fix, definitely defense-in-depth) we chattr docker-runc to be immutable + for _, f := range dv.MarkImmutable { + c.AddTask(&nodetasks.Chattr{ + File: f, + Mode: "+i", + Deps: []fi.Task{packageTask}, }) } @@ -883,6 +1091,21 @@ func (b *DockerBuilder) buildSysconfig(c *fi.ModelBuilderContext) error { } } + // RHEL-family / docker has a bug with 17.x where it fails to use overlay2 because it does a broken kernel check + if b.Distribution.IsRHELFamily() { + dockerVersion := b.dockerVersion() + if strings.HasPrefix(dockerVersion, "17.") { + storageOpts := strings.Join(docker.StorageOpts, ",") + if strings.Contains(storageOpts, "overlay2.override_kernel_check=1") { + // Already there + } else if !strings.Contains(storageOpts, "overlay2.override_kernel_check") { + docker.StorageOpts = append(docker.StorageOpts, "overlay2.override_kernel_check=1") + } else { + glog.Infof("detected image was RHEL and overlay2.override_kernel_check=1 was probably needed, but overlay2.override_kernel_check was already set (%q) so won't set", storageOpts) + } + } + } + flagsString, err := flagbuilder.BuildFlags(&docker) if err != nil { return fmt.Errorf("error building docker flags: %v", err) diff --git a/nodeup/pkg/model/docker_test.go b/nodeup/pkg/model/docker_test.go index de6215ddf72bb..db6fd5b17d10f 100644 --- a/nodeup/pkg/model/docker_test.go +++ b/nodeup/pkg/model/docker_test.go @@ -17,6 +17,11 @@ limitations under the License. package model import ( + "crypto/sha1" + "encoding/hex" + "io" + "net/http" + "os" "path" "testing" @@ -26,6 +31,35 @@ import ( "k8s.io/kops/upup/pkg/fi" ) +func TestDockerHashes(t *testing.T) { + if os.Getenv("VERIFY_HASHES") == "" { + t.Skip("VERIFY_HASHES not set, won't download & verify docker hashes") + } + + for _, dockerVersion := range dockerVersions { + u := dockerVersion.Source + + resp, err := http.Get(u) + if err != nil { + t.Errorf("%s: error fetching: %v", u, err) + continue + } + defer resp.Body.Close() + + hasher := sha1.New() + if _, err := io.Copy(hasher, resp.Body); err != nil { + t.Errorf("%s: error reading: %v", u, err) + continue + } + + hash := hex.EncodeToString(hasher.Sum(nil)) + if hash != dockerVersion.Hash { + t.Errorf("%s: hash was %q", dockerVersion.Source, hash) + continue + } + } +} + func TestDockerBuilder_Simple(t *testing.T) { runDockerBuilderTest(t, "simple") } diff --git a/nodeup/pkg/model/etcd_manager_tls.go b/nodeup/pkg/model/etcd_manager_tls.go new file mode 100644 index 0000000000000..6fac2b4c2e900 --- /dev/null +++ b/nodeup/pkg/model/etcd_manager_tls.go @@ -0,0 +1,63 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package model + +import ( + "github.com/golang/glog" + "k8s.io/kops/upup/pkg/fi" +) + +// EtcdManagerTLSBuilder configures TLS support for etcd-manager +type EtcdManagerTLSBuilder struct { + *NodeupModelContext +} + +var _ fi.ModelBuilder = &EtcdManagerTLSBuilder{} + +// Build is responsible for TLS configuration for etcd-manager +func (b *EtcdManagerTLSBuilder) Build(ctx *fi.ModelBuilderContext) error { + if !b.IsMaster { + return nil + } + + for _, k := range []string{"main", "events"} { + d := "/etc/kubernetes/pki/etcd-manager-" + k + + keys := make(map[string]string) + keys["etcd-manager-ca"] = "etcd-manager-ca-" + k + + for fileName, keystoreName := range keys { + cert, err := b.KeyStore.FindCert(keystoreName) + if err != nil { + return err + } + if cert == nil { + glog.Warningf("keypair %q not found, won't configure", keystoreName) + continue + } + + if err := b.BuildCertificateTask(ctx, keystoreName, d+"/"+fileName+".crt"); err != nil { + return err + } + if err := b.BuildPrivateKeyTask(ctx, keystoreName, d+"/"+fileName+".key"); err != nil { + return err + } + } + } + + return nil +} diff --git a/nodeup/pkg/model/kube_apiserver.go b/nodeup/pkg/model/kube_apiserver.go index 42806ca438d29..30e68703bbec1 100644 --- a/nodeup/pkg/model/kube_apiserver.go +++ b/nodeup/pkg/model/kube_apiserver.go @@ -283,8 +283,12 @@ func (b *KubeAPIServerBuilder) buildPod() (*v1.Pod, error) { kubeAPIServer.ClientCAFile = filepath.Join(b.PathSrvKubernetes(), "ca.crt") kubeAPIServer.TLSCertFile = filepath.Join(b.PathSrvKubernetes(), "server.cert") kubeAPIServer.TLSPrivateKeyFile = filepath.Join(b.PathSrvKubernetes(), "server.key") - kubeAPIServer.BasicAuthFile = filepath.Join(b.PathSrvKubernetes(), "basic_auth.csv") - kubeAPIServer.TokenAuthFile = filepath.Join(b.PathSrvKubernetes(), "known_tokens.csv") + if !kubeAPIServer.DisableBasicAuth { + kubeAPIServer.BasicAuthFile = filepath.Join(b.PathSrvKubernetes(), "basic_auth.csv") + } + if !kubeAPIServer.DisableTokenAuth { + kubeAPIServer.TokenAuthFile = filepath.Join(b.PathSrvKubernetes(), "known_tokens.csv") + } if b.UseEtcdTLS() { kubeAPIServer.EtcdCAFile = filepath.Join(b.PathSrvKubernetes(), "ca.crt") diff --git a/nodeup/pkg/model/kube_proxy.go b/nodeup/pkg/model/kube_proxy.go index fbefcba843528..4a335b805049f 100644 --- a/nodeup/pkg/model/kube_proxy.go +++ b/nodeup/pkg/model/kube_proxy.go @@ -102,7 +102,7 @@ func (b *KubeProxyBuilder) Build(c *fi.ModelBuilderContext) error { return nil } -// buildPod is responsble constructing the pod spec +// buildPod is responsible constructing the pod spec func (b *KubeProxyBuilder) buildPod() (*v1.Pod, error) { c := b.Cluster.Spec.KubeProxy if c == nil { diff --git a/nodeup/pkg/model/kubelet.go b/nodeup/pkg/model/kubelet.go index 446fd9c217329..2db4768615f63 100644 --- a/nodeup/pkg/model/kubelet.go +++ b/nodeup/pkg/model/kubelet.go @@ -37,6 +37,7 @@ import ( "k8s.io/kops/pkg/pki" "k8s.io/kops/pkg/systemd" "k8s.io/kops/upup/pkg/fi" + "k8s.io/kops/upup/pkg/fi/cloudup/awsup" "k8s.io/kops/upup/pkg/fi/nodeup/nodetasks" "k8s.io/kops/util/pkg/reflectutils" ) @@ -88,7 +89,15 @@ func (b *KubeletBuilder) Build(c *fi.ModelBuilderContext) error { Mode: s("0755"), }) } - + { + if kubeletConfig.PodManifestPath != "" { + t, err := b.buildManifestDirectory(kubeletConfig) + if err != nil { + return err + } + c.AddTask(t) + } + } { // @check if bootstrap tokens are enabled and create the appropreiate certificates if b.UseBootstrapTokens() { @@ -149,6 +158,16 @@ func (b *KubeletBuilder) kubeletPath() string { return kubeletCommand } +// buildManifestDirectory creates the directory where kubelet expects static manifests to reside +func (b *KubeletBuilder) buildManifestDirectory(kubeletConfig *kops.KubeletConfigSpec) (*nodetasks.File, error) { + directory := &nodetasks.File{ + Path: kubeletConfig.PodManifestPath, + Type: nodetasks.FileType_Directory, + Mode: s("0755"), + } + return directory, nil +} + // buildSystemdEnvironmentFile renders the environment file for the kubelet func (b *KubeletBuilder) buildSystemdEnvironmentFile(kubeletConfig *kops.KubeletConfigSpec) (*nodetasks.File, error) { // @step: ensure the masters do not get a bootstrap configuration @@ -156,6 +175,15 @@ func (b *KubeletBuilder) buildSystemdEnvironmentFile(kubeletConfig *kops.Kubelet kubeletConfig.BootstrapKubeconfig = "" } + if kubeletConfig.ExperimentalAllowedUnsafeSysctls != nil { + // The ExperimentalAllowedUnsafeSysctls flag was renamed in k/k #63717 + if b.IsKubernetesGTE("1.11") { + glog.V(1).Info("ExperimentalAllowedUnsafeSysctls was renamed in k8s 1.11+, please use AllowedUnsafeSysctls instead.") + kubeletConfig.AllowedUnsafeSysctls = append(kubeletConfig.ExperimentalAllowedUnsafeSysctls, kubeletConfig.AllowedUnsafeSysctls...) + kubeletConfig.ExperimentalAllowedUnsafeSysctls = nil + } + } + // TODO: Dump the separate file for flags - just complexity! flags, err := flagbuilder.BuildFlags(kubeletConfig) if err != nil { @@ -441,6 +469,42 @@ func (b *KubeletBuilder) buildKubeletConfigSpec() (*kops.KubeletConfigSpec, erro c.BootstrapKubeconfig = "" } + if b.Cluster.Spec.Networking != nil && b.Cluster.Spec.Networking.AmazonVPC != nil { + instanceType, err := awsup.GetMachineTypeInfo(b.InstanceGroup.Spec.MachineType) + if err != nil { + return c, err + } + + // Default maximum pods per node defined by KubeletConfiguration, but + // respect any value the user sets explicitly. + maxPods := int32(110) + if c.MaxPods != nil { + maxPods = *c.MaxPods + } + + // AWS VPC CNI plugin-specific maximum pod calculation based on: + // https://github.com/aws/amazon-vpc-cni-k8s/blob/f52ad45/README.md + // + // Treat the calculated value as a hard max, since networking with the CNI + // plugin won't work correctly once we exceed that maximum. + enis := instanceType.InstanceENIs + ips := instanceType.InstanceIPsPerENI + if enis > 0 && ips > 0 { + instanceMaxPods := enis*(ips-1) + 2 + if int32(instanceMaxPods) < maxPods { + maxPods = int32(instanceMaxPods) + } + } + + // Write back values that could have changed + c.MaxPods = &maxPods + if b.InstanceGroup.Spec.Kubelet != nil { + if b.InstanceGroup.Spec.Kubelet.MaxPods == nil { + b.InstanceGroup.Spec.Kubelet.MaxPods = &maxPods + } + } + } + if b.InstanceGroup.Spec.Kubelet != nil { reflectutils.JsonMergeStruct(c, b.InstanceGroup.Spec.Kubelet) } diff --git a/nodeup/pkg/model/kubelet_test.go b/nodeup/pkg/model/kubelet_test.go index e19a8ce23ab11..dddd64246033e 100644 --- a/nodeup/pkg/model/kubelet_test.go +++ b/nodeup/pkg/model/kubelet_test.go @@ -182,6 +182,15 @@ func Test_RunKubeletBuilder(t *testing.T) { } context.AddTask(fileTask) + { + task, err := builder.buildManifestDirectory(kubeletConfig) + if err != nil { + t.Fatalf("error from KubeletBuilder buildManifestDirectory: %v", err) + return + } + context.AddTask(task) + } + testutils.ValidateTasks(t, basedir, context) } diff --git a/nodeup/pkg/model/logrotate.go b/nodeup/pkg/model/logrotate.go index e78e50df293f7..53eed5fd625f1 100644 --- a/nodeup/pkg/model/logrotate.go +++ b/nodeup/pkg/model/logrotate.go @@ -113,7 +113,8 @@ func (b *LogrotateBuilder) addLogrotateService(c *fi.ModelBuilderContext) error } type logRotateOptions struct { - MaxSize string + MaxSize string + DateFormat string } func (b *LogrotateBuilder) addLogRotate(c *fi.ModelBuilderContext, name, path string, options logRotateOptions) { @@ -121,6 +122,12 @@ func (b *LogrotateBuilder) addLogRotate(c *fi.ModelBuilderContext, name, path st options.MaxSize = "100M" } + // CoreOS sets "dateext" options, and maxsize-based rotation will fail if + // the file has been previously rotated on the same calendar date. + if b.Distribution == distros.DistributionCoreOS { + options.DateFormat = "-%Y%m%d-%s" + } + lines := []string{ path + "{", " rotate 5", @@ -129,12 +136,20 @@ func (b *LogrotateBuilder) addLogRotate(c *fi.ModelBuilderContext, name, path st " notifempty", " delaycompress", " maxsize " + options.MaxSize, + } + + if options.DateFormat != "" { + lines = append(lines, " dateformat "+options.DateFormat) + } + + lines = append( + lines, " daily", " create 0644 root root", "}", - } + ) - contents := strings.Join(lines, "\n") + contents := strings.Join(lines, "\n") + "\n" c.AddTask(&nodetasks.File{ Path: "/etc/logrotate.d/" + name, diff --git a/nodeup/pkg/model/network.go b/nodeup/pkg/model/network.go index 65550e9eba586..d661ea704bd35 100644 --- a/nodeup/pkg/model/network.go +++ b/nodeup/pkg/model/network.go @@ -43,6 +43,7 @@ func (b *NetworkBuilder) Build(c *fi.ModelBuilderContext) error { } else if networking.External != nil { // external is based on kubenet assetNames = append(assetNames, "bridge", "host-local", "loopback") + } else if networking.CNI != nil || networking.Weave != nil || networking.Flannel != nil || networking.Calico != nil || networking.Canal != nil || networking.Kuberouter != nil || networking.Romana != nil || networking.AmazonVPC != nil || networking.Cilium != nil { assetNames = append(assetNames, "bridge", "host-local", "loopback", "ptp") // Do we need tuning? @@ -57,6 +58,8 @@ func (b *NetworkBuilder) Build(c *fi.ModelBuilderContext) error { // TODO combine with External // Kopeio is based on kubenet / external assetNames = append(assetNames, "bridge", "host-local", "loopback") + } else if networking.LyftVPC != nil { + assetNames = append(assetNames, "cni-ipvlan-vpc-k8s-ipam", "cni-ipvlan-vpc-k8s-ipvlan", "cni-ipvlan-vpc-k8s-tool", "cni-ipvlan-vpc-k8s-unnumbered-ptp", "loopback") } else { return fmt.Errorf("no networking mode set") } diff --git a/nodeup/pkg/model/protokube.go b/nodeup/pkg/model/protokube.go index c198bcdc3ea05..4f60a3866bb5c 100644 --- a/nodeup/pkg/model/protokube.go +++ b/nodeup/pkg/model/protokube.go @@ -69,12 +69,12 @@ func (t *ProtokubeBuilder) Build(c *fi.ModelBuilderContext) error { // retrieve the etcd peer certificates and private keys from the keystore if t.UseEtcdTLS() { - for _, x := range []string{"etcd", "etcd-client"} { + for _, x := range []string{"etcd", "etcd-peer", "etcd-client"} { if err := t.BuildCertificateTask(c, x, fmt.Sprintf("%s.pem", x)); err != nil { return err } } - for _, x := range []string{"etcd", "etcd-client"} { + for _, x := range []string{"etcd", "etcd-peer", "etcd-client"} { if err := t.BuildPrivateKeyTask(c, x, fmt.Sprintf("%s-key.pem", x)); err != nil { return err } @@ -290,8 +290,8 @@ func (t *ProtokubeBuilder) ProtokubeFlags(k8sVersion semver.Version) (*Protokube // check if we are using tls and add the options to protokube if t.UseEtcdTLS() { f.PeerTLSCaFile = s(filepath.Join(t.PathSrvKubernetes(), "ca.crt")) - f.PeerTLSCertFile = s(filepath.Join(t.PathSrvKubernetes(), "etcd.pem")) - f.PeerTLSKeyFile = s(filepath.Join(t.PathSrvKubernetes(), "etcd-key.pem")) + f.PeerTLSCertFile = s(filepath.Join(t.PathSrvKubernetes(), "etcd-peer.pem")) + f.PeerTLSKeyFile = s(filepath.Join(t.PathSrvKubernetes(), "etcd-peer-key.pem")) f.TLSCAFile = s(filepath.Join(t.PathSrvKubernetes(), "ca.crt")) f.TLSCertFile = s(filepath.Join(t.PathSrvKubernetes(), "etcd.pem")) f.TLSKeyFile = s(filepath.Join(t.PathSrvKubernetes(), "etcd-key.pem")) diff --git a/nodeup/pkg/model/sysctls.go b/nodeup/pkg/model/sysctls.go index 141d617b088cd..fec6353af2347 100644 --- a/nodeup/pkg/model/sysctls.go +++ b/nodeup/pkg/model/sysctls.go @@ -52,6 +52,14 @@ func (b *SysctlBuilder) Build(c *fi.ModelBuilderContext) error { "kernel.softlockup_all_cpu_backtrace = 1", "") + // See https://github.com/kubernetes/kops/issues/6342 + portRange := b.Cluster.Spec.KubeAPIServer.ServiceNodePortRange + if portRange == "" { + portRange = "30000-32767" // Default kube-apiserver ServiceNodePortRange + } + sysctls = append(sysctls, "net.ipv4.ip_local_reserved_ports = "+portRange, + "") + // See https://github.com/kubernetes/kube-deploy/issues/261 sysctls = append(sysctls, "# Increase the number of connections", diff --git a/nodeup/pkg/model/tests/kubelet/featuregates/cluster.yaml b/nodeup/pkg/model/tests/kubelet/featuregates/cluster.yaml index 3639005c04841..6c3b281b42dbe 100644 --- a/nodeup/pkg/model/tests/kubelet/featuregates/cluster.yaml +++ b/nodeup/pkg/model/tests/kubelet/featuregates/cluster.yaml @@ -22,6 +22,7 @@ spec: featureGates: ExperimentalCriticalPodAnnotation: "true" AllowExtTrafficLocalEndpoints: "false" + podManifestPath: "/etc/kubernetes/manifests" kubernetesVersion: v1.5.0 masterInternalName: api.internal.minimal.example.com masterPublicName: api.minimal.example.com diff --git a/nodeup/pkg/model/tests/kubelet/featuregates/tasks.yaml b/nodeup/pkg/model/tests/kubelet/featuregates/tasks.yaml index a0d60b74f39a2..ad79c9e4a8c8c 100644 --- a/nodeup/pkg/model/tests/kubelet/featuregates/tasks.yaml +++ b/nodeup/pkg/model/tests/kubelet/featuregates/tasks.yaml @@ -1,5 +1,9 @@ +mode: "0755" +path: /etc/kubernetes/manifests +type: directory +--- contents: | - DAEMON_ARGS="--feature-gates=AllowExtTrafficLocalEndpoints=false,ExperimentalCriticalPodAnnotation=true --node-labels=kubernetes.io/role=node,node-role.kubernetes.io/node= --cni-bin-dir=/opt/cni/bin/ --cni-conf-dir=/etc/cni/net.d/ --network-plugin-dir=/opt/cni/bin/" + DAEMON_ARGS="--feature-gates=AllowExtTrafficLocalEndpoints=false,ExperimentalCriticalPodAnnotation=true --node-labels=kubernetes.io/role=node,node-role.kubernetes.io/node= --pod-manifest-path=/etc/kubernetes/manifests --cni-bin-dir=/opt/cni/bin/ --cni-conf-dir=/etc/cni/net.d/ --network-plugin-dir=/opt/cni/bin/" HOME="/root" path: /etc/sysconfig/kubelet type: file diff --git a/pkg/apis/kops/cluster.go b/pkg/apis/kops/cluster.go index 53e6334172bd3..7dd5b108930c8 100644 --- a/pkg/apis/kops/cluster.go +++ b/pkg/apis/kops/cluster.go @@ -164,6 +164,8 @@ type ClusterSpec struct { IAM *IAMSpec `json:"iam,omitempty"` // EncryptionConfig controls if encryption is enabled EncryptionConfig *bool `json:"encryptionConfig,omitempty"` + // DisableSubnetTags controls if subnets are tagged in AWS + DisableSubnetTags bool `json:"disableSubnetTags,omitempty"` // Target allows for us to nest extra config for targets such as terraform Target *TargetSpec `json:"target,omitempty"` } @@ -323,7 +325,7 @@ type LoadBalancerAccessSpec struct { SecurityGroupOverride *string `json:"securityGroupOverride,omitempty"` // AdditionalSecurityGroups attaches additional security groups (e.g. sg-123456). AdditionalSecurityGroups []string `json:"additionalSecurityGroups,omitempty"` - // UseForInternalApi indicates wether the LB should be used by the kubelet + // UseForInternalApi indicates whether the LB should be used by the kubelet UseForInternalApi bool `json:"useForInternalApi,omitempty"` // SSLCertificate allows you to specify the ACM cert to be used the LB SSLCertificate string `json:"sslCertificate,omitempty"` @@ -331,7 +333,7 @@ type LoadBalancerAccessSpec struct { // KubeDNSConfig defines the kube dns configuration type KubeDNSConfig struct { - // CacheMaxSize is the maximum entries to keep in dnsmaq + // CacheMaxSize is the maximum entries to keep in dnsmasq CacheMaxSize int `json:"cacheMaxSize,omitempty"` // CacheMaxConcurrent is the maximum number of concurrent queries for dnsmasq CacheMaxConcurrent int `json:"cacheMaxConcurrent,omitempty"` @@ -536,6 +538,8 @@ func (c *Cluster) FillDefaults() error { c.Spec.Networking.Cilium.Version = CiliumDefaultVersion } // OK + } else if c.Spec.Networking.LyftVPC != nil { + // OK } else { // No networking model selected; choose Kubenet c.Spec.Networking.Kubenet = &KubenetNetworkingSpec{} diff --git a/pkg/apis/kops/componentconfig.go b/pkg/apis/kops/componentconfig.go index 94718d3546c96..44e08759c3919 100644 --- a/pkg/apis/kops/componentconfig.go +++ b/pkg/apis/kops/componentconfig.go @@ -165,7 +165,10 @@ type KubeletConfigSpec struct { // Tells the Kubelet to fail to start if swap is enabled on the node. FailSwapOn *bool `json:"failSwapOn,omitempty" flag:"fail-swap-on"` // ExperimentalAllowedUnsafeSysctls are passed to the kubelet config to whitelist allowable sysctls + // Was promoted to beta and renamed. https://github.com/kubernetes/kubernetes/pull/63717 ExperimentalAllowedUnsafeSysctls []string `json:"experimentalAllowedUnsafeSysctls,omitempty" flag:"experimental-allowed-unsafe-sysctls"` + // AllowedUnsafeSysctls are passed to the kubelet config to whitelist allowable sysctls + AllowedUnsafeSysctls []string `json:"allowedUnsafeSysctls,omitempty" flag:"allowed-unsafe-sysctls"` // StreamingConnectionIdleTimeout is the maximum time a streaming connection can be idle before the connection is automatically closed StreamingConnectionIdleTimeout *metav1.Duration `json:"streamingConnectionIdleTimeout,omitempty" flag:"streaming-connection-idle-timeout"` // DockerDisableSharedPID uses a shared PID namespace for containers in a pod. @@ -176,6 +179,14 @@ type KubeletConfigSpec struct { AuthenticationTokenWebhook *bool `json:"authenticationTokenWebhook,omitempty" flag:"authentication-token-webhook"` // AuthenticationTokenWebhook sets the duration to cache responses from the webhook token authenticator. Default is 2m. (default 2m0s) AuthenticationTokenWebhookCacheTTL *metav1.Duration `json:"authenticationTokenWebhookCacheTtl,omitempty" flag:"authentication-token-webhook-cache-ttl"` + // EventQps If > 0, limit event creations per second to this value. If 0, unlimited. + EventQPS *int32 `json:"eventQps,omitempty" flag:"event-qps"` + // MakeIptablesUtilChains will ensure iptables utility rules are present on host. + MakeIptablesUtilChains *bool `json:"makeIptablesUtilChains,omitempty" flag:"make-iptables-util-chains"` + // CAdvisorPort The port of the localhost cAdvisor endpoint (set to 0 to disable) (default 4194) + CAdvisorPort *int32 `json:"cadvisorPort,omitempty" flag:"cadvisor-port"` + // ProtectKernelDefaults Default kubelet behaviour for kernel tuning. If set, kubelet errors if any of kernel tunables is different than kubelet defaults. + ProtectKernelDefaults *bool `json:"protectKernelDefaults,omitempty" flag:"protect-kernel-defaults"` } // KubeProxyConfig defines the configuration for a proxy @@ -224,6 +235,10 @@ type KubeProxyConfig struct { type KubeAPIServerConfig struct { // Image is the docker container used Image string `json:"image,omitempty"` + // DisableBasicAuth removes the --basic-auth-file flag + DisableBasicAuth bool `json:"disableBasicAuth,omitempty"` + // EnableTokenAuth removes the --token-auth-file flag + DisableTokenAuth bool `json:"disableTokenAuth,omitempty"` // LogLevel is the logging level of the api LogLevel int32 `json:"logLevel,omitempty" flag:"v" flag-empty:"0"` // CloudProvider is the name of the cloudProvider we are using, aws, gce etcd @@ -270,6 +285,8 @@ type KubeAPIServerConfig struct { TLSCertFile string `json:"tlsCertFile,omitempty" flag:"tls-cert-file"` // TODO: Remove unused TLSPrivateKeyFile TLSPrivateKeyFile string `json:"tlsPrivateKeyFile,omitempty" flag:"tls-private-key-file"` + // TLSCipherSuites is a list of allowed TLS ciphers suites + TLSCipherSuites []string `json:"tlsCipherSuites,omitempty" flag:"tls-cipher-suites"` // TODO: Remove unused TokenAuthFile TokenAuthFile string `json:"tokenAuthFile,omitempty" flag:"token-auth-file"` // AllowPrivileged indicates if we can run privileged containers @@ -358,13 +375,18 @@ type KubeAPIServerConfig struct { // EtcdQuorumRead configures the etcd-quorum-read flag, which forces consistent reads from etcd EtcdQuorumRead *bool `json:"etcdQuorumRead,omitempty" flag:"etcd-quorum-read"` - // MinRequestTimeout configures the minimum number of seconds a handler must keep a request open before timing it out. // Currently only honored by the watch request handler MinRequestTimeout *int32 `json:"minRequestTimeout,omitempty" flag:"min-request-timeout"` // Memory limit for apiserver in MB (used to configure sizes of caches, etc.) TargetRamMb int32 `json:"targetRamMb,omitempty" flag:"target-ram-mb" flag-empty:"0"` + // Enable profiling via web interface + Profiling *bool `json:"profiling,omitempty" flag:"profiling"` + // Verify service account token + ServiceAccountLookup *bool `json:"serviceAccountLookup,omitempty" flag:"service-account-lookup"` + // Repair malformed requests from clients + RepairMalformedUpdates *bool `json:"repairMalformedUpdates,omitempty" flag:"repair-malformed-updates"` } // KubeControllerManagerConfig is the configuration for the controller @@ -430,6 +452,8 @@ type KubeControllerManagerConfig struct { HorizontalPodAutoscalerUseRestClients *bool `json:"horizontalPodAutoscalerUseRestClients,omitempty" flag:"horizontal-pod-autoscaler-use-rest-clients"` // FeatureGates is set of key=value pairs that describe feature gates for alpha/experimental features. FeatureGates map[string]string `json:"featureGates,omitempty" flag:"feature-gates"` + // Enable profiling via web interface + Profiling *bool `json:"profiling,omitempty" flag:"profiling"` } // CloudControllerManagerConfig is the configuration of the cloud controller @@ -467,6 +491,8 @@ type KubeSchedulerConfig struct { LogLevel int32 `json:"logLevel,omitempty" flag:"v"` // Image is the docker image to use Image string `json:"image,omitempty"` + // Enable profiling via web interface + Profiling *bool `json:"profiling,omitempty" flag:"profiling"` // LeaderElection defines the configuration of leader election client. LeaderElection *LeaderElectionConfiguration `json:"leaderElection,omitempty"` // UsePolicyConfigMap enable setting the scheduler policy from a configmap diff --git a/pkg/apis/kops/instancegroup.go b/pkg/apis/kops/instancegroup.go index 52274bf224c03..2e4840cc95ad6 100644 --- a/pkg/apis/kops/instancegroup.go +++ b/pkg/apis/kops/instancegroup.go @@ -123,7 +123,7 @@ type InstanceGroupSpec struct { DetailedInstanceMonitoring *bool `json:"detailedInstanceMonitoring,omitempty"` // IAMProfileSpec defines the identity of the cloud group iam profile (AWS only). IAM *IAMProfileSpec `json:"iam,omitempty"` - // SecurityGroupOverride overrides the defaut security group created by Kops for this IG (AWS only). + // SecurityGroupOverride overrides the default security group created by Kops for this IG (AWS only). SecurityGroupOverride *string `json:"securityGroupOverride,omitempty"` } diff --git a/pkg/apis/kops/networking.go b/pkg/apis/kops/networking.go index 33ba74d5dab86..e4671604ac028 100644 --- a/pkg/apis/kops/networking.go +++ b/pkg/apis/kops/networking.go @@ -31,6 +31,7 @@ type NetworkingSpec struct { Romana *RomanaNetworkingSpec `json:"romana,omitempty"` AmazonVPC *AmazonVPCNetworkingSpec `json:"amazonvpc,omitempty"` Cilium *CiliumNetworkingSpec `json:"cilium,omitempty"` + LyftVPC *LyftVPCNetworkingSpec `json:"lyftvpc,omitempty"` } // ClassicNetworkingSpec is the specification of classic networking mode, integrated into kubernetes @@ -87,6 +88,8 @@ type CalicoNetworkingSpec struct { PrometheusGoMetricsEnabled bool `json:"prometheusGoMetricsEnabled,omitempty"` // PrometheusProcessMetricsEnabled enables Prometheus process metrics collection PrometheusProcessMetricsEnabled bool `json:"prometheusProcessMetricsEnabled,omitempty"` + // MajorVersion is the version of Calico to use + MajorVersion string `json:"majorVersion,omitempty"` } // CanalNetworkingSpec declares that we want Canal networking @@ -188,3 +191,8 @@ type CiliumNetworkingSpec struct { TracePayloadLen int `json:"tracePayloadlen,omitempty"` Tunnel string `json:"tunnel,omitempty"` } + +// LyftIpVlanNetworkingSpec declares that we want to use the cni-ipvlan-vpc-k8s CNI networking +type LyftVPCNetworkingSpec struct { + SubnetTags map[string]string `json:"subnetTags,omitempty"` +} diff --git a/pkg/apis/kops/v1alpha1/cluster.go b/pkg/apis/kops/v1alpha1/cluster.go index 78df8b0a535a3..64ecdb6d98474 100644 --- a/pkg/apis/kops/v1alpha1/cluster.go +++ b/pkg/apis/kops/v1alpha1/cluster.go @@ -163,6 +163,8 @@ type ClusterSpec struct { IAM *IAMSpec `json:"iam,omitempty"` // EncryptionConfig holds the encryption config EncryptionConfig *bool `json:"encryptionConfig,omitempty"` + // DisableSubnetTags controls if subnets are tagged in AWS + DisableSubnetTags bool `json:"DisableSubnetTags,omitempty"` // Target allows for us to nest extra config for targets such as terraform Target *TargetSpec `json:"target,omitempty"` } @@ -322,7 +324,7 @@ type LoadBalancerAccessSpec struct { SecurityGroupOverride *string `json:"securityGroupOverride,omitempty"` // AdditionalSecurityGroups attaches additional security groups (e.g. sg-123456). AdditionalSecurityGroups []string `json:"additionalSecurityGroups,omitempty"` - // UseForInternalApi indicates wether the LB should be used by the kubelet + // UseForInternalApi indicates whether the LB should be used by the kubelet UseForInternalApi bool `json:"useForInternalApi,omitempty"` // SSLCertificate allows you to specify the ACM cert to be used the LB SSLCertificate string `json:"sslCertificate,omitempty"` @@ -330,7 +332,7 @@ type LoadBalancerAccessSpec struct { // KubeDNSConfig defines the kube dns configuration type KubeDNSConfig struct { - // CacheMaxSize is the maximum entries to keep in dnsmaq + // CacheMaxSize is the maximum entries to keep in dnsmasq CacheMaxSize int `json:"cacheMaxSize,omitempty"` // CacheMaxConcurrent is the maximum number of concurrent queries for dnsmasq CacheMaxConcurrent int `json:"cacheMaxConcurrent,omitempty"` diff --git a/pkg/apis/kops/v1alpha1/componentconfig.go b/pkg/apis/kops/v1alpha1/componentconfig.go index 8669377ff8d52..ac2b88b4f70b2 100644 --- a/pkg/apis/kops/v1alpha1/componentconfig.go +++ b/pkg/apis/kops/v1alpha1/componentconfig.go @@ -165,7 +165,10 @@ type KubeletConfigSpec struct { // Tells the Kubelet to fail to start if swap is enabled on the node. FailSwapOn *bool `json:"failSwapOn,omitempty" flag:"fail-swap-on"` // ExperimentalAllowedUnsafeSysctls are passed to the kubelet config to whitelist allowable sysctls - ExperimentalAllowedUnsafeSysctls []string `json:"experimental_allowed_unsafe_sysctls,omitempty" flag:"experimental-allowed-unsafe-sysctls"` + // Was promoted to beta and renamed. https://github.com/kubernetes/kubernetes/pull/63717 + ExperimentalAllowedUnsafeSysctls []string `json:"experimentalAllowedUnsafeSysctls,omitempty" flag:"experimental-allowed-unsafe-sysctls"` + // AllowedUnsafeSysctls are passed to the kubelet config to whitelist allowable sysctls + AllowedUnsafeSysctls []string `json:"allowedUnsafeSysctls,omitempty" flag:"allowed-unsafe-sysctls"` // StreamingConnectionIdleTimeout is the maximum time a streaming connection can be idle before the connection is automatically closed StreamingConnectionIdleTimeout *metav1.Duration `json:"streamingConnectionIdleTimeout,omitempty" flag:"streaming-connection-idle-timeout"` // DockerDisableSharedPID uses a shared PID namespace for containers in a pod. @@ -176,6 +179,14 @@ type KubeletConfigSpec struct { AuthenticationTokenWebhook *bool `json:"authenticationTokenWebhook,omitempty" flag:"authentication-token-webhook"` // AuthenticationTokenWebhook sets the duration to cache responses from the webhook token authenticator. Default is 2m. (default 2m0s) AuthenticationTokenWebhookCacheTTL *metav1.Duration `json:"authenticationTokenWebhookCacheTtl,omitempty" flag:"authentication-token-webhook-cache-ttl"` + // EventQps If > 0, limit event creations per second to this value. If 0, unlimited. + EventQPS *int32 `json:"eventQps,omitempty" flag:"event-qps"` + // MakeIptablesUtilChains will ensure iptables utility rules are present on host. + MakeIptablesUtilChains *bool `json:"makeIptablesUtilChains,omitempty" flag:"make-iptables-util-chains"` + // CAdvisorPort The port of the localhost cAdvisor endpoint (set to 0 to disable) (default 4194) + CAdvisorPort *int32 `json:"cadvisorPort,omitempty" flag:"cadvisor-port"` + // ProtectKernelDefaults Default kubelet behaviour for kernel tuning. If set, kubelet errors if any of kernel tunables is different than kubelet defaults. + ProtectKernelDefaults *bool `json:"protectKernelDefaults,omitempty" flag:"protect-kernel-defaults"` } // KubeProxyConfig defines the configuration for a proxy @@ -224,6 +235,10 @@ type KubeProxyConfig struct { type KubeAPIServerConfig struct { // Image is the docker container used Image string `json:"image,omitempty"` + // DisableBasicAuth removes the --basic-auth-file flag + DisableBasicAuth bool `json:"disableBasicAuth,omitempty"` + // EnableTokenAuth removes the --token-auth-file flag + DisableTokenAuth bool `json:"disableTokenAuth,omitempty"` // LogLevel is the logging level of the api LogLevel int32 `json:"logLevel,omitempty" flag:"v" flag-empty:"0"` // CloudProvider is the name of the cloudProvider we are using, aws, gce etcd @@ -270,6 +285,8 @@ type KubeAPIServerConfig struct { TLSCertFile string `json:"tlsCertFile,omitempty" flag:"tls-cert-file"` // TODO: Remove unused TLSPrivateKeyFile TLSPrivateKeyFile string `json:"tlsPrivateKeyFile,omitempty" flag:"tls-private-key-file"` + // TLSCipherSuites is a list of allowed TLS ciphers suites + TLSCipherSuites []string `json:"tlsCipherSuites,omitempty" flag:"tls-cipher-suites"` // TODO: Remove unused TokenAuthFile TokenAuthFile string `json:"tokenAuthFile,omitempty" flag:"token-auth-file"` // AllowPrivileged indicates if we can run privileged containers @@ -358,13 +375,18 @@ type KubeAPIServerConfig struct { // EtcdQuorumRead configures the etcd-quorum-read flag, which forces consistent reads from etcd EtcdQuorumRead *bool `json:"etcdQuorumRead,omitempty" flag:"etcd-quorum-read"` - // MinRequestTimeout configures the minimum number of seconds a handler must keep a request open before timing it out. // Currently only honored by the watch request handler MinRequestTimeout *int32 `json:"minRequestTimeout,omitempty" flag:"min-request-timeout"` // Memory limit for apiserver in MB (used to configure sizes of caches, etc.) TargetRamMb int32 `json:"targetRamMb,omitempty" flag:"target-ram-mb" flag-empty:"0"` + // Enable profiling via web interface. + Profiling *bool `json:"profiling,omitempty" flag:"profiling"` + // Verify service account token + ServiceAccountLookup *bool `json:"serviceAccountLookup,omitempty" flag:"service-account-lookup"` + // Repair malformed requests from clients + RepairMalformedUpdates *bool `json:"repairMalformedUpdates,omitempty" flag:"repair-malformed-updates"` } // KubeControllerManagerConfig is the configuration for the controller @@ -430,6 +452,8 @@ type KubeControllerManagerConfig struct { HorizontalPodAutoscalerUseRestClients *bool `json:"horizontalPodAutoscalerUseRestClients,omitempty" flag:"horizontal-pod-autoscaler-use-rest-clients"` // FeatureGates is set of key=value pairs that describe feature gates for alpha/experimental features. FeatureGates map[string]string `json:"featureGates,omitempty" flag:"feature-gates"` + // Enable profiling via web interface. + Profiling *bool `json:"profiling,omitempty" flag:"profiling"` } // CloudControllerManagerConfig is the configuration of the cloud controller @@ -467,6 +491,8 @@ type KubeSchedulerConfig struct { LogLevel int32 `json:"logLevel,omitempty" flag:"v"` // Image is the docker image to use Image string `json:"image,omitempty"` + // Enable profiling via web interface + Profiling bool `json:"profiling,omitempty" flag:"profiling"` // LeaderElection defines the configuration of leader election client. LeaderElection *LeaderElectionConfiguration `json:"leaderElection,omitempty"` // UsePolicyConfigMap enable setting the scheduler policy from a configmap diff --git a/pkg/apis/kops/v1alpha1/instancegroup.go b/pkg/apis/kops/v1alpha1/instancegroup.go index 3465f076766fa..73edc196156a3 100644 --- a/pkg/apis/kops/v1alpha1/instancegroup.go +++ b/pkg/apis/kops/v1alpha1/instancegroup.go @@ -102,7 +102,7 @@ type InstanceGroupSpec struct { DetailedInstanceMonitoring *bool `json:"detailedInstanceMonitoring,omitempty"` // IAMProfileSpec defines the identity of the cloud group iam profile (AWS only). IAM *IAMProfileSpec `json:"iam,omitempty"` - // SecurityGroupOverride overrides the defaut security group created by Kops for this IG (AWS only). + // SecurityGroupOverride overrides the default security group created by Kops for this IG (AWS only). SecurityGroupOverride *string `json:"securityGroupOverride,omitempty"` } diff --git a/pkg/apis/kops/v1alpha1/networking.go b/pkg/apis/kops/v1alpha1/networking.go index d30ce6c6081a4..61d57108b79b4 100644 --- a/pkg/apis/kops/v1alpha1/networking.go +++ b/pkg/apis/kops/v1alpha1/networking.go @@ -31,6 +31,7 @@ type NetworkingSpec struct { Romana *RomanaNetworkingSpec `json:"romana,omitempty"` AmazonVPC *AmazonVPCNetworkingSpec `json:"amazonvpc,omitempty"` Cilium *CiliumNetworkingSpec `json:"cilium,omitempty"` + LyftVPC *LyftVPCNetworkingSpec `json:"lyftvpc,omitempty"` } // ClassicNetworkingSpec is the specification of classic networking mode, integrated into kubernetes @@ -87,6 +88,8 @@ type CalicoNetworkingSpec struct { PrometheusGoMetricsEnabled bool `json:"prometheusGoMetricsEnabled,omitempty"` // PrometheusProcessMetricsEnabled enables Prometheus process metrics collection PrometheusProcessMetricsEnabled bool `json:"prometheusProcessMetricsEnabled,omitempty"` + // MajorVersion is the version of Calico to use + MajorVersion string `json:"majorVersion,omitempty"` } // CanalNetworkingSpec declares that we want Canal networking @@ -185,3 +188,8 @@ type CiliumNetworkingSpec struct { TracePayloadLen int `json:"tracePayloadlen,omitempty"` Tunnel string `json:"tunnel,omitempty"` } + +// LyftIpVlanNetworkingSpec declares that we want to use the cni-ipvlan-vpc-k8s CNI networking +type LyftVPCNetworkingSpec struct { + SubnetTags map[string]string `json:"subnetTags,omitempty"` +} diff --git a/pkg/apis/kops/v1alpha1/zz_generated.conversion.go b/pkg/apis/kops/v1alpha1/zz_generated.conversion.go index 97f005eadabda..ca8e812cc84cd 100644 --- a/pkg/apis/kops/v1alpha1/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha1/zz_generated.conversion.go @@ -1,7 +1,7 @@ // +build !ignore_autogenerated /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -21,6 +21,7 @@ limitations under the License. package v1alpha1 import ( + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" conversion "k8s.io/apimachinery/pkg/conversion" runtime "k8s.io/apimachinery/pkg/runtime" kops "k8s.io/kops/pkg/apis/kops" @@ -136,6 +137,8 @@ func RegisterConversions(scheme *runtime.Scheme) error { Convert_kops_LoadBalancer_To_v1alpha1_LoadBalancer, Convert_v1alpha1_LoadBalancerAccessSpec_To_kops_LoadBalancerAccessSpec, Convert_kops_LoadBalancerAccessSpec_To_v1alpha1_LoadBalancerAccessSpec, + Convert_v1alpha1_LyftVPCNetworkingSpec_To_kops_LyftVPCNetworkingSpec, + Convert_kops_LyftVPCNetworkingSpec_To_v1alpha1_LyftVPCNetworkingSpec, Convert_v1alpha1_NetworkingSpec_To_kops_NetworkingSpec, Convert_kops_NetworkingSpec_To_v1alpha1_NetworkingSpec, Convert_v1alpha1_NodeAuthorizationSpec_To_kops_NodeAuthorizationSpec, @@ -453,6 +456,7 @@ func autoConvert_v1alpha1_CalicoNetworkingSpec_To_kops_CalicoNetworkingSpec(in * out.PrometheusMetricsPort = in.PrometheusMetricsPort out.PrometheusGoMetricsEnabled = in.PrometheusGoMetricsEnabled out.PrometheusProcessMetricsEnabled = in.PrometheusProcessMetricsEnabled + out.MajorVersion = in.MajorVersion return nil } @@ -469,6 +473,7 @@ func autoConvert_kops_CalicoNetworkingSpec_To_v1alpha1_CalicoNetworkingSpec(in * out.PrometheusMetricsPort = in.PrometheusMetricsPort out.PrometheusGoMetricsEnabled = in.PrometheusGoMetricsEnabled out.PrometheusProcessMetricsEnabled = in.PrometheusProcessMetricsEnabled + out.MajorVersion = in.MajorVersion return nil } @@ -1066,6 +1071,7 @@ func autoConvert_v1alpha1_ClusterSpec_To_kops_ClusterSpec(in *ClusterSpec, out * out.IAM = nil } out.EncryptionConfig = in.EncryptionConfig + out.DisableSubnetTags = in.DisableSubnetTags if in.Target != nil { in, out := &in.Target, &out.Target *out = new(kops.TargetSpec) @@ -1332,6 +1338,7 @@ func autoConvert_kops_ClusterSpec_To_v1alpha1_ClusterSpec(in *kops.ClusterSpec, out.IAM = nil } out.EncryptionConfig = in.EncryptionConfig + out.DisableSubnetTags = in.DisableSubnetTags if in.Target != nil { in, out := &in.Target, &out.Target *out = new(TargetSpec) @@ -2163,6 +2170,8 @@ func Convert_kops_KopeioNetworkingSpec_To_v1alpha1_KopeioNetworkingSpec(in *kops func autoConvert_v1alpha1_KubeAPIServerConfig_To_kops_KubeAPIServerConfig(in *KubeAPIServerConfig, out *kops.KubeAPIServerConfig, s conversion.Scope) error { out.Image = in.Image + out.DisableBasicAuth = in.DisableBasicAuth + out.DisableTokenAuth = in.DisableTokenAuth out.LogLevel = in.LogLevel out.CloudProvider = in.CloudProvider out.SecurePort = in.SecurePort @@ -2186,6 +2195,7 @@ func autoConvert_v1alpha1_KubeAPIServerConfig_To_kops_KubeAPIServerConfig(in *Ku out.ClientCAFile = in.ClientCAFile out.TLSCertFile = in.TLSCertFile out.TLSPrivateKeyFile = in.TLSPrivateKeyFile + out.TLSCipherSuites = in.TLSCipherSuites out.TokenAuthFile = in.TokenAuthFile out.AllowPrivileged = in.AllowPrivileged out.APIServerCount = in.APIServerCount @@ -2226,6 +2236,9 @@ func autoConvert_v1alpha1_KubeAPIServerConfig_To_kops_KubeAPIServerConfig(in *Ku out.EtcdQuorumRead = in.EtcdQuorumRead out.MinRequestTimeout = in.MinRequestTimeout out.TargetRamMb = in.TargetRamMb + out.Profiling = in.Profiling + out.ServiceAccountLookup = in.ServiceAccountLookup + out.RepairMalformedUpdates = in.RepairMalformedUpdates return nil } @@ -2236,6 +2249,8 @@ func Convert_v1alpha1_KubeAPIServerConfig_To_kops_KubeAPIServerConfig(in *KubeAP func autoConvert_kops_KubeAPIServerConfig_To_v1alpha1_KubeAPIServerConfig(in *kops.KubeAPIServerConfig, out *KubeAPIServerConfig, s conversion.Scope) error { out.Image = in.Image + out.DisableBasicAuth = in.DisableBasicAuth + out.DisableTokenAuth = in.DisableTokenAuth out.LogLevel = in.LogLevel out.CloudProvider = in.CloudProvider out.SecurePort = in.SecurePort @@ -2259,6 +2274,7 @@ func autoConvert_kops_KubeAPIServerConfig_To_v1alpha1_KubeAPIServerConfig(in *ko out.ClientCAFile = in.ClientCAFile out.TLSCertFile = in.TLSCertFile out.TLSPrivateKeyFile = in.TLSPrivateKeyFile + out.TLSCipherSuites = in.TLSCipherSuites out.TokenAuthFile = in.TokenAuthFile out.AllowPrivileged = in.AllowPrivileged out.APIServerCount = in.APIServerCount @@ -2299,6 +2315,9 @@ func autoConvert_kops_KubeAPIServerConfig_To_v1alpha1_KubeAPIServerConfig(in *ko out.EtcdQuorumRead = in.EtcdQuorumRead out.MinRequestTimeout = in.MinRequestTimeout out.TargetRamMb = in.TargetRamMb + out.Profiling = in.Profiling + out.ServiceAccountLookup = in.ServiceAccountLookup + out.RepairMalformedUpdates = in.RepairMalformedUpdates return nil } @@ -2341,6 +2360,7 @@ func autoConvert_v1alpha1_KubeControllerManagerConfig_To_kops_KubeControllerMana out.HorizontalPodAutoscalerUpscaleDelay = in.HorizontalPodAutoscalerUpscaleDelay out.HorizontalPodAutoscalerUseRestClients = in.HorizontalPodAutoscalerUseRestClients out.FeatureGates = in.FeatureGates + out.Profiling = in.Profiling return nil } @@ -2383,6 +2403,7 @@ func autoConvert_kops_KubeControllerManagerConfig_To_v1alpha1_KubeControllerMana out.HorizontalPodAutoscalerUpscaleDelay = in.HorizontalPodAutoscalerUpscaleDelay out.HorizontalPodAutoscalerUseRestClients = in.HorizontalPodAutoscalerUseRestClients out.FeatureGates = in.FeatureGates + out.Profiling = in.Profiling return nil } @@ -2487,6 +2508,9 @@ func autoConvert_v1alpha1_KubeSchedulerConfig_To_kops_KubeSchedulerConfig(in *Ku out.Master = in.Master out.LogLevel = in.LogLevel out.Image = in.Image + if err := v1.Convert_bool_To_Pointer_bool(&in.Profiling, &out.Profiling, s); err != nil { + return err + } if in.LeaderElection != nil { in, out := &in.LeaderElection, &out.LeaderElection *out = new(kops.LeaderElectionConfiguration) @@ -2510,6 +2534,9 @@ func autoConvert_kops_KubeSchedulerConfig_To_v1alpha1_KubeSchedulerConfig(in *ko out.Master = in.Master out.LogLevel = in.LogLevel out.Image = in.Image + if err := v1.Convert_Pointer_bool_To_bool(&in.Profiling, &out.Profiling, s); err != nil { + return err + } if in.LeaderElection != nil { in, out := &in.LeaderElection, &out.LeaderElection *out = new(LeaderElectionConfiguration) @@ -2592,11 +2619,16 @@ func autoConvert_v1alpha1_KubeletConfigSpec_To_kops_KubeletConfigSpec(in *Kubele out.VolumeStatsAggPeriod = in.VolumeStatsAggPeriod out.FailSwapOn = in.FailSwapOn out.ExperimentalAllowedUnsafeSysctls = in.ExperimentalAllowedUnsafeSysctls + out.AllowedUnsafeSysctls = in.AllowedUnsafeSysctls out.StreamingConnectionIdleTimeout = in.StreamingConnectionIdleTimeout out.DockerDisableSharedPID = in.DockerDisableSharedPID out.RootDir = in.RootDir out.AuthenticationTokenWebhook = in.AuthenticationTokenWebhook out.AuthenticationTokenWebhookCacheTTL = in.AuthenticationTokenWebhookCacheTTL + out.EventQPS = in.EventQPS + out.MakeIptablesUtilChains = in.MakeIptablesUtilChains + out.CAdvisorPort = in.CAdvisorPort + out.ProtectKernelDefaults = in.ProtectKernelDefaults return nil } @@ -2668,11 +2700,16 @@ func autoConvert_kops_KubeletConfigSpec_To_v1alpha1_KubeletConfigSpec(in *kops.K out.VolumeStatsAggPeriod = in.VolumeStatsAggPeriod out.FailSwapOn = in.FailSwapOn out.ExperimentalAllowedUnsafeSysctls = in.ExperimentalAllowedUnsafeSysctls + out.AllowedUnsafeSysctls = in.AllowedUnsafeSysctls out.StreamingConnectionIdleTimeout = in.StreamingConnectionIdleTimeout out.DockerDisableSharedPID = in.DockerDisableSharedPID out.RootDir = in.RootDir out.AuthenticationTokenWebhook = in.AuthenticationTokenWebhook out.AuthenticationTokenWebhookCacheTTL = in.AuthenticationTokenWebhookCacheTTL + out.EventQPS = in.EventQPS + out.MakeIptablesUtilChains = in.MakeIptablesUtilChains + out.CAdvisorPort = in.CAdvisorPort + out.ProtectKernelDefaults = in.ProtectKernelDefaults return nil } @@ -2789,6 +2826,26 @@ func Convert_kops_LoadBalancerAccessSpec_To_v1alpha1_LoadBalancerAccessSpec(in * return autoConvert_kops_LoadBalancerAccessSpec_To_v1alpha1_LoadBalancerAccessSpec(in, out, s) } +func autoConvert_v1alpha1_LyftVPCNetworkingSpec_To_kops_LyftVPCNetworkingSpec(in *LyftVPCNetworkingSpec, out *kops.LyftVPCNetworkingSpec, s conversion.Scope) error { + out.SubnetTags = in.SubnetTags + return nil +} + +// Convert_v1alpha1_LyftVPCNetworkingSpec_To_kops_LyftVPCNetworkingSpec is an autogenerated conversion function. +func Convert_v1alpha1_LyftVPCNetworkingSpec_To_kops_LyftVPCNetworkingSpec(in *LyftVPCNetworkingSpec, out *kops.LyftVPCNetworkingSpec, s conversion.Scope) error { + return autoConvert_v1alpha1_LyftVPCNetworkingSpec_To_kops_LyftVPCNetworkingSpec(in, out, s) +} + +func autoConvert_kops_LyftVPCNetworkingSpec_To_v1alpha1_LyftVPCNetworkingSpec(in *kops.LyftVPCNetworkingSpec, out *LyftVPCNetworkingSpec, s conversion.Scope) error { + out.SubnetTags = in.SubnetTags + return nil +} + +// Convert_kops_LyftVPCNetworkingSpec_To_v1alpha1_LyftVPCNetworkingSpec is an autogenerated conversion function. +func Convert_kops_LyftVPCNetworkingSpec_To_v1alpha1_LyftVPCNetworkingSpec(in *kops.LyftVPCNetworkingSpec, out *LyftVPCNetworkingSpec, s conversion.Scope) error { + return autoConvert_kops_LyftVPCNetworkingSpec_To_v1alpha1_LyftVPCNetworkingSpec(in, out, s) +} + func autoConvert_v1alpha1_NetworkingSpec_To_kops_NetworkingSpec(in *NetworkingSpec, out *kops.NetworkingSpec, s conversion.Scope) error { if in.Classic != nil { in, out := &in.Classic, &out.Classic @@ -2907,6 +2964,15 @@ func autoConvert_v1alpha1_NetworkingSpec_To_kops_NetworkingSpec(in *NetworkingSp } else { out.Cilium = nil } + if in.LyftVPC != nil { + in, out := &in.LyftVPC, &out.LyftVPC + *out = new(kops.LyftVPCNetworkingSpec) + if err := Convert_v1alpha1_LyftVPCNetworkingSpec_To_kops_LyftVPCNetworkingSpec(*in, *out, s); err != nil { + return err + } + } else { + out.LyftVPC = nil + } return nil } @@ -3033,6 +3099,15 @@ func autoConvert_kops_NetworkingSpec_To_v1alpha1_NetworkingSpec(in *kops.Network } else { out.Cilium = nil } + if in.LyftVPC != nil { + in, out := &in.LyftVPC, &out.LyftVPC + *out = new(LyftVPCNetworkingSpec) + if err := Convert_kops_LyftVPCNetworkingSpec_To_v1alpha1_LyftVPCNetworkingSpec(*in, *out, s); err != nil { + return err + } + } else { + out.LyftVPC = nil + } return nil } diff --git a/pkg/apis/kops/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/kops/v1alpha1/zz_generated.deepcopy.go index 0288a34d4e2cc..0edffb3382c99 100644 --- a/pkg/apis/kops/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/kops/v1alpha1/zz_generated.deepcopy.go @@ -1,7 +1,7 @@ // +build !ignore_autogenerated /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -1917,6 +1917,11 @@ func (in *KubeAPIServerConfig) DeepCopyInto(out *KubeAPIServerConfig) { *out = make([]string, len(*in)) copy(*out, *in) } + if in.TLSCipherSuites != nil { + in, out := &in.TLSCipherSuites, &out.TLSCipherSuites + *out = make([]string, len(*in)) + copy(*out, *in) + } if in.AllowPrivileged != nil { in, out := &in.AllowPrivileged, &out.AllowPrivileged if *in == nil { @@ -2181,6 +2186,33 @@ func (in *KubeAPIServerConfig) DeepCopyInto(out *KubeAPIServerConfig) { **out = **in } } + if in.Profiling != nil { + in, out := &in.Profiling, &out.Profiling + if *in == nil { + *out = nil + } else { + *out = new(bool) + **out = **in + } + } + if in.ServiceAccountLookup != nil { + in, out := &in.ServiceAccountLookup, &out.ServiceAccountLookup + if *in == nil { + *out = nil + } else { + *out = new(bool) + **out = **in + } + } + if in.RepairMalformedUpdates != nil { + in, out := &in.RepairMalformedUpdates, &out.RepairMalformedUpdates + if *in == nil { + *out = nil + } else { + *out = new(bool) + **out = **in + } + } return } @@ -2344,6 +2376,15 @@ func (in *KubeControllerManagerConfig) DeepCopyInto(out *KubeControllerManagerCo (*out)[key] = val } } + if in.Profiling != nil { + in, out := &in.Profiling, &out.Profiling + if *in == nil { + *out = nil + } else { + *out = new(bool) + **out = **in + } + } return } @@ -2788,6 +2829,11 @@ func (in *KubeletConfigSpec) DeepCopyInto(out *KubeletConfigSpec) { *out = make([]string, len(*in)) copy(*out, *in) } + if in.AllowedUnsafeSysctls != nil { + in, out := &in.AllowedUnsafeSysctls, &out.AllowedUnsafeSysctls + *out = make([]string, len(*in)) + copy(*out, *in) + } if in.StreamingConnectionIdleTimeout != nil { in, out := &in.StreamingConnectionIdleTimeout, &out.StreamingConnectionIdleTimeout if *in == nil { @@ -2824,6 +2870,42 @@ func (in *KubeletConfigSpec) DeepCopyInto(out *KubeletConfigSpec) { **out = **in } } + if in.EventQPS != nil { + in, out := &in.EventQPS, &out.EventQPS + if *in == nil { + *out = nil + } else { + *out = new(int32) + **out = **in + } + } + if in.MakeIptablesUtilChains != nil { + in, out := &in.MakeIptablesUtilChains, &out.MakeIptablesUtilChains + if *in == nil { + *out = nil + } else { + *out = new(bool) + **out = **in + } + } + if in.CAdvisorPort != nil { + in, out := &in.CAdvisorPort, &out.CAdvisorPort + if *in == nil { + *out = nil + } else { + *out = new(int32) + **out = **in + } + } + if in.ProtectKernelDefaults != nil { + in, out := &in.ProtectKernelDefaults, &out.ProtectKernelDefaults + if *in == nil { + *out = nil + } else { + *out = new(bool) + **out = **in + } + } return } @@ -2967,6 +3049,29 @@ func (in *LoadBalancerAccessSpec) DeepCopy() *LoadBalancerAccessSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LyftVPCNetworkingSpec) DeepCopyInto(out *LyftVPCNetworkingSpec) { + *out = *in + if in.SubnetTags != nil { + in, out := &in.SubnetTags, &out.SubnetTags + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LyftVPCNetworkingSpec. +func (in *LyftVPCNetworkingSpec) DeepCopy() *LyftVPCNetworkingSpec { + if in == nil { + return nil + } + out := new(LyftVPCNetworkingSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NetworkingSpec) DeepCopyInto(out *NetworkingSpec) { *out = *in @@ -3087,6 +3192,15 @@ func (in *NetworkingSpec) DeepCopyInto(out *NetworkingSpec) { (*in).DeepCopyInto(*out) } } + if in.LyftVPC != nil { + in, out := &in.LyftVPC, &out.LyftVPC + if *in == nil { + *out = nil + } else { + *out = new(LyftVPCNetworkingSpec) + (*in).DeepCopyInto(*out) + } + } return } diff --git a/pkg/apis/kops/v1alpha1/zz_generated.defaults.go b/pkg/apis/kops/v1alpha1/zz_generated.defaults.go index 536f360071481..96886b13709e2 100644 --- a/pkg/apis/kops/v1alpha1/zz_generated.defaults.go +++ b/pkg/apis/kops/v1alpha1/zz_generated.defaults.go @@ -1,7 +1,7 @@ // +build !ignore_autogenerated /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/apis/kops/v1alpha2/cluster.go b/pkg/apis/kops/v1alpha2/cluster.go index 4884d8751d618..0aec4f76585b9 100644 --- a/pkg/apis/kops/v1alpha2/cluster.go +++ b/pkg/apis/kops/v1alpha2/cluster.go @@ -164,6 +164,8 @@ type ClusterSpec struct { IAM *IAMSpec `json:"iam,omitempty"` // EncryptionConfig holds the encryption config EncryptionConfig *bool `json:"encryptionConfig,omitempty"` + // DisableSubnetTags controls if subnets are tagged in AWS + DisableSubnetTags bool `json:"DisableSubnetTags,omitempty"` // Target allows for us to nest extra config for targets such as terraform Target *TargetSpec `json:"target,omitempty"` } @@ -323,7 +325,7 @@ type LoadBalancerAccessSpec struct { SecurityGroupOverride *string `json:"securityGroupOverride,omitempty"` // AdditionalSecurityGroups attaches additional security groups (e.g. sg-123456). AdditionalSecurityGroups []string `json:"additionalSecurityGroups,omitempty"` - // UseForInternalApi indicates wether the LB should be used by the kubelet + // UseForInternalApi indicates whether the LB should be used by the kubelet UseForInternalApi bool `json:"useForInternalApi,omitempty"` // SSLCertificate allows you to specify the ACM cert to be used the LB SSLCertificate string `json:"sslCertificate,omitempty"` @@ -331,7 +333,7 @@ type LoadBalancerAccessSpec struct { // KubeDNSConfig defines the kube dns configuration type KubeDNSConfig struct { - // CacheMaxSize is the maximum entries to keep in dnsmaq + // CacheMaxSize is the maximum entries to keep in dnsmasq CacheMaxSize int `json:"cacheMaxSize,omitempty"` // CacheMaxConcurrent is the maximum number of concurrent queries for dnsmasq CacheMaxConcurrent int `json:"cacheMaxConcurrent,omitempty"` diff --git a/pkg/apis/kops/v1alpha2/componentconfig.go b/pkg/apis/kops/v1alpha2/componentconfig.go index 3d50d3a39f96e..a920c2d8b31bf 100644 --- a/pkg/apis/kops/v1alpha2/componentconfig.go +++ b/pkg/apis/kops/v1alpha2/componentconfig.go @@ -165,7 +165,10 @@ type KubeletConfigSpec struct { // Tells the Kubelet to fail to start if swap is enabled on the node. FailSwapOn *bool `json:"failSwapOn,omitempty" flag:"fail-swap-on"` // ExperimentalAllowedUnsafeSysctls are passed to the kubelet config to whitelist allowable sysctls - ExperimentalAllowedUnsafeSysctls []string `json:"experimental_allowed_unsafe_sysctls,omitempty" flag:"experimental-allowed-unsafe-sysctls"` + // Was promoted to beta and renamed. https://github.com/kubernetes/kubernetes/pull/63717 + ExperimentalAllowedUnsafeSysctls []string `json:"experimentalAllowedUnsafeSysctls,omitempty" flag:"experimental-allowed-unsafe-sysctls"` + // AllowedUnsafeSysctls are passed to the kubelet config to whitelist allowable sysctls + AllowedUnsafeSysctls []string `json:"allowedUnsafeSysctls,omitempty" flag:"allowed-unsafe-sysctls"` // StreamingConnectionIdleTimeout is the maximum time a streaming connection can be idle before the connection is automatically closed StreamingConnectionIdleTimeout *metav1.Duration `json:"streamingConnectionIdleTimeout,omitempty" flag:"streaming-connection-idle-timeout"` // DockerDisableSharedPID uses a shared PID namespace for containers in a pod. @@ -176,6 +179,14 @@ type KubeletConfigSpec struct { AuthenticationTokenWebhook *bool `json:"authenticationTokenWebhook,omitempty" flag:"authentication-token-webhook"` // AuthenticationTokenWebhook sets the duration to cache responses from the webhook token authenticator. Default is 2m. (default 2m0s) AuthenticationTokenWebhookCacheTTL *metav1.Duration `json:"authenticationTokenWebhookCacheTtl,omitempty" flag:"authentication-token-webhook-cache-ttl"` + // EventQps If > 0, limit event creations per second to this value. If 0, unlimited. + EventQPS *int32 `json:"eventQps,omitempty" flag:"event-qps"` + // MakeIptablesUtilChains will ensure iptables utility rules are present on host. + MakeIptablesUtilChains *bool `json:"makeIptablesUtilChains,omitempty" flag:"make-iptables-util-chains"` + // CAdvisorPort The port of the localhost cAdvisor endpoint (set to 0 to disable) (default 4194) + CAdvisorPort *int32 `json:"cadvisorPort,omitempty" flag:"cadvisor-port"` + // ProtectKernelDefaults Default kubelet behaviour for kernel tuning. If set, kubelet errors if any of kernel tunables is different than kubelet defaults. + ProtectKernelDefaults *bool `json:"protectKernelDefaults,omitempty" flag:"protect-kernel-defaults"` } // KubeProxyConfig defines the configuration for a proxy @@ -224,6 +235,10 @@ type KubeProxyConfig struct { type KubeAPIServerConfig struct { // Image is the docker container used Image string `json:"image,omitempty"` + // DisableBasicAuth removes the --basic-auth-file flag + DisableBasicAuth bool `json:"disableBasicAuth,omitempty"` + // EnableTokenAuth removes the --token-auth-file flag + DisableTokenAuth bool `json:"disableTokenAuth,omitempty"` // LogLevel is the logging level of the api LogLevel int32 `json:"logLevel,omitempty" flag:"v" flag-empty:"0"` // CloudProvider is the name of the cloudProvider we are using, aws, gce etcd @@ -270,6 +285,8 @@ type KubeAPIServerConfig struct { TLSCertFile string `json:"tlsCertFile,omitempty" flag:"tls-cert-file"` // TODO: Remove unused TLSPrivateKeyFile TLSPrivateKeyFile string `json:"tlsPrivateKeyFile,omitempty" flag:"tls-private-key-file"` + // TLSCipherSuites is a list of allowed TLS ciphers suites + TLSCipherSuites []string `json:"tlsCipherSuites,omitempty" flag:"tls-cipher-suites"` // TODO: Remove unused TokenAuthFile TokenAuthFile string `json:"tokenAuthFile,omitempty" flag:"token-auth-file"` // AllowPrivileged indicates if we can run privileged containers @@ -358,13 +375,18 @@ type KubeAPIServerConfig struct { // EtcdQuorumRead configures the etcd-quorum-read flag, which forces consistent reads from etcd EtcdQuorumRead *bool `json:"etcdQuorumRead,omitempty" flag:"etcd-quorum-read"` - // MinRequestTimeout configures the minimum number of seconds a handler must keep a request open before timing it out. // Currently only honored by the watch request handler MinRequestTimeout *int32 `json:"minRequestTimeout,omitempty" flag:"min-request-timeout"` // Memory limit for apiserver in MB (used to configure sizes of caches, etc.) TargetRamMb int32 `json:"targetRamMb,omitempty" flag:"target-ram-mb" flag-empty:"0"` + // Enable profiling via web interface. + Profiling *bool `json:"profiling,omitempty" flag:"profiling"` + // Verify service account token + ServiceAccountLookup *bool `json:"serviceAccountLookup,omitempty" flag:"service-account-lookup"` + // Repair malformed requests from clients + RepairMalformedUpdates *bool `json:"repairMalformedUpdates,omitempty" flag:"repair-malformed-updates"` } // KubeControllerManagerConfig is the configuration for the controller @@ -430,6 +452,8 @@ type KubeControllerManagerConfig struct { HorizontalPodAutoscalerUseRestClients *bool `json:"horizontalPodAutoscalerUseRestClients,omitempty" flag:"horizontal-pod-autoscaler-use-rest-clients"` // FeatureGates is set of key=value pairs that describe feature gates for alpha/experimental features. FeatureGates map[string]string `json:"featureGates,omitempty" flag:"feature-gates"` + // Enable profiling via web interface. + Profiling *bool `json:"profiling,omitempty" flag:"profiling"` } // CloudControllerManagerConfig is the configuration of the cloud controller @@ -467,6 +491,8 @@ type KubeSchedulerConfig struct { LogLevel int32 `json:"logLevel,omitempty" flag:"v"` // Image is the docker image to use Image string `json:"image,omitempty"` + // Enable profiling via web interface + Profiling bool `json:"profiling,omitempty" flag:"profiling"` // LeaderElection defines the configuration of leader election client. LeaderElection *LeaderElectionConfiguration `json:"leaderElection,omitempty"` // UsePolicyConfigMap enable setting the scheduler policy from a configmap diff --git a/pkg/apis/kops/v1alpha2/instancegroup.go b/pkg/apis/kops/v1alpha2/instancegroup.go index d6c351713671c..773196b3d7a10 100644 --- a/pkg/apis/kops/v1alpha2/instancegroup.go +++ b/pkg/apis/kops/v1alpha2/instancegroup.go @@ -111,7 +111,7 @@ type InstanceGroupSpec struct { DetailedInstanceMonitoring *bool `json:"detailedInstanceMonitoring,omitempty"` // IAMProfileSpec defines the identity of the cloud group iam profile (AWS only). IAM *IAMProfileSpec `json:"iam,omitempty"` - // SecurityGroupOverride overrides the defaut security group created by Kops for this IG (AWS only). + // SecurityGroupOverride overrides the default security group created by Kops for this IG (AWS only). SecurityGroupOverride *string `json:"securityGroupOverride,omitempty"` } diff --git a/pkg/apis/kops/v1alpha2/networking.go b/pkg/apis/kops/v1alpha2/networking.go index b6c60ffe25588..5b952f3b96f58 100644 --- a/pkg/apis/kops/v1alpha2/networking.go +++ b/pkg/apis/kops/v1alpha2/networking.go @@ -31,6 +31,7 @@ type NetworkingSpec struct { Romana *RomanaNetworkingSpec `json:"romana,omitempty"` AmazonVPC *AmazonVPCNetworkingSpec `json:"amazonvpc,omitempty"` Cilium *CiliumNetworkingSpec `json:"cilium,omitempty"` + LyftVPC *LyftVPCNetworkingSpec `json:"lyftvpc,omitempty"` } // ClassicNetworkingSpec is the specification of classic networking mode, integrated into kubernetes @@ -87,6 +88,8 @@ type CalicoNetworkingSpec struct { PrometheusGoMetricsEnabled bool `json:"prometheusGoMetricsEnabled,omitempty"` // PrometheusProcessMetricsEnabled enables Prometheus process metrics collection PrometheusProcessMetricsEnabled bool `json:"prometheusProcessMetricsEnabled,omitempty"` + // MajorVersion is the version of Calico to use + MajorVersion string `json:"majorVersion,omitempty"` } // CanalNetworkingSpec declares that we want Canal networking @@ -186,3 +189,8 @@ type CiliumNetworkingSpec struct { TracePayloadLen int `json:"tracePayloadlen,omitempty"` Tunnel string `json:"tunnel,omitempty"` } + +// LyftIpVlanNetworkingSpec declares that we want to use the cni-ipvlan-vpc-k8s CNI networking +type LyftVPCNetworkingSpec struct { + SubnetTags map[string]string `json:"subnetTags,omitempty"` +} diff --git a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go index 1afc069620d4b..1cf7cba72d027 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go @@ -1,7 +1,7 @@ // +build !ignore_autogenerated /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -21,6 +21,7 @@ limitations under the License. package v1alpha2 import ( + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" conversion "k8s.io/apimachinery/pkg/conversion" runtime "k8s.io/apimachinery/pkg/runtime" kops "k8s.io/kops/pkg/apis/kops" @@ -148,6 +149,8 @@ func RegisterConversions(scheme *runtime.Scheme) error { Convert_kops_LoadBalancer_To_v1alpha2_LoadBalancer, Convert_v1alpha2_LoadBalancerAccessSpec_To_kops_LoadBalancerAccessSpec, Convert_kops_LoadBalancerAccessSpec_To_v1alpha2_LoadBalancerAccessSpec, + Convert_v1alpha2_LyftVPCNetworkingSpec_To_kops_LyftVPCNetworkingSpec, + Convert_kops_LyftVPCNetworkingSpec_To_v1alpha2_LyftVPCNetworkingSpec, Convert_v1alpha2_NetworkingSpec_To_kops_NetworkingSpec, Convert_kops_NetworkingSpec_To_v1alpha2_NetworkingSpec, Convert_v1alpha2_NodeAuthorizationSpec_To_kops_NodeAuthorizationSpec, @@ -489,6 +492,7 @@ func autoConvert_v1alpha2_CalicoNetworkingSpec_To_kops_CalicoNetworkingSpec(in * out.PrometheusMetricsPort = in.PrometheusMetricsPort out.PrometheusGoMetricsEnabled = in.PrometheusGoMetricsEnabled out.PrometheusProcessMetricsEnabled = in.PrometheusProcessMetricsEnabled + out.MajorVersion = in.MajorVersion return nil } @@ -505,6 +509,7 @@ func autoConvert_kops_CalicoNetworkingSpec_To_v1alpha2_CalicoNetworkingSpec(in * out.PrometheusMetricsPort = in.PrometheusMetricsPort out.PrometheusGoMetricsEnabled = in.PrometheusGoMetricsEnabled out.PrometheusProcessMetricsEnabled = in.PrometheusProcessMetricsEnabled + out.MajorVersion = in.MajorVersion return nil } @@ -1113,6 +1118,7 @@ func autoConvert_v1alpha2_ClusterSpec_To_kops_ClusterSpec(in *ClusterSpec, out * out.IAM = nil } out.EncryptionConfig = in.EncryptionConfig + out.DisableSubnetTags = in.DisableSubnetTags if in.Target != nil { in, out := &in.Target, &out.Target *out = new(kops.TargetSpec) @@ -1394,6 +1400,7 @@ func autoConvert_kops_ClusterSpec_To_v1alpha2_ClusterSpec(in *kops.ClusterSpec, out.IAM = nil } out.EncryptionConfig = in.EncryptionConfig + out.DisableSubnetTags = in.DisableSubnetTags if in.Target != nil { in, out := &in.Target, &out.Target *out = new(TargetSpec) @@ -2427,6 +2434,8 @@ func Convert_kops_KopeioNetworkingSpec_To_v1alpha2_KopeioNetworkingSpec(in *kops func autoConvert_v1alpha2_KubeAPIServerConfig_To_kops_KubeAPIServerConfig(in *KubeAPIServerConfig, out *kops.KubeAPIServerConfig, s conversion.Scope) error { out.Image = in.Image + out.DisableBasicAuth = in.DisableBasicAuth + out.DisableTokenAuth = in.DisableTokenAuth out.LogLevel = in.LogLevel out.CloudProvider = in.CloudProvider out.SecurePort = in.SecurePort @@ -2450,6 +2459,7 @@ func autoConvert_v1alpha2_KubeAPIServerConfig_To_kops_KubeAPIServerConfig(in *Ku out.ClientCAFile = in.ClientCAFile out.TLSCertFile = in.TLSCertFile out.TLSPrivateKeyFile = in.TLSPrivateKeyFile + out.TLSCipherSuites = in.TLSCipherSuites out.TokenAuthFile = in.TokenAuthFile out.AllowPrivileged = in.AllowPrivileged out.APIServerCount = in.APIServerCount @@ -2490,6 +2500,9 @@ func autoConvert_v1alpha2_KubeAPIServerConfig_To_kops_KubeAPIServerConfig(in *Ku out.EtcdQuorumRead = in.EtcdQuorumRead out.MinRequestTimeout = in.MinRequestTimeout out.TargetRamMb = in.TargetRamMb + out.Profiling = in.Profiling + out.ServiceAccountLookup = in.ServiceAccountLookup + out.RepairMalformedUpdates = in.RepairMalformedUpdates return nil } @@ -2500,6 +2513,8 @@ func Convert_v1alpha2_KubeAPIServerConfig_To_kops_KubeAPIServerConfig(in *KubeAP func autoConvert_kops_KubeAPIServerConfig_To_v1alpha2_KubeAPIServerConfig(in *kops.KubeAPIServerConfig, out *KubeAPIServerConfig, s conversion.Scope) error { out.Image = in.Image + out.DisableBasicAuth = in.DisableBasicAuth + out.DisableTokenAuth = in.DisableTokenAuth out.LogLevel = in.LogLevel out.CloudProvider = in.CloudProvider out.SecurePort = in.SecurePort @@ -2523,6 +2538,7 @@ func autoConvert_kops_KubeAPIServerConfig_To_v1alpha2_KubeAPIServerConfig(in *ko out.ClientCAFile = in.ClientCAFile out.TLSCertFile = in.TLSCertFile out.TLSPrivateKeyFile = in.TLSPrivateKeyFile + out.TLSCipherSuites = in.TLSCipherSuites out.TokenAuthFile = in.TokenAuthFile out.AllowPrivileged = in.AllowPrivileged out.APIServerCount = in.APIServerCount @@ -2563,6 +2579,9 @@ func autoConvert_kops_KubeAPIServerConfig_To_v1alpha2_KubeAPIServerConfig(in *ko out.EtcdQuorumRead = in.EtcdQuorumRead out.MinRequestTimeout = in.MinRequestTimeout out.TargetRamMb = in.TargetRamMb + out.Profiling = in.Profiling + out.ServiceAccountLookup = in.ServiceAccountLookup + out.RepairMalformedUpdates = in.RepairMalformedUpdates return nil } @@ -2605,6 +2624,7 @@ func autoConvert_v1alpha2_KubeControllerManagerConfig_To_kops_KubeControllerMana out.HorizontalPodAutoscalerUpscaleDelay = in.HorizontalPodAutoscalerUpscaleDelay out.HorizontalPodAutoscalerUseRestClients = in.HorizontalPodAutoscalerUseRestClients out.FeatureGates = in.FeatureGates + out.Profiling = in.Profiling return nil } @@ -2647,6 +2667,7 @@ func autoConvert_kops_KubeControllerManagerConfig_To_v1alpha2_KubeControllerMana out.HorizontalPodAutoscalerUpscaleDelay = in.HorizontalPodAutoscalerUpscaleDelay out.HorizontalPodAutoscalerUseRestClients = in.HorizontalPodAutoscalerUseRestClients out.FeatureGates = in.FeatureGates + out.Profiling = in.Profiling return nil } @@ -2751,6 +2772,9 @@ func autoConvert_v1alpha2_KubeSchedulerConfig_To_kops_KubeSchedulerConfig(in *Ku out.Master = in.Master out.LogLevel = in.LogLevel out.Image = in.Image + if err := v1.Convert_bool_To_Pointer_bool(&in.Profiling, &out.Profiling, s); err != nil { + return err + } if in.LeaderElection != nil { in, out := &in.LeaderElection, &out.LeaderElection *out = new(kops.LeaderElectionConfiguration) @@ -2774,6 +2798,9 @@ func autoConvert_kops_KubeSchedulerConfig_To_v1alpha2_KubeSchedulerConfig(in *ko out.Master = in.Master out.LogLevel = in.LogLevel out.Image = in.Image + if err := v1.Convert_Pointer_bool_To_bool(&in.Profiling, &out.Profiling, s); err != nil { + return err + } if in.LeaderElection != nil { in, out := &in.LeaderElection, &out.LeaderElection *out = new(LeaderElectionConfiguration) @@ -2856,11 +2883,16 @@ func autoConvert_v1alpha2_KubeletConfigSpec_To_kops_KubeletConfigSpec(in *Kubele out.VolumeStatsAggPeriod = in.VolumeStatsAggPeriod out.FailSwapOn = in.FailSwapOn out.ExperimentalAllowedUnsafeSysctls = in.ExperimentalAllowedUnsafeSysctls + out.AllowedUnsafeSysctls = in.AllowedUnsafeSysctls out.StreamingConnectionIdleTimeout = in.StreamingConnectionIdleTimeout out.DockerDisableSharedPID = in.DockerDisableSharedPID out.RootDir = in.RootDir out.AuthenticationTokenWebhook = in.AuthenticationTokenWebhook out.AuthenticationTokenWebhookCacheTTL = in.AuthenticationTokenWebhookCacheTTL + out.EventQPS = in.EventQPS + out.MakeIptablesUtilChains = in.MakeIptablesUtilChains + out.CAdvisorPort = in.CAdvisorPort + out.ProtectKernelDefaults = in.ProtectKernelDefaults return nil } @@ -2932,11 +2964,16 @@ func autoConvert_kops_KubeletConfigSpec_To_v1alpha2_KubeletConfigSpec(in *kops.K out.VolumeStatsAggPeriod = in.VolumeStatsAggPeriod out.FailSwapOn = in.FailSwapOn out.ExperimentalAllowedUnsafeSysctls = in.ExperimentalAllowedUnsafeSysctls + out.AllowedUnsafeSysctls = in.AllowedUnsafeSysctls out.StreamingConnectionIdleTimeout = in.StreamingConnectionIdleTimeout out.DockerDisableSharedPID = in.DockerDisableSharedPID out.RootDir = in.RootDir out.AuthenticationTokenWebhook = in.AuthenticationTokenWebhook out.AuthenticationTokenWebhookCacheTTL = in.AuthenticationTokenWebhookCacheTTL + out.EventQPS = in.EventQPS + out.MakeIptablesUtilChains = in.MakeIptablesUtilChains + out.CAdvisorPort = in.CAdvisorPort + out.ProtectKernelDefaults = in.ProtectKernelDefaults return nil } @@ -3053,6 +3090,26 @@ func Convert_kops_LoadBalancerAccessSpec_To_v1alpha2_LoadBalancerAccessSpec(in * return autoConvert_kops_LoadBalancerAccessSpec_To_v1alpha2_LoadBalancerAccessSpec(in, out, s) } +func autoConvert_v1alpha2_LyftVPCNetworkingSpec_To_kops_LyftVPCNetworkingSpec(in *LyftVPCNetworkingSpec, out *kops.LyftVPCNetworkingSpec, s conversion.Scope) error { + out.SubnetTags = in.SubnetTags + return nil +} + +// Convert_v1alpha2_LyftVPCNetworkingSpec_To_kops_LyftVPCNetworkingSpec is an autogenerated conversion function. +func Convert_v1alpha2_LyftVPCNetworkingSpec_To_kops_LyftVPCNetworkingSpec(in *LyftVPCNetworkingSpec, out *kops.LyftVPCNetworkingSpec, s conversion.Scope) error { + return autoConvert_v1alpha2_LyftVPCNetworkingSpec_To_kops_LyftVPCNetworkingSpec(in, out, s) +} + +func autoConvert_kops_LyftVPCNetworkingSpec_To_v1alpha2_LyftVPCNetworkingSpec(in *kops.LyftVPCNetworkingSpec, out *LyftVPCNetworkingSpec, s conversion.Scope) error { + out.SubnetTags = in.SubnetTags + return nil +} + +// Convert_kops_LyftVPCNetworkingSpec_To_v1alpha2_LyftVPCNetworkingSpec is an autogenerated conversion function. +func Convert_kops_LyftVPCNetworkingSpec_To_v1alpha2_LyftVPCNetworkingSpec(in *kops.LyftVPCNetworkingSpec, out *LyftVPCNetworkingSpec, s conversion.Scope) error { + return autoConvert_kops_LyftVPCNetworkingSpec_To_v1alpha2_LyftVPCNetworkingSpec(in, out, s) +} + func autoConvert_v1alpha2_NetworkingSpec_To_kops_NetworkingSpec(in *NetworkingSpec, out *kops.NetworkingSpec, s conversion.Scope) error { if in.Classic != nil { in, out := &in.Classic, &out.Classic @@ -3171,6 +3228,15 @@ func autoConvert_v1alpha2_NetworkingSpec_To_kops_NetworkingSpec(in *NetworkingSp } else { out.Cilium = nil } + if in.LyftVPC != nil { + in, out := &in.LyftVPC, &out.LyftVPC + *out = new(kops.LyftVPCNetworkingSpec) + if err := Convert_v1alpha2_LyftVPCNetworkingSpec_To_kops_LyftVPCNetworkingSpec(*in, *out, s); err != nil { + return err + } + } else { + out.LyftVPC = nil + } return nil } @@ -3297,6 +3363,15 @@ func autoConvert_kops_NetworkingSpec_To_v1alpha2_NetworkingSpec(in *kops.Network } else { out.Cilium = nil } + if in.LyftVPC != nil { + in, out := &in.LyftVPC, &out.LyftVPC + *out = new(LyftVPCNetworkingSpec) + if err := Convert_kops_LyftVPCNetworkingSpec_To_v1alpha2_LyftVPCNetworkingSpec(*in, *out, s); err != nil { + return err + } + } else { + out.LyftVPC = nil + } return nil } diff --git a/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go b/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go index cc992a4278fac..1d093adcaacfc 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go @@ -1,7 +1,7 @@ // +build !ignore_autogenerated /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -1998,6 +1998,11 @@ func (in *KubeAPIServerConfig) DeepCopyInto(out *KubeAPIServerConfig) { *out = make([]string, len(*in)) copy(*out, *in) } + if in.TLSCipherSuites != nil { + in, out := &in.TLSCipherSuites, &out.TLSCipherSuites + *out = make([]string, len(*in)) + copy(*out, *in) + } if in.AllowPrivileged != nil { in, out := &in.AllowPrivileged, &out.AllowPrivileged if *in == nil { @@ -2262,6 +2267,33 @@ func (in *KubeAPIServerConfig) DeepCopyInto(out *KubeAPIServerConfig) { **out = **in } } + if in.Profiling != nil { + in, out := &in.Profiling, &out.Profiling + if *in == nil { + *out = nil + } else { + *out = new(bool) + **out = **in + } + } + if in.ServiceAccountLookup != nil { + in, out := &in.ServiceAccountLookup, &out.ServiceAccountLookup + if *in == nil { + *out = nil + } else { + *out = new(bool) + **out = **in + } + } + if in.RepairMalformedUpdates != nil { + in, out := &in.RepairMalformedUpdates, &out.RepairMalformedUpdates + if *in == nil { + *out = nil + } else { + *out = new(bool) + **out = **in + } + } return } @@ -2425,6 +2457,15 @@ func (in *KubeControllerManagerConfig) DeepCopyInto(out *KubeControllerManagerCo (*out)[key] = val } } + if in.Profiling != nil { + in, out := &in.Profiling, &out.Profiling + if *in == nil { + *out = nil + } else { + *out = new(bool) + **out = **in + } + } return } @@ -2869,6 +2910,11 @@ func (in *KubeletConfigSpec) DeepCopyInto(out *KubeletConfigSpec) { *out = make([]string, len(*in)) copy(*out, *in) } + if in.AllowedUnsafeSysctls != nil { + in, out := &in.AllowedUnsafeSysctls, &out.AllowedUnsafeSysctls + *out = make([]string, len(*in)) + copy(*out, *in) + } if in.StreamingConnectionIdleTimeout != nil { in, out := &in.StreamingConnectionIdleTimeout, &out.StreamingConnectionIdleTimeout if *in == nil { @@ -2905,6 +2951,42 @@ func (in *KubeletConfigSpec) DeepCopyInto(out *KubeletConfigSpec) { **out = **in } } + if in.EventQPS != nil { + in, out := &in.EventQPS, &out.EventQPS + if *in == nil { + *out = nil + } else { + *out = new(int32) + **out = **in + } + } + if in.MakeIptablesUtilChains != nil { + in, out := &in.MakeIptablesUtilChains, &out.MakeIptablesUtilChains + if *in == nil { + *out = nil + } else { + *out = new(bool) + **out = **in + } + } + if in.CAdvisorPort != nil { + in, out := &in.CAdvisorPort, &out.CAdvisorPort + if *in == nil { + *out = nil + } else { + *out = new(int32) + **out = **in + } + } + if in.ProtectKernelDefaults != nil { + in, out := &in.ProtectKernelDefaults, &out.ProtectKernelDefaults + if *in == nil { + *out = nil + } else { + *out = new(bool) + **out = **in + } + } return } @@ -3048,6 +3130,29 @@ func (in *LoadBalancerAccessSpec) DeepCopy() *LoadBalancerAccessSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LyftVPCNetworkingSpec) DeepCopyInto(out *LyftVPCNetworkingSpec) { + *out = *in + if in.SubnetTags != nil { + in, out := &in.SubnetTags, &out.SubnetTags + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LyftVPCNetworkingSpec. +func (in *LyftVPCNetworkingSpec) DeepCopy() *LyftVPCNetworkingSpec { + if in == nil { + return nil + } + out := new(LyftVPCNetworkingSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NetworkingSpec) DeepCopyInto(out *NetworkingSpec) { *out = *in @@ -3168,6 +3273,15 @@ func (in *NetworkingSpec) DeepCopyInto(out *NetworkingSpec) { (*in).DeepCopyInto(*out) } } + if in.LyftVPC != nil { + in, out := &in.LyftVPC, &out.LyftVPC + if *in == nil { + *out = nil + } else { + *out = new(LyftVPCNetworkingSpec) + (*in).DeepCopyInto(*out) + } + } return } diff --git a/pkg/apis/kops/v1alpha2/zz_generated.defaults.go b/pkg/apis/kops/v1alpha2/zz_generated.defaults.go index a8e7066afbbe1..83b43cbd4ecad 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.defaults.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.defaults.go @@ -1,7 +1,7 @@ // +build !ignore_autogenerated /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/apis/kops/validation/BUILD.bazel b/pkg/apis/kops/validation/BUILD.bazel index 4f5ca4939314f..f7b2f54d2c00b 100644 --- a/pkg/apis/kops/validation/BUILD.bazel +++ b/pkg/apis/kops/validation/BUILD.bazel @@ -19,6 +19,7 @@ go_library( "//pkg/featureflag:go_default_library", "//pkg/model/components:go_default_library", "//pkg/model/iam:go_default_library", + "//pkg/util/subnet:go_default_library", "//upup/pkg/fi:go_default_library", "//upup/pkg/fi/cloudup/awsup:go_default_library", "//vendor/github.com/blang/semver:go_default_library", @@ -35,7 +36,6 @@ go_test( name = "go_default_test", srcs = [ "aws_test.go", - "helpers_test.go", "instancegroup_test.go", "validation_test.go", ], diff --git a/pkg/apis/kops/validation/cluster.go b/pkg/apis/kops/validation/cluster.go index 44e36be697be1..5d2e1890b4d30 100644 --- a/pkg/apis/kops/validation/cluster.go +++ b/pkg/apis/kops/validation/cluster.go @@ -44,7 +44,9 @@ func ValidateClusterUpdate(obj *kops.Cluster, status *kops.ClusterStatus, old *k fp := field.NewPath("spec", "etcdClusters").Key(k) oldCluster := oldClusters[k] - allErrs = append(allErrs, validateEtcdClusterUpdate(fp, newCluster, status, oldCluster)...) + if oldCluster != nil { + allErrs = append(allErrs, validateEtcdClusterUpdate(fp, newCluster, status, oldCluster)...) + } } for k := range oldClusters { newCluster := newClusters[k] diff --git a/pkg/apis/kops/validation/helpers.go b/pkg/apis/kops/validation/helpers.go index 20da4d726a3c5..5931228b8690d 100644 --- a/pkg/apis/kops/validation/helpers.go +++ b/pkg/apis/kops/validation/helpers.go @@ -17,32 +17,11 @@ limitations under the License. package validation import ( - "net" "net/url" "k8s.io/apimachinery/pkg/util/validation/field" ) -// isSubnet checks if child is a subnet of parent -func isSubnet(parent *net.IPNet, child *net.IPNet) bool { - parentOnes, parentBits := parent.Mask.Size() - childOnes, childBits := child.Mask.Size() - if childBits != parentBits { - return false - } - if parentOnes > childOnes { - return false - } - childMasked := child.IP.Mask(parent.Mask) - parentMasked := parent.IP.Mask(parent.Mask) - return childMasked.Equal(parentMasked) -} - -// subnetsOverlap checks if two subnets overlap -func subnetsOverlap(l *net.IPNet, r *net.IPNet) bool { - return l.Contains(r.IP) || r.Contains(l.IP) -} - func isValidAPIServersURL(s string) bool { u, err := url.Parse(s) if err != nil { diff --git a/pkg/apis/kops/validation/helpers_test.go b/pkg/apis/kops/validation/helpers_test.go deleted file mode 100644 index c1a9130ebb3ce..0000000000000 --- a/pkg/apis/kops/validation/helpers_test.go +++ /dev/null @@ -1,75 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package validation - -import ( - "net" - "testing" -) - -func Test_isSubnet(t *testing.T) { - grid := []struct { - L string - R string - IsSubnet bool - }{ - { - L: "192.168.1.0/24", - R: "192.168.0.0/24", - IsSubnet: false, - }, - { - L: "192.168.0.0/16", - R: "192.168.0.0/24", - IsSubnet: true, - }, - { - L: "192.168.0.0/24", - R: "192.168.0.0/16", - IsSubnet: false, - }, - { - L: "192.168.0.0/16", - R: "192.168.0.0/16", - IsSubnet: true, // Not a strict subnet - }, - { - L: "192.168.0.1/16", - R: "192.168.0.0/24", - IsSubnet: true, - }, - { - L: "0.0.0.0/0", - R: "101.0.1.0/32", - IsSubnet: true, - }, - } - for _, g := range grid { - _, l, err := net.ParseCIDR(g.L) - if err != nil { - t.Fatalf("error parsing %q: %v", g.L, err) - } - _, r, err := net.ParseCIDR(g.R) - if err != nil { - t.Fatalf("error parsing %q: %v", g.R, err) - } - actual := isSubnet(l, r) - if actual != g.IsSubnet { - t.Errorf("isSubnet(%q, %q) = %v, expected %v", g.L, g.R, actual, g.IsSubnet) - } - } -} diff --git a/pkg/apis/kops/validation/legacy.go b/pkg/apis/kops/validation/legacy.go index 72195c5ff52f7..8b42543d96288 100644 --- a/pkg/apis/kops/validation/legacy.go +++ b/pkg/apis/kops/validation/legacy.go @@ -27,6 +27,7 @@ import ( "k8s.io/kops/pkg/apis/kops/util" "k8s.io/kops/pkg/featureflag" "k8s.io/kops/pkg/model/components" + "k8s.io/kops/pkg/util/subnet" "k8s.io/kops/upup/pkg/fi" "github.com/blang/semver" @@ -190,7 +191,8 @@ func ValidateCluster(c *kops.Cluster, strict bool) *field.Error { return field.Invalid(fieldSpec.Child("NonMasqueradeCIDR"), nonMasqueradeCIDRString, "Cluster had an invalid NonMasqueradeCIDR") } - if networkCIDR != nil && subnetsOverlap(nonMasqueradeCIDR, networkCIDR) && c.Spec.Networking != nil && c.Spec.Networking.AmazonVPC == nil { + if networkCIDR != nil && subnet.Overlap(nonMasqueradeCIDR, networkCIDR) && c.Spec.Networking != nil && c.Spec.Networking.AmazonVPC == nil && c.Spec.Networking.LyftVPC == nil { + return field.Invalid(fieldSpec.Child("NonMasqueradeCIDR"), nonMasqueradeCIDRString, fmt.Sprintf("NonMasqueradeCIDR %q cannot overlap with NetworkCIDR %q", nonMasqueradeCIDRString, c.Spec.NetworkCIDR)) } @@ -220,7 +222,7 @@ func ValidateCluster(c *kops.Cluster, strict bool) *field.Error { return field.Invalid(fieldSpec.Child("ServiceClusterIPRange"), serviceClusterIPRangeString, "Cluster had an invalid ServiceClusterIPRange") } - if !isSubnet(nonMasqueradeCIDR, serviceClusterIPRange) { + if !subnet.BelongsTo(nonMasqueradeCIDR, serviceClusterIPRange) { return field.Invalid(fieldSpec.Child("ServiceClusterIPRange"), serviceClusterIPRangeString, fmt.Sprintf("ServiceClusterIPRange %q must be a subnet of NonMasqueradeCIDR %q", serviceClusterIPRangeString, c.Spec.NonMasqueradeCIDR)) } @@ -266,7 +268,7 @@ func ValidateCluster(c *kops.Cluster, strict bool) *field.Error { return field.Invalid(fieldSpec.Child("KubeControllerManager", "ClusterCIDR"), clusterCIDRString, "Cluster had an invalid KubeControllerManager.ClusterCIDR") } - if !isSubnet(nonMasqueradeCIDR, clusterCIDR) { + if !subnet.BelongsTo(nonMasqueradeCIDR, clusterCIDR) { return field.Invalid(fieldSpec.Child("KubeControllerManager", "ClusterCIDR"), clusterCIDRString, fmt.Sprintf("KubeControllerManager.ClusterCIDR %q must be a subnet of NonMasqueradeCIDR %q", clusterCIDRString, c.Spec.NonMasqueradeCIDR)) } } @@ -596,8 +598,8 @@ func ValidateCluster(c *kops.Cluster, strict bool) *field.Error { } } - if c.Spec.Networking != nil && c.Spec.Networking.AmazonVPC != nil && - (c.Spec.CloudProvider != "aws") { + if c.Spec.Networking != nil && (c.Spec.Networking.AmazonVPC != nil || c.Spec.Networking.LyftVPC != nil) && + c.Spec.CloudProvider != "aws" { return field.Invalid(fieldSpec.Child("Networking"), "amazon-vpc-routed-eni", "amazon-vpc-routed-eni networking is supported only in AWS") } @@ -609,6 +611,10 @@ func ValidateCluster(c *kops.Cluster, strict bool) *field.Error { if c.Spec.Networking != nil && c.Spec.Networking.AmazonVPC != nil { return field.Invalid(fieldSpec.Child("Networking"), "amazon-vpc-routed-eni", "amazon-vpc-routed-eni networking is not supported with kubernetes versions 1.6 or lower") } + + if c.Spec.Networking != nil && c.Spec.Networking.LyftVPC != nil { + return field.Invalid(fieldSpec.Child("Networking"), "cni-ipvlan-vpc-k8s", "cni-ipvlan-vpc-k8s networking is not supported with kubernetes versions 1.6 or lower") + } } // Cilium specific validation rules @@ -625,12 +631,12 @@ func ValidateCluster(c *kops.Cluster, strict bool) *field.Error { // validateSubnetCIDR is responsible for validating subnets are part of the CIRDs assigned to the cluster. func validateSubnetCIDR(networkCIDR *net.IPNet, additionalNetworkCIDRs []*net.IPNet, subnetCIDR *net.IPNet) bool { - if isSubnet(networkCIDR, subnetCIDR) { + if subnet.BelongsTo(networkCIDR, subnetCIDR) { return true } for _, additionalNetworkCIDR := range additionalNetworkCIDRs { - if isSubnet(additionalNetworkCIDR, subnetCIDR) { + if subnet.BelongsTo(additionalNetworkCIDR, subnetCIDR) { return true } } diff --git a/pkg/apis/kops/validation/validation.go b/pkg/apis/kops/validation/validation.go index 68831907296cd..f7ff84a249728 100644 --- a/pkg/apis/kops/validation/validation.go +++ b/pkg/apis/kops/validation/validation.go @@ -21,11 +21,14 @@ import ( "net" "strings" + "github.com/blang/semver" + "k8s.io/apimachinery/pkg/api/validation" utilnet "k8s.io/apimachinery/pkg/util/net" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/validation/field" "k8s.io/kops/pkg/apis/kops" + "k8s.io/kops/pkg/model/components" "k8s.io/kops/pkg/model/iam" ) @@ -94,6 +97,9 @@ func validateClusterSpec(spec *kops.ClusterSpec, fieldPath *field.Path) field.Er if spec.Networking != nil { allErrs = append(allErrs, validateNetworking(spec.Networking, fieldPath.Child("networking"))...) + if spec.Networking.Calico != nil { + allErrs = append(allErrs, validateNetworkingCalico(spec.Networking.Calico, spec.EtcdClusters[0], fieldPath.Child("networking").Child("Calico"))...) + } } // IAM additionalPolicies @@ -341,3 +347,44 @@ func validateEtcdClusterSpec(spec *kops.EtcdClusterSpec, fieldPath *field.Path) return errs } + +func ValidateEtcdVersionForCalicoV3(e *kops.EtcdClusterSpec, majorVersion string, fldPath *field.Path) field.ErrorList { + allErrs := field.ErrorList{} + + version := e.Version + if e.Version == "" { + version = components.DefaultEtcd2Version + } + sem, err := semver.Parse(strings.TrimPrefix(version, "v")) + if err != nil { + allErrs = append(allErrs, field.InternalError(fldPath.Child("MajorVersion"), fmt.Errorf("Failed to parse Etcd version to check compatibility: %s", err))) + } + + if sem.Major != 3 { + if e.Version == "" { + allErrs = append(allErrs, + field.Invalid(fldPath.Child("MajorVersion"), majorVersion, + fmt.Sprintf("Unable to use v3 when ETCD version for %s cluster is default(%s)", + e.Name, components.DefaultEtcd2Version))) + } else { + allErrs = append(allErrs, + field.Invalid(fldPath.Child("MajorVersion"), majorVersion, + fmt.Sprintf("Unable to use v3 when ETCD version for %s cluster is %s", e.Name, e.Version))) + } + } + return allErrs +} + +func validateNetworkingCalico(v *kops.CalicoNetworkingSpec, e *kops.EtcdClusterSpec, fldPath *field.Path) field.ErrorList { + allErrs := field.ErrorList{} + switch v.MajorVersion { + case "": + // OK: + case "v3": + allErrs = append(allErrs, ValidateEtcdVersionForCalicoV3(e, v.MajorVersion, fldPath)...) + default: + allErrs = append(allErrs, field.NotSupported(fldPath.Child("MajorVersion"), v.MajorVersion, []string{"v3"})) + } + + return allErrs +} diff --git a/pkg/apis/kops/validation/validation_test.go b/pkg/apis/kops/validation/validation_test.go index 83dafb04ed2d9..c3cc2c31b0670 100644 --- a/pkg/apis/kops/validation/validation_test.go +++ b/pkg/apis/kops/validation/validation_test.go @@ -291,3 +291,47 @@ func Test_Validate_AdditionalPolicies(t *testing.T) { testErrors(t, g.Input, errs, g.ExpectedErrors) } } + +type caliInput struct { + Calico *kops.CalicoNetworkingSpec + Etcd *kops.EtcdClusterSpec +} + +func Test_Validate_Calico(t *testing.T) { + grid := []struct { + Input caliInput + ExpectedErrors []string + }{ + { + Input: caliInput{ + Calico: &kops.CalicoNetworkingSpec{}, + Etcd: &kops.EtcdClusterSpec{}, + }, + }, + { + Input: caliInput{ + Calico: &kops.CalicoNetworkingSpec{ + MajorVersion: "v3", + }, + Etcd: &kops.EtcdClusterSpec{ + Version: "3.2.18", + }, + }, + }, + { + Input: caliInput{ + Calico: &kops.CalicoNetworkingSpec{ + MajorVersion: "v3", + }, + Etcd: &kops.EtcdClusterSpec{ + Version: "2.2.18", + }, + }, + ExpectedErrors: []string{"Invalid value::Calico.MajorVersion"}, + }, + } + for _, g := range grid { + errs := validateNetworkingCalico(g.Input.Calico, g.Input.Etcd, field.NewPath("Calico")) + testErrors(t, g.Input, errs, g.ExpectedErrors) + } +} diff --git a/pkg/apis/kops/zz_generated.deepcopy.go b/pkg/apis/kops/zz_generated.deepcopy.go index f35bd86fd2d3f..1653c7f9ff2b4 100644 --- a/pkg/apis/kops/zz_generated.deepcopy.go +++ b/pkg/apis/kops/zz_generated.deepcopy.go @@ -1,7 +1,7 @@ // +build !ignore_autogenerated /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -2186,6 +2186,11 @@ func (in *KubeAPIServerConfig) DeepCopyInto(out *KubeAPIServerConfig) { *out = make([]string, len(*in)) copy(*out, *in) } + if in.TLSCipherSuites != nil { + in, out := &in.TLSCipherSuites, &out.TLSCipherSuites + *out = make([]string, len(*in)) + copy(*out, *in) + } if in.AllowPrivileged != nil { in, out := &in.AllowPrivileged, &out.AllowPrivileged if *in == nil { @@ -2450,6 +2455,33 @@ func (in *KubeAPIServerConfig) DeepCopyInto(out *KubeAPIServerConfig) { **out = **in } } + if in.Profiling != nil { + in, out := &in.Profiling, &out.Profiling + if *in == nil { + *out = nil + } else { + *out = new(bool) + **out = **in + } + } + if in.ServiceAccountLookup != nil { + in, out := &in.ServiceAccountLookup, &out.ServiceAccountLookup + if *in == nil { + *out = nil + } else { + *out = new(bool) + **out = **in + } + } + if in.RepairMalformedUpdates != nil { + in, out := &in.RepairMalformedUpdates, &out.RepairMalformedUpdates + if *in == nil { + *out = nil + } else { + *out = new(bool) + **out = **in + } + } return } @@ -2613,6 +2645,15 @@ func (in *KubeControllerManagerConfig) DeepCopyInto(out *KubeControllerManagerCo (*out)[key] = val } } + if in.Profiling != nil { + in, out := &in.Profiling, &out.Profiling + if *in == nil { + *out = nil + } else { + *out = new(bool) + **out = **in + } + } return } @@ -2744,6 +2785,15 @@ func (in *KubeProxyConfig) DeepCopy() *KubeProxyConfig { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *KubeSchedulerConfig) DeepCopyInto(out *KubeSchedulerConfig) { *out = *in + if in.Profiling != nil { + in, out := &in.Profiling, &out.Profiling + if *in == nil { + *out = nil + } else { + *out = new(bool) + **out = **in + } + } if in.LeaderElection != nil { in, out := &in.LeaderElection, &out.LeaderElection if *in == nil { @@ -3057,6 +3107,11 @@ func (in *KubeletConfigSpec) DeepCopyInto(out *KubeletConfigSpec) { *out = make([]string, len(*in)) copy(*out, *in) } + if in.AllowedUnsafeSysctls != nil { + in, out := &in.AllowedUnsafeSysctls, &out.AllowedUnsafeSysctls + *out = make([]string, len(*in)) + copy(*out, *in) + } if in.StreamingConnectionIdleTimeout != nil { in, out := &in.StreamingConnectionIdleTimeout, &out.StreamingConnectionIdleTimeout if *in == nil { @@ -3093,6 +3148,42 @@ func (in *KubeletConfigSpec) DeepCopyInto(out *KubeletConfigSpec) { **out = **in } } + if in.EventQPS != nil { + in, out := &in.EventQPS, &out.EventQPS + if *in == nil { + *out = nil + } else { + *out = new(int32) + **out = **in + } + } + if in.MakeIptablesUtilChains != nil { + in, out := &in.MakeIptablesUtilChains, &out.MakeIptablesUtilChains + if *in == nil { + *out = nil + } else { + *out = new(bool) + **out = **in + } + } + if in.CAdvisorPort != nil { + in, out := &in.CAdvisorPort, &out.CAdvisorPort + if *in == nil { + *out = nil + } else { + *out = new(int32) + **out = **in + } + } + if in.ProtectKernelDefaults != nil { + in, out := &in.ProtectKernelDefaults, &out.ProtectKernelDefaults + if *in == nil { + *out = nil + } else { + *out = new(bool) + **out = **in + } + } return } @@ -3252,6 +3343,29 @@ func (in *LoadBalancerAccessSpec) DeepCopy() *LoadBalancerAccessSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LyftVPCNetworkingSpec) DeepCopyInto(out *LyftVPCNetworkingSpec) { + *out = *in + if in.SubnetTags != nil { + in, out := &in.SubnetTags, &out.SubnetTags + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LyftVPCNetworkingSpec. +func (in *LyftVPCNetworkingSpec) DeepCopy() *LyftVPCNetworkingSpec { + if in == nil { + return nil + } + out := new(LyftVPCNetworkingSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NetworkingSpec) DeepCopyInto(out *NetworkingSpec) { *out = *in @@ -3372,6 +3486,15 @@ func (in *NetworkingSpec) DeepCopyInto(out *NetworkingSpec) { (*in).DeepCopyInto(*out) } } + if in.LyftVPC != nil { + in, out := &in.LyftVPC, &out.LyftVPC + if *in == nil { + *out = nil + } else { + *out = new(LyftVPCNetworkingSpec) + (*in).DeepCopyInto(*out) + } + } return } diff --git a/pkg/client/clientset_generated/clientset/clientset.go b/pkg/client/clientset_generated/clientset/clientset.go index 9575b87a0647c..594fa0386d560 100644 --- a/pkg/client/clientset_generated/clientset/clientset.go +++ b/pkg/client/clientset_generated/clientset/clientset.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/client/clientset_generated/clientset/doc.go b/pkg/client/clientset_generated/clientset/doc.go index 3421911a75a9a..42d06de7ad3a9 100644 --- a/pkg/client/clientset_generated/clientset/doc.go +++ b/pkg/client/clientset_generated/clientset/doc.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/client/clientset_generated/clientset/fake/clientset_generated.go b/pkg/client/clientset_generated/clientset/fake/clientset_generated.go index 3db2008b01eff..e1546c9406342 100644 --- a/pkg/client/clientset_generated/clientset/fake/clientset_generated.go +++ b/pkg/client/clientset_generated/clientset/fake/clientset_generated.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/client/clientset_generated/clientset/fake/doc.go b/pkg/client/clientset_generated/clientset/fake/doc.go index 0bc260bcaa22a..acfa6173b9e8e 100644 --- a/pkg/client/clientset_generated/clientset/fake/doc.go +++ b/pkg/client/clientset_generated/clientset/fake/doc.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/client/clientset_generated/clientset/fake/register.go b/pkg/client/clientset_generated/clientset/fake/register.go index fa7c7f84d9324..248982e49adca 100644 --- a/pkg/client/clientset_generated/clientset/fake/register.go +++ b/pkg/client/clientset_generated/clientset/fake/register.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/client/clientset_generated/clientset/scheme/doc.go b/pkg/client/clientset_generated/clientset/scheme/doc.go index 5c5c8debb6b4f..7f61dc1f9f6ec 100644 --- a/pkg/client/clientset_generated/clientset/scheme/doc.go +++ b/pkg/client/clientset_generated/clientset/scheme/doc.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/client/clientset_generated/clientset/scheme/register.go b/pkg/client/clientset_generated/clientset/scheme/register.go index 969dae2f7f033..ae73019ebd578 100644 --- a/pkg/client/clientset_generated/clientset/scheme/register.go +++ b/pkg/client/clientset_generated/clientset/scheme/register.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/client/clientset_generated/clientset/typed/kops/internalversion/cluster.go b/pkg/client/clientset_generated/clientset/typed/kops/internalversion/cluster.go index 0d70898e7b99a..3a565b1bf2188 100644 --- a/pkg/client/clientset_generated/clientset/typed/kops/internalversion/cluster.go +++ b/pkg/client/clientset_generated/clientset/typed/kops/internalversion/cluster.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/client/clientset_generated/clientset/typed/kops/internalversion/doc.go b/pkg/client/clientset_generated/clientset/typed/kops/internalversion/doc.go index b057e52013085..e168891358392 100644 --- a/pkg/client/clientset_generated/clientset/typed/kops/internalversion/doc.go +++ b/pkg/client/clientset_generated/clientset/typed/kops/internalversion/doc.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/client/clientset_generated/clientset/typed/kops/internalversion/fake/doc.go b/pkg/client/clientset_generated/clientset/typed/kops/internalversion/fake/doc.go index 87a1873edccb3..ab4fd43ad6e01 100644 --- a/pkg/client/clientset_generated/clientset/typed/kops/internalversion/fake/doc.go +++ b/pkg/client/clientset_generated/clientset/typed/kops/internalversion/fake/doc.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/client/clientset_generated/clientset/typed/kops/internalversion/fake/fake_cluster.go b/pkg/client/clientset_generated/clientset/typed/kops/internalversion/fake/fake_cluster.go index a746be11a653b..e2caab37e29ff 100644 --- a/pkg/client/clientset_generated/clientset/typed/kops/internalversion/fake/fake_cluster.go +++ b/pkg/client/clientset_generated/clientset/typed/kops/internalversion/fake/fake_cluster.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/client/clientset_generated/clientset/typed/kops/internalversion/fake/fake_instancegroup.go b/pkg/client/clientset_generated/clientset/typed/kops/internalversion/fake/fake_instancegroup.go index 6647d8c828c67..e219d03b2723f 100644 --- a/pkg/client/clientset_generated/clientset/typed/kops/internalversion/fake/fake_instancegroup.go +++ b/pkg/client/clientset_generated/clientset/typed/kops/internalversion/fake/fake_instancegroup.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/client/clientset_generated/clientset/typed/kops/internalversion/fake/fake_keyset.go b/pkg/client/clientset_generated/clientset/typed/kops/internalversion/fake/fake_keyset.go index 0aeedde4a3743..7a514572fc6ae 100644 --- a/pkg/client/clientset_generated/clientset/typed/kops/internalversion/fake/fake_keyset.go +++ b/pkg/client/clientset_generated/clientset/typed/kops/internalversion/fake/fake_keyset.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/client/clientset_generated/clientset/typed/kops/internalversion/fake/fake_kops_client.go b/pkg/client/clientset_generated/clientset/typed/kops/internalversion/fake/fake_kops_client.go index 9b67dbac25ab0..0da4252affd49 100644 --- a/pkg/client/clientset_generated/clientset/typed/kops/internalversion/fake/fake_kops_client.go +++ b/pkg/client/clientset_generated/clientset/typed/kops/internalversion/fake/fake_kops_client.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/client/clientset_generated/clientset/typed/kops/internalversion/fake/fake_sshcredential.go b/pkg/client/clientset_generated/clientset/typed/kops/internalversion/fake/fake_sshcredential.go index 0a6bb3149071c..9c89c1047493e 100644 --- a/pkg/client/clientset_generated/clientset/typed/kops/internalversion/fake/fake_sshcredential.go +++ b/pkg/client/clientset_generated/clientset/typed/kops/internalversion/fake/fake_sshcredential.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/client/clientset_generated/clientset/typed/kops/internalversion/generated_expansion.go b/pkg/client/clientset_generated/clientset/typed/kops/internalversion/generated_expansion.go index 41aba5b81adc6..f305c12cfbb3b 100644 --- a/pkg/client/clientset_generated/clientset/typed/kops/internalversion/generated_expansion.go +++ b/pkg/client/clientset_generated/clientset/typed/kops/internalversion/generated_expansion.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/client/clientset_generated/clientset/typed/kops/internalversion/instancegroup.go b/pkg/client/clientset_generated/clientset/typed/kops/internalversion/instancegroup.go index cd8e8468eb19d..d7c6568f5b103 100644 --- a/pkg/client/clientset_generated/clientset/typed/kops/internalversion/instancegroup.go +++ b/pkg/client/clientset_generated/clientset/typed/kops/internalversion/instancegroup.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/client/clientset_generated/clientset/typed/kops/internalversion/keyset.go b/pkg/client/clientset_generated/clientset/typed/kops/internalversion/keyset.go index dfa6de0843a89..0cd39a62e6e6c 100644 --- a/pkg/client/clientset_generated/clientset/typed/kops/internalversion/keyset.go +++ b/pkg/client/clientset_generated/clientset/typed/kops/internalversion/keyset.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/client/clientset_generated/clientset/typed/kops/internalversion/kops_client.go b/pkg/client/clientset_generated/clientset/typed/kops/internalversion/kops_client.go index da7f81f41ad6a..0d7dd465f21f3 100644 --- a/pkg/client/clientset_generated/clientset/typed/kops/internalversion/kops_client.go +++ b/pkg/client/clientset_generated/clientset/typed/kops/internalversion/kops_client.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/client/clientset_generated/clientset/typed/kops/internalversion/sshcredential.go b/pkg/client/clientset_generated/clientset/typed/kops/internalversion/sshcredential.go index 0657c4f30f4f5..ec520733c7edc 100644 --- a/pkg/client/clientset_generated/clientset/typed/kops/internalversion/sshcredential.go +++ b/pkg/client/clientset_generated/clientset/typed/kops/internalversion/sshcredential.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/client/clientset_generated/clientset/typed/kops/v1alpha1/cluster.go b/pkg/client/clientset_generated/clientset/typed/kops/v1alpha1/cluster.go index a0595c2e55e76..adbd851ba8010 100644 --- a/pkg/client/clientset_generated/clientset/typed/kops/v1alpha1/cluster.go +++ b/pkg/client/clientset_generated/clientset/typed/kops/v1alpha1/cluster.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/client/clientset_generated/clientset/typed/kops/v1alpha1/doc.go b/pkg/client/clientset_generated/clientset/typed/kops/v1alpha1/doc.go index 69ca30111b4cd..9752e759caa07 100644 --- a/pkg/client/clientset_generated/clientset/typed/kops/v1alpha1/doc.go +++ b/pkg/client/clientset_generated/clientset/typed/kops/v1alpha1/doc.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/client/clientset_generated/clientset/typed/kops/v1alpha1/fake/doc.go b/pkg/client/clientset_generated/clientset/typed/kops/v1alpha1/fake/doc.go index 87a1873edccb3..ab4fd43ad6e01 100644 --- a/pkg/client/clientset_generated/clientset/typed/kops/v1alpha1/fake/doc.go +++ b/pkg/client/clientset_generated/clientset/typed/kops/v1alpha1/fake/doc.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/client/clientset_generated/clientset/typed/kops/v1alpha1/fake/fake_cluster.go b/pkg/client/clientset_generated/clientset/typed/kops/v1alpha1/fake/fake_cluster.go index f01ad222b6dc8..2908735bef423 100644 --- a/pkg/client/clientset_generated/clientset/typed/kops/v1alpha1/fake/fake_cluster.go +++ b/pkg/client/clientset_generated/clientset/typed/kops/v1alpha1/fake/fake_cluster.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/client/clientset_generated/clientset/typed/kops/v1alpha1/fake/fake_instancegroup.go b/pkg/client/clientset_generated/clientset/typed/kops/v1alpha1/fake/fake_instancegroup.go index 9815a09d966f0..c575da471ada2 100644 --- a/pkg/client/clientset_generated/clientset/typed/kops/v1alpha1/fake/fake_instancegroup.go +++ b/pkg/client/clientset_generated/clientset/typed/kops/v1alpha1/fake/fake_instancegroup.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/client/clientset_generated/clientset/typed/kops/v1alpha1/fake/fake_kops_client.go b/pkg/client/clientset_generated/clientset/typed/kops/v1alpha1/fake/fake_kops_client.go index a90ba1813f5c2..d3d688686ac80 100644 --- a/pkg/client/clientset_generated/clientset/typed/kops/v1alpha1/fake/fake_kops_client.go +++ b/pkg/client/clientset_generated/clientset/typed/kops/v1alpha1/fake/fake_kops_client.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/client/clientset_generated/clientset/typed/kops/v1alpha1/fake/fake_sshcredential.go b/pkg/client/clientset_generated/clientset/typed/kops/v1alpha1/fake/fake_sshcredential.go index eb77602e270db..4508a54a25741 100644 --- a/pkg/client/clientset_generated/clientset/typed/kops/v1alpha1/fake/fake_sshcredential.go +++ b/pkg/client/clientset_generated/clientset/typed/kops/v1alpha1/fake/fake_sshcredential.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/client/clientset_generated/clientset/typed/kops/v1alpha1/generated_expansion.go b/pkg/client/clientset_generated/clientset/typed/kops/v1alpha1/generated_expansion.go index 79fabfc3a2ada..37793a8c40516 100644 --- a/pkg/client/clientset_generated/clientset/typed/kops/v1alpha1/generated_expansion.go +++ b/pkg/client/clientset_generated/clientset/typed/kops/v1alpha1/generated_expansion.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/client/clientset_generated/clientset/typed/kops/v1alpha1/instancegroup.go b/pkg/client/clientset_generated/clientset/typed/kops/v1alpha1/instancegroup.go index 7bed24d96d61b..fd2c8117f35b1 100644 --- a/pkg/client/clientset_generated/clientset/typed/kops/v1alpha1/instancegroup.go +++ b/pkg/client/clientset_generated/clientset/typed/kops/v1alpha1/instancegroup.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/client/clientset_generated/clientset/typed/kops/v1alpha1/kops_client.go b/pkg/client/clientset_generated/clientset/typed/kops/v1alpha1/kops_client.go index c04b76b2b776e..2cf72b4fcea5c 100644 --- a/pkg/client/clientset_generated/clientset/typed/kops/v1alpha1/kops_client.go +++ b/pkg/client/clientset_generated/clientset/typed/kops/v1alpha1/kops_client.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/client/clientset_generated/clientset/typed/kops/v1alpha1/sshcredential.go b/pkg/client/clientset_generated/clientset/typed/kops/v1alpha1/sshcredential.go index 084b8539f91b4..0efc5fac8b25d 100644 --- a/pkg/client/clientset_generated/clientset/typed/kops/v1alpha1/sshcredential.go +++ b/pkg/client/clientset_generated/clientset/typed/kops/v1alpha1/sshcredential.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/client/clientset_generated/clientset/typed/kops/v1alpha2/cluster.go b/pkg/client/clientset_generated/clientset/typed/kops/v1alpha2/cluster.go index 84b0ec2b5fe6a..c5a40f129d955 100644 --- a/pkg/client/clientset_generated/clientset/typed/kops/v1alpha2/cluster.go +++ b/pkg/client/clientset_generated/clientset/typed/kops/v1alpha2/cluster.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/client/clientset_generated/clientset/typed/kops/v1alpha2/doc.go b/pkg/client/clientset_generated/clientset/typed/kops/v1alpha2/doc.go index 01dd46de84d8d..e4cda4fe26734 100644 --- a/pkg/client/clientset_generated/clientset/typed/kops/v1alpha2/doc.go +++ b/pkg/client/clientset_generated/clientset/typed/kops/v1alpha2/doc.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/client/clientset_generated/clientset/typed/kops/v1alpha2/fake/doc.go b/pkg/client/clientset_generated/clientset/typed/kops/v1alpha2/fake/doc.go index 87a1873edccb3..ab4fd43ad6e01 100644 --- a/pkg/client/clientset_generated/clientset/typed/kops/v1alpha2/fake/doc.go +++ b/pkg/client/clientset_generated/clientset/typed/kops/v1alpha2/fake/doc.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/client/clientset_generated/clientset/typed/kops/v1alpha2/fake/fake_cluster.go b/pkg/client/clientset_generated/clientset/typed/kops/v1alpha2/fake/fake_cluster.go index 94a3dcffe6a08..db23945ceed65 100644 --- a/pkg/client/clientset_generated/clientset/typed/kops/v1alpha2/fake/fake_cluster.go +++ b/pkg/client/clientset_generated/clientset/typed/kops/v1alpha2/fake/fake_cluster.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/client/clientset_generated/clientset/typed/kops/v1alpha2/fake/fake_instancegroup.go b/pkg/client/clientset_generated/clientset/typed/kops/v1alpha2/fake/fake_instancegroup.go index f4cd2508e1a8a..e78cf043f6641 100644 --- a/pkg/client/clientset_generated/clientset/typed/kops/v1alpha2/fake/fake_instancegroup.go +++ b/pkg/client/clientset_generated/clientset/typed/kops/v1alpha2/fake/fake_instancegroup.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/client/clientset_generated/clientset/typed/kops/v1alpha2/fake/fake_keyset.go b/pkg/client/clientset_generated/clientset/typed/kops/v1alpha2/fake/fake_keyset.go index c3a2bba02993d..d6a5e767d7882 100644 --- a/pkg/client/clientset_generated/clientset/typed/kops/v1alpha2/fake/fake_keyset.go +++ b/pkg/client/clientset_generated/clientset/typed/kops/v1alpha2/fake/fake_keyset.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/client/clientset_generated/clientset/typed/kops/v1alpha2/fake/fake_kops_client.go b/pkg/client/clientset_generated/clientset/typed/kops/v1alpha2/fake/fake_kops_client.go index 564c3d183dc64..e1c8bc25d5588 100644 --- a/pkg/client/clientset_generated/clientset/typed/kops/v1alpha2/fake/fake_kops_client.go +++ b/pkg/client/clientset_generated/clientset/typed/kops/v1alpha2/fake/fake_kops_client.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/client/clientset_generated/clientset/typed/kops/v1alpha2/fake/fake_sshcredential.go b/pkg/client/clientset_generated/clientset/typed/kops/v1alpha2/fake/fake_sshcredential.go index eb505ade8d3e1..e15c83d6932b4 100644 --- a/pkg/client/clientset_generated/clientset/typed/kops/v1alpha2/fake/fake_sshcredential.go +++ b/pkg/client/clientset_generated/clientset/typed/kops/v1alpha2/fake/fake_sshcredential.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/client/clientset_generated/clientset/typed/kops/v1alpha2/generated_expansion.go b/pkg/client/clientset_generated/clientset/typed/kops/v1alpha2/generated_expansion.go index 9c61848846512..9b98449352673 100644 --- a/pkg/client/clientset_generated/clientset/typed/kops/v1alpha2/generated_expansion.go +++ b/pkg/client/clientset_generated/clientset/typed/kops/v1alpha2/generated_expansion.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/client/clientset_generated/clientset/typed/kops/v1alpha2/instancegroup.go b/pkg/client/clientset_generated/clientset/typed/kops/v1alpha2/instancegroup.go index eb2675ba21f23..1176424573a81 100644 --- a/pkg/client/clientset_generated/clientset/typed/kops/v1alpha2/instancegroup.go +++ b/pkg/client/clientset_generated/clientset/typed/kops/v1alpha2/instancegroup.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/client/clientset_generated/clientset/typed/kops/v1alpha2/keyset.go b/pkg/client/clientset_generated/clientset/typed/kops/v1alpha2/keyset.go index e6bcd4e5f9050..add0eb31415cc 100644 --- a/pkg/client/clientset_generated/clientset/typed/kops/v1alpha2/keyset.go +++ b/pkg/client/clientset_generated/clientset/typed/kops/v1alpha2/keyset.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/client/clientset_generated/clientset/typed/kops/v1alpha2/kops_client.go b/pkg/client/clientset_generated/clientset/typed/kops/v1alpha2/kops_client.go index a3d396341b4c0..a62b6a5a0a5fc 100644 --- a/pkg/client/clientset_generated/clientset/typed/kops/v1alpha2/kops_client.go +++ b/pkg/client/clientset_generated/clientset/typed/kops/v1alpha2/kops_client.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/client/clientset_generated/clientset/typed/kops/v1alpha2/sshcredential.go b/pkg/client/clientset_generated/clientset/typed/kops/v1alpha2/sshcredential.go index 00d427824b8da..026630b8e9e9e 100644 --- a/pkg/client/clientset_generated/clientset/typed/kops/v1alpha2/sshcredential.go +++ b/pkg/client/clientset_generated/clientset/typed/kops/v1alpha2/sshcredential.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/client/clientset_generated/internalclientset/clientset.go b/pkg/client/clientset_generated/internalclientset/clientset.go index b23d4f12b61ca..da7d7249213cd 100644 --- a/pkg/client/clientset_generated/internalclientset/clientset.go +++ b/pkg/client/clientset_generated/internalclientset/clientset.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/client/clientset_generated/internalclientset/doc.go b/pkg/client/clientset_generated/internalclientset/doc.go index 22ccee9196839..b48cc80c37a31 100644 --- a/pkg/client/clientset_generated/internalclientset/doc.go +++ b/pkg/client/clientset_generated/internalclientset/doc.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/client/clientset_generated/internalclientset/fake/clientset_generated.go b/pkg/client/clientset_generated/internalclientset/fake/clientset_generated.go index 78aefd7b88aed..a53091807ad71 100644 --- a/pkg/client/clientset_generated/internalclientset/fake/clientset_generated.go +++ b/pkg/client/clientset_generated/internalclientset/fake/clientset_generated.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/client/clientset_generated/internalclientset/fake/doc.go b/pkg/client/clientset_generated/internalclientset/fake/doc.go index 0bc260bcaa22a..acfa6173b9e8e 100644 --- a/pkg/client/clientset_generated/internalclientset/fake/doc.go +++ b/pkg/client/clientset_generated/internalclientset/fake/doc.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/client/clientset_generated/internalclientset/fake/register.go b/pkg/client/clientset_generated/internalclientset/fake/register.go index fa7c7f84d9324..248982e49adca 100644 --- a/pkg/client/clientset_generated/internalclientset/fake/register.go +++ b/pkg/client/clientset_generated/internalclientset/fake/register.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/client/clientset_generated/internalclientset/scheme/doc.go b/pkg/client/clientset_generated/internalclientset/scheme/doc.go index 5c5c8debb6b4f..7f61dc1f9f6ec 100644 --- a/pkg/client/clientset_generated/internalclientset/scheme/doc.go +++ b/pkg/client/clientset_generated/internalclientset/scheme/doc.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/client/clientset_generated/internalclientset/scheme/register.go b/pkg/client/clientset_generated/internalclientset/scheme/register.go index 969dae2f7f033..ae73019ebd578 100644 --- a/pkg/client/clientset_generated/internalclientset/scheme/register.go +++ b/pkg/client/clientset_generated/internalclientset/scheme/register.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/client/clientset_generated/internalclientset/typed/kops/internalversion/cluster.go b/pkg/client/clientset_generated/internalclientset/typed/kops/internalversion/cluster.go index e118ae90c4441..c14763494cbe0 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/kops/internalversion/cluster.go +++ b/pkg/client/clientset_generated/internalclientset/typed/kops/internalversion/cluster.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/client/clientset_generated/internalclientset/typed/kops/internalversion/doc.go b/pkg/client/clientset_generated/internalclientset/typed/kops/internalversion/doc.go index b057e52013085..e168891358392 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/kops/internalversion/doc.go +++ b/pkg/client/clientset_generated/internalclientset/typed/kops/internalversion/doc.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/client/clientset_generated/internalclientset/typed/kops/internalversion/fake/doc.go b/pkg/client/clientset_generated/internalclientset/typed/kops/internalversion/fake/doc.go index 87a1873edccb3..ab4fd43ad6e01 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/kops/internalversion/fake/doc.go +++ b/pkg/client/clientset_generated/internalclientset/typed/kops/internalversion/fake/doc.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/client/clientset_generated/internalclientset/typed/kops/internalversion/fake/fake_cluster.go b/pkg/client/clientset_generated/internalclientset/typed/kops/internalversion/fake/fake_cluster.go index a746be11a653b..e2caab37e29ff 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/kops/internalversion/fake/fake_cluster.go +++ b/pkg/client/clientset_generated/internalclientset/typed/kops/internalversion/fake/fake_cluster.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/client/clientset_generated/internalclientset/typed/kops/internalversion/fake/fake_instancegroup.go b/pkg/client/clientset_generated/internalclientset/typed/kops/internalversion/fake/fake_instancegroup.go index 6647d8c828c67..e219d03b2723f 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/kops/internalversion/fake/fake_instancegroup.go +++ b/pkg/client/clientset_generated/internalclientset/typed/kops/internalversion/fake/fake_instancegroup.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/client/clientset_generated/internalclientset/typed/kops/internalversion/fake/fake_keyset.go b/pkg/client/clientset_generated/internalclientset/typed/kops/internalversion/fake/fake_keyset.go index 0aeedde4a3743..7a514572fc6ae 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/kops/internalversion/fake/fake_keyset.go +++ b/pkg/client/clientset_generated/internalclientset/typed/kops/internalversion/fake/fake_keyset.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/client/clientset_generated/internalclientset/typed/kops/internalversion/fake/fake_kops_client.go b/pkg/client/clientset_generated/internalclientset/typed/kops/internalversion/fake/fake_kops_client.go index 307f3699a1bc6..37eec8de4e4ad 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/kops/internalversion/fake/fake_kops_client.go +++ b/pkg/client/clientset_generated/internalclientset/typed/kops/internalversion/fake/fake_kops_client.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/client/clientset_generated/internalclientset/typed/kops/internalversion/fake/fake_sshcredential.go b/pkg/client/clientset_generated/internalclientset/typed/kops/internalversion/fake/fake_sshcredential.go index 0a6bb3149071c..9c89c1047493e 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/kops/internalversion/fake/fake_sshcredential.go +++ b/pkg/client/clientset_generated/internalclientset/typed/kops/internalversion/fake/fake_sshcredential.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/client/clientset_generated/internalclientset/typed/kops/internalversion/generated_expansion.go b/pkg/client/clientset_generated/internalclientset/typed/kops/internalversion/generated_expansion.go index 41aba5b81adc6..f305c12cfbb3b 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/kops/internalversion/generated_expansion.go +++ b/pkg/client/clientset_generated/internalclientset/typed/kops/internalversion/generated_expansion.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/client/clientset_generated/internalclientset/typed/kops/internalversion/instancegroup.go b/pkg/client/clientset_generated/internalclientset/typed/kops/internalversion/instancegroup.go index ee251de1be94d..6f23a76af4c10 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/kops/internalversion/instancegroup.go +++ b/pkg/client/clientset_generated/internalclientset/typed/kops/internalversion/instancegroup.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/client/clientset_generated/internalclientset/typed/kops/internalversion/keyset.go b/pkg/client/clientset_generated/internalclientset/typed/kops/internalversion/keyset.go index 944a5ca542958..03341688b9f30 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/kops/internalversion/keyset.go +++ b/pkg/client/clientset_generated/internalclientset/typed/kops/internalversion/keyset.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/client/clientset_generated/internalclientset/typed/kops/internalversion/kops_client.go b/pkg/client/clientset_generated/internalclientset/typed/kops/internalversion/kops_client.go index 7988b7aeaa919..c7afe0f2c02fe 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/kops/internalversion/kops_client.go +++ b/pkg/client/clientset_generated/internalclientset/typed/kops/internalversion/kops_client.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/client/clientset_generated/internalclientset/typed/kops/internalversion/sshcredential.go b/pkg/client/clientset_generated/internalclientset/typed/kops/internalversion/sshcredential.go index 6f6fcdb08fb07..f5a95be0087a5 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/kops/internalversion/sshcredential.go +++ b/pkg/client/clientset_generated/internalclientset/typed/kops/internalversion/sshcredential.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha1/cluster.go b/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha1/cluster.go index a16d4ac7c84fb..de841f68a3b16 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha1/cluster.go +++ b/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha1/cluster.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha1/doc.go b/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha1/doc.go index 69ca30111b4cd..9752e759caa07 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha1/doc.go +++ b/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha1/doc.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha1/fake/doc.go b/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha1/fake/doc.go index 87a1873edccb3..ab4fd43ad6e01 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha1/fake/doc.go +++ b/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha1/fake/doc.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha1/fake/fake_cluster.go b/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha1/fake/fake_cluster.go index f01ad222b6dc8..2908735bef423 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha1/fake/fake_cluster.go +++ b/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha1/fake/fake_cluster.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha1/fake/fake_instancegroup.go b/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha1/fake/fake_instancegroup.go index 9815a09d966f0..c575da471ada2 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha1/fake/fake_instancegroup.go +++ b/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha1/fake/fake_instancegroup.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha1/fake/fake_kops_client.go b/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha1/fake/fake_kops_client.go index 83fa6ae395f28..ad0da22ba9a54 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha1/fake/fake_kops_client.go +++ b/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha1/fake/fake_kops_client.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha1/fake/fake_sshcredential.go b/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha1/fake/fake_sshcredential.go index eb77602e270db..4508a54a25741 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha1/fake/fake_sshcredential.go +++ b/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha1/fake/fake_sshcredential.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha1/generated_expansion.go b/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha1/generated_expansion.go index 79fabfc3a2ada..37793a8c40516 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha1/generated_expansion.go +++ b/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha1/generated_expansion.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha1/instancegroup.go b/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha1/instancegroup.go index ab4bfe535a786..52c12d2ba4919 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha1/instancegroup.go +++ b/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha1/instancegroup.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha1/kops_client.go b/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha1/kops_client.go index bfb6cc782c1c0..6d59b63ffafb8 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha1/kops_client.go +++ b/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha1/kops_client.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha1/sshcredential.go b/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha1/sshcredential.go index ee9c67557a640..ecb20d456e53f 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha1/sshcredential.go +++ b/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha1/sshcredential.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha2/cluster.go b/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha2/cluster.go index 548d7bb46ad09..1b76948afb7ce 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha2/cluster.go +++ b/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha2/cluster.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha2/doc.go b/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha2/doc.go index 01dd46de84d8d..e4cda4fe26734 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha2/doc.go +++ b/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha2/doc.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha2/fake/doc.go b/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha2/fake/doc.go index 87a1873edccb3..ab4fd43ad6e01 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha2/fake/doc.go +++ b/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha2/fake/doc.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha2/fake/fake_cluster.go b/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha2/fake/fake_cluster.go index 94a3dcffe6a08..db23945ceed65 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha2/fake/fake_cluster.go +++ b/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha2/fake/fake_cluster.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha2/fake/fake_instancegroup.go b/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha2/fake/fake_instancegroup.go index f4cd2508e1a8a..e78cf043f6641 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha2/fake/fake_instancegroup.go +++ b/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha2/fake/fake_instancegroup.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha2/fake/fake_keyset.go b/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha2/fake/fake_keyset.go index c3a2bba02993d..d6a5e767d7882 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha2/fake/fake_keyset.go +++ b/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha2/fake/fake_keyset.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha2/fake/fake_kops_client.go b/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha2/fake/fake_kops_client.go index e6f0d4986f30f..b7bea481afd8a 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha2/fake/fake_kops_client.go +++ b/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha2/fake/fake_kops_client.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha2/fake/fake_sshcredential.go b/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha2/fake/fake_sshcredential.go index eb505ade8d3e1..e15c83d6932b4 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha2/fake/fake_sshcredential.go +++ b/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha2/fake/fake_sshcredential.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha2/generated_expansion.go b/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha2/generated_expansion.go index 9c61848846512..9b98449352673 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha2/generated_expansion.go +++ b/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha2/generated_expansion.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha2/instancegroup.go b/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha2/instancegroup.go index 17bff7fa28d3c..4cb8bbb9264b3 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha2/instancegroup.go +++ b/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha2/instancegroup.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha2/keyset.go b/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha2/keyset.go index 47bc758a122c1..9a844221deb5a 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha2/keyset.go +++ b/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha2/keyset.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha2/kops_client.go b/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha2/kops_client.go index 09c345c643aa6..e57f2f5fced3f 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha2/kops_client.go +++ b/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha2/kops_client.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha2/sshcredential.go b/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha2/sshcredential.go index ea966a6302720..4bbc8aa6998b4 100644 --- a/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha2/sshcredential.go +++ b/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha2/sshcredential.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/client/simple/api/clientset.go b/pkg/client/simple/api/clientset.go index 6ee5994134231..bc783c5bf89ad 100644 --- a/pkg/client/simple/api/clientset.go +++ b/pkg/client/simple/api/clientset.go @@ -72,7 +72,7 @@ func (c *RESTClientset) ConfigBaseFor(cluster *kops.Cluster) (vfs.Path, error) { if cluster.Spec.ConfigBase != "" { return vfs.Context.BuildVfsPath(cluster.Spec.ConfigBase) } - // URL for clusters looks like https:///apis/kops/v1alpha2/namespaces//clusters/ + // URL for clusters looks like https:///apis/kops/v1alpha2/namespaces//clusters/ // We probably want to add a subresource for full resources return vfs.Context.BuildVfsPath(c.BaseURL.String()) } diff --git a/pkg/commands/set_cluster.go b/pkg/commands/set_cluster.go index a45b283b52e85..4c4a9b08d36b8 100644 --- a/pkg/commands/set_cluster.go +++ b/pkg/commands/set_cluster.go @@ -19,6 +19,7 @@ package commands import ( "fmt" "io" + "strconv" "strings" "github.com/spf13/cobra" @@ -84,13 +85,36 @@ func SetClusterFields(fields []string, cluster *api.Cluster, instanceGroups []*a cluster.Spec.NodePortAccess = append(cluster.Spec.NodePortAccess, kv[1]) case "spec.kubernetesVersion": cluster.Spec.KubernetesVersion = kv[1] + + case "cluster.spec.etcdClusters[*].enableEtcdTLS": + v, err := strconv.ParseBool(kv[1]) + if err != nil { + return fmt.Errorf("unknown boolean value: %q", kv[1]) + } + for _, c := range cluster.Spec.EtcdClusters { + c.EnableEtcdTLS = v + } + + case "cluster.spec.etcdClusters[*].enableTLSAuth": + v, err := strconv.ParseBool(kv[1]) + if err != nil { + return fmt.Errorf("unknown boolean value: %q", kv[1]) + } + for _, c := range cluster.Spec.EtcdClusters { + c.EnableTLSAuth = v + } + case "cluster.spec.etcdClusters[*].version": for _, c := range cluster.Spec.EtcdClusters { c.Version = kv[1] } case "cluster.spec.etcdClusters[*].provider": + p, err := toEtcdProviderType(kv[1]) + if err != nil { + return err + } for _, etcd := range cluster.Spec.EtcdClusters { - etcd.Provider = api.EtcdProviderType(kv[1]) + etcd.Provider = p } case "cluster.spec.etcdClusters[*].manager.image": for _, etcd := range cluster.Spec.EtcdClusters { @@ -105,3 +129,15 @@ func SetClusterFields(fields []string, cluster *api.Cluster, instanceGroups []*a } return nil } + +func toEtcdProviderType(in string) (api.EtcdProviderType, error) { + s := strings.ToLower(in) + switch s { + case "legacy": + return api.EtcdProviderTypeLegacy, nil + case "manager": + return api.EtcdProviderTypeManager, nil + default: + return api.EtcdProviderTypeManager, fmt.Errorf("unknown etcd provider type %q", in) + } +} diff --git a/pkg/model/awsmodel/BUILD.bazel b/pkg/model/awsmodel/BUILD.bazel index b5efd0e979d82..b7d97f18cdef2 100644 --- a/pkg/model/awsmodel/BUILD.bazel +++ b/pkg/model/awsmodel/BUILD.bazel @@ -13,6 +13,7 @@ go_library( deps = [ "//pkg/apis/kops:go_default_library", "//pkg/dns:go_default_library", + "//pkg/featureflag:go_default_library", "//pkg/model:go_default_library", "//pkg/model/defaults:go_default_library", "//upup/pkg/fi:go_default_library", diff --git a/pkg/model/awsmodel/api_loadbalancer.go b/pkg/model/awsmodel/api_loadbalancer.go index af7ece83791bc..5a7ca79d9933b 100644 --- a/pkg/model/awsmodel/api_loadbalancer.go +++ b/pkg/model/awsmodel/api_loadbalancer.go @@ -25,6 +25,7 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/pkg/dns" + "k8s.io/kops/pkg/featureflag" "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/cloudup/awstasks" "k8s.io/kops/upup/pkg/fi/fitasks" @@ -254,16 +255,21 @@ func (b *APILoadBalancerBuilder) Build(c *fi.ModelBuilderContext) error { masterKeypair.AlternateNameTasks = append(masterKeypair.AlternateNameTasks, elb) } - for _, ig := range b.MasterInstanceGroups() { - t := &awstasks.LoadBalancerAttachment{ - Name: s("api-" + ig.ObjectMeta.Name), - Lifecycle: b.Lifecycle, + // When Spotinst Elastigroups are used, there is no need to create + // a separate task for the attachment of the load balancer since this + // is already done as part of the Elastigroup's creation, if needed. + if !featureflag.Spotinst.Enabled() { + for _, ig := range b.MasterInstanceGroups() { + t := &awstasks.LoadBalancerAttachment{ + Name: s("api-" + ig.ObjectMeta.Name), + Lifecycle: b.Lifecycle, + + LoadBalancer: b.LinkToELB("api"), + AutoscalingGroup: b.LinkToAutoscalingGroup(ig), + } - LoadBalancer: b.LinkToELB("api"), - AutoscalingGroup: b.LinkToAutoscalingGroup(ig), + c.AddTask(t) } - - c.AddTask(t) } return nil diff --git a/pkg/model/awsmodel/autoscalinggroup.go b/pkg/model/awsmodel/autoscalinggroup.go index aa50828cd1753..c651ed056c8a3 100644 --- a/pkg/model/awsmodel/autoscalinggroup.go +++ b/pkg/model/awsmodel/autoscalinggroup.go @@ -81,8 +81,9 @@ func (b *AutoscalingGroupModelBuilder) Build(c *fi.ModelBuilderContext) error { if ig.Spec.SecurityGroupOverride != nil { glog.V(1).Infof("WARNING: You are overwriting the Instance Groups, Security Group. When this is done you are responsible for ensure the correct rules!") + sgName := fmt.Sprintf("%v-%v", fi.StringValue(ig.Spec.SecurityGroupOverride), ig.Spec.Role) sgLink = &awstasks.SecurityGroup{ - Name: ig.Spec.SecurityGroupOverride, + Name: &sgName, ID: ig.Spec.SecurityGroupOverride, Shared: fi.Bool(true), } diff --git a/pkg/model/bastion.go b/pkg/model/bastion.go index 47f75305b3f84..54aa9511ebeb6 100644 --- a/pkg/model/bastion.go +++ b/pkg/model/bastion.go @@ -21,6 +21,7 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/kops/pkg/apis/kops" + "k8s.io/kops/pkg/featureflag" "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/cloudup/awstasks" ) @@ -236,18 +237,23 @@ func (b *BastionModelBuilder) Build(c *fi.ModelBuilderContext) error { c.AddTask(elb) } - for _, ig := range bastionInstanceGroups { - // We build the ASG when we iterate over the instance groups + // When Spotinst Elastigroups are used, there is no need to create + // a separate task for the attachment of the load balancer since this + // is already done as part of the Elastigroup's creation, if needed. + if !featureflag.Spotinst.Enabled() { + for _, ig := range bastionInstanceGroups { + // We build the ASG when we iterate over the instance groups - // Attach the ELB to the ASG - t := &awstasks.LoadBalancerAttachment{ - Name: s("bastion-elb-attachment"), - Lifecycle: b.Lifecycle, + // Attach the ELB to the ASG + t := &awstasks.LoadBalancerAttachment{ + Name: s("bastion-elb-attachment"), + Lifecycle: b.Lifecycle, - LoadBalancer: elb, - AutoscalingGroup: b.LinkToAutoscalingGroup(ig), + LoadBalancer: elb, + AutoscalingGroup: b.LinkToAutoscalingGroup(ig), + } + c.AddTask(t) } - c.AddTask(t) } bastionPublicName := "" diff --git a/pkg/model/components/context.go b/pkg/model/components/context.go index 0bce1cb49598e..21e2cfcde1da4 100644 --- a/pkg/model/components/context.go +++ b/pkg/model/components/context.go @@ -78,7 +78,7 @@ func UsesKubenet(clusterSpec *kops.ClusterSpec) (bool, error) { } else if networking.External != nil { // external is based on kubenet return true, nil - } else if networking.CNI != nil || networking.Weave != nil || networking.Flannel != nil || networking.Calico != nil || networking.Canal != nil || networking.Kuberouter != nil || networking.Romana != nil || networking.AmazonVPC != nil || networking.Cilium != nil { + } else if networking.CNI != nil || networking.Weave != nil || networking.Flannel != nil || networking.Calico != nil || networking.Canal != nil || networking.Kuberouter != nil || networking.Romana != nil || networking.AmazonVPC != nil || networking.Cilium != nil || networking.LyftVPC != nil { return false, nil } else if networking.Kopeio != nil { // Kopeio is based on kubenet / external diff --git a/pkg/model/components/docker.go b/pkg/model/components/docker.go index 4a8bae23d5321..82f1983dd2fef 100644 --- a/pkg/model/components/docker.go +++ b/pkg/model/components/docker.go @@ -53,7 +53,9 @@ func (b *DockerOptionsBuilder) BuildOptions(o interface{}) error { } dockerVersion := "" - if sv.Major == 1 && sv.Minor >= 9 { + if sv.Major == 1 && sv.Minor >= 12 { + dockerVersion = "18.06.3" + } else if sv.Major == 1 && sv.Minor >= 9 { dockerVersion = "17.03.2" } else if sv.Major == 1 && sv.Minor >= 8 { dockerVersion = "1.13.1" diff --git a/pkg/model/components/etcdmanager/model.go b/pkg/model/components/etcdmanager/model.go index eb2d01dd4e1fd..e02d144b26fa1 100644 --- a/pkg/model/components/etcdmanager/model.go +++ b/pkg/model/components/etcdmanager/model.go @@ -100,6 +100,8 @@ func (b *EtcdManagerBuilder) Build(c *fi.ModelBuilderContext) error { return err } + format := string(fi.KeysetFormatV1Alpha2) + c.AddTask(&fitasks.ManagedFile{ Contents: fi.WrapResource(fi.NewBytesResource(d)), Lifecycle: b.Lifecycle, @@ -107,14 +109,22 @@ func (b *EtcdManagerBuilder) Build(c *fi.ModelBuilderContext) error { Location: fi.String("backups/etcd/" + etcdCluster.Name + "/control/etcd-cluster-spec"), Name: fi.String("etcd-cluster-spec-" + name), }) + + // We create a CA keypair to enable secure communication + c.AddTask(&fitasks.Keypair{ + Name: fi.String("etcd-manager-ca-" + etcdCluster.Name), + Subject: "cn=etcd-manager-ca-" + etcdCluster.Name, + Type: "ca", + Format: format, + }) } return nil } type etcdClusterSpec struct { - MemberCount int32 `json:"member_count,omitempty"` - EtcdVersion string `json:"etcd_version,omitempty"` + MemberCount int32 `json:"memberCount,omitempty"` + EtcdVersion string `json:"etcdVersion,omitempty"` } func (b *EtcdManagerBuilder) buildManifest(etcdCluster *kops.EtcdClusterSpec) (*v1.Pod, error) { @@ -159,7 +169,7 @@ metadata: namespace: kube-system spec: containers: - - image: kopeio/etcd-manager:1.0.20181001 + - image: kopeio/etcd-manager:3.0.20190224 name: etcd-manager resources: requests: @@ -174,6 +184,8 @@ spec: # We write artificial hostnames into etc hosts for the etcd nodes, so they have stable names - mountPath: /etc/hosts name: hosts + - mountPath: /etc/kubernetes/pki/etcd-manager + name: pki hostNetwork: true hostPID: true # helps with mounting volumes from inside a container volumes: @@ -185,6 +197,10 @@ spec: path: /etc/hosts type: File name: hosts + - hostPath: + path: /etc/kubernetes/pki/etcd-manager + type: DirectoryOrCreate + name: pki ` // buildPod creates the pod spec, based on the EtcdClusterSpec @@ -300,6 +316,7 @@ func (b *EtcdManagerBuilder) buildPod(etcdCluster *kops.EtcdClusterSpec) (*v1.Po BackupStore: backupStore, GrpcPort: grpcPort, DNSSuffix: dnsInternalSuffix, + EtcdInsecure: !isTLS, } config.LogVerbosity = 8 @@ -389,12 +406,30 @@ func (b *EtcdManagerBuilder) buildPod(etcdCluster *kops.EtcdClusterSpec) (*v1.Po }, }, }) + } - if isTLS { - return nil, fmt.Errorf("TLS not supported for etcd-manager") + { + foundPKI := false + for i := range pod.Spec.Volumes { + v := &pod.Spec.Volumes[i] + if v.Name == "pki" { + if v.HostPath == nil { + return nil, fmt.Errorf("found PKI volume, but HostPath was nil") + } + dirname := "etcd-manager-" + etcdCluster.Name + v.HostPath.Path = "/etc/kubernetes/pki/" + dirname + foundPKI = true + } + } + if !foundPKI { + return nil, fmt.Errorf("did not find PKI volume") } } + if isTLS { + return nil, fmt.Errorf("TLS not supported for etcd-manager") + } + kubemanifest.MarkPodAsCritical(pod) return pod, nil @@ -408,6 +443,15 @@ type config struct { // Containerized is set if etcd-manager is running in a container Containerized bool `flag:"containerized"` + // PKIDir is set to the directory for PKI keys, used to secure commucations between etcd-manager peers + PKIDir string `flag:"pki-dir"` + + // Insecure can be used to turn off tls for etcd-manager (compare with EtcdInsecure) + Insecure bool `flag:"insecure"` + + // EtcdInsecure can be used to turn off tls for etcd itself (compare with Insecure) + EtcdInsecure bool `flag:"etcd-insecure"` + Address string `flag:"address"` PeerUrls string `flag:"peer-urls"` GrpcPort int `flag:"grpc-port"` diff --git a/pkg/model/components/etcdmanager/tests/minimal/tasks.yaml b/pkg/model/components/etcdmanager/tests/minimal/tasks.yaml index 4ea8ad7b17301..ec165508d7a6b 100644 --- a/pkg/model/components/etcdmanager/tests/minimal/tasks.yaml +++ b/pkg/model/components/etcdmanager/tests/minimal/tasks.yaml @@ -1,8 +1,26 @@ +Lifecycle: null +Name: etcd-manager-ca-events +Signer: null +alternateNameTasks: null +alternateNames: null +format: v1alpha2 +subject: cn=etcd-manager-ca-events +type: ca +--- +Lifecycle: null +Name: etcd-manager-ca-main +Signer: null +alternateNameTasks: null +alternateNames: null +format: v1alpha2 +subject: cn=etcd-manager-ca-main +type: ca +--- Contents: Name: "" Resource: |- { - "member_count": 1 + "memberCount": 1 } Lifecycle: null Location: backups/etcd/events/control/etcd-cluster-spec @@ -12,7 +30,7 @@ Contents: Name: "" Resource: |- { - "member_count": 1 + "memberCount": 1 } Lifecycle: null Location: backups/etcd/main/control/etcd-cluster-spec @@ -39,11 +57,12 @@ Contents: - mkfifo /tmp/pipe; (tee -a /var/log/etcd.log < /tmp/pipe & ) ; exec /etcd-manager --backup-store=memfs://clusters.example.com/minimal.example.com/backups/etcd-events --client-urls=http://__name__:4002 --cluster-name=etcd-events --containerized=true - --dns-suffix=.internal.minimal.example.com --grpc-port=3997 --peer-urls=http://__name__:2381 - --quarantine-client-urls=http://__name__:3995 --v=8 --volume-name-tag=k8s.io/etcd/events - --volume-provider=aws --volume-tag=k8s.io/etcd/events --volume-tag=k8s.io/role/master=1 - --volume-tag=kubernetes.io/cluster/minimal.example.com=owned > /tmp/pipe 2>&1 - image: kopeio/etcd-manager:1.0.20181001 + --dns-suffix=.internal.minimal.example.com --etcd-insecure=true --grpc-port=3997 + --insecure=false --peer-urls=http://__name__:2381 --quarantine-client-urls=http://__name__:3995 + --v=8 --volume-name-tag=k8s.io/etcd/events --volume-provider=aws --volume-tag=k8s.io/etcd/events + --volume-tag=k8s.io/role/master=1 --volume-tag=kubernetes.io/cluster/minimal.example.com=owned + > /tmp/pipe 2>&1 + image: kopeio/etcd-manager:3.0.20190224 name: etcd-manager resources: requests: @@ -55,6 +74,8 @@ Contents: name: rootfs - mountPath: /etc/hosts name: hosts + - mountPath: /etc/kubernetes/pki/etcd-manager + name: pki - mountPath: /var/log/etcd.log name: varlogetcd hostNetwork: true @@ -71,6 +92,10 @@ Contents: path: /etc/hosts type: File name: hosts + - hostPath: + path: /etc/kubernetes/pki/etcd-manager-events + type: DirectoryOrCreate + name: pki - hostPath: path: /var/log/etcd-events.log type: FileOrCreate @@ -101,11 +126,12 @@ Contents: - mkfifo /tmp/pipe; (tee -a /var/log/etcd.log < /tmp/pipe & ) ; exec /etcd-manager --backup-store=memfs://clusters.example.com/minimal.example.com/backups/etcd-main --client-urls=http://__name__:4001 --cluster-name=etcd --containerized=true - --dns-suffix=.internal.minimal.example.com --grpc-port=3996 --peer-urls=http://__name__:2380 - --quarantine-client-urls=http://__name__:3994 --v=8 --volume-name-tag=k8s.io/etcd/main - --volume-provider=aws --volume-tag=k8s.io/etcd/main --volume-tag=k8s.io/role/master=1 - --volume-tag=kubernetes.io/cluster/minimal.example.com=owned > /tmp/pipe 2>&1 - image: kopeio/etcd-manager:1.0.20181001 + --dns-suffix=.internal.minimal.example.com --etcd-insecure=true --grpc-port=3996 + --insecure=false --peer-urls=http://__name__:2380 --quarantine-client-urls=http://__name__:3994 + --v=8 --volume-name-tag=k8s.io/etcd/main --volume-provider=aws --volume-tag=k8s.io/etcd/main + --volume-tag=k8s.io/role/master=1 --volume-tag=kubernetes.io/cluster/minimal.example.com=owned + > /tmp/pipe 2>&1 + image: kopeio/etcd-manager:3.0.20190224 name: etcd-manager resources: requests: @@ -117,6 +143,8 @@ Contents: name: rootfs - mountPath: /etc/hosts name: hosts + - mountPath: /etc/kubernetes/pki/etcd-manager + name: pki - mountPath: /var/log/etcd.log name: varlogetcd hostNetwork: true @@ -133,6 +161,10 @@ Contents: path: /etc/hosts type: File name: hosts + - hostPath: + path: /etc/kubernetes/pki/etcd-manager-main + type: DirectoryOrCreate + name: pki - hostPath: path: /var/log/etcd.log type: FileOrCreate diff --git a/pkg/model/components/kubecontrollermanager.go b/pkg/model/components/kubecontrollermanager.go index d5116f58a42ae..bb4d4cede4dd7 100644 --- a/pkg/model/components/kubecontrollermanager.go +++ b/pkg/model/components/kubecontrollermanager.go @@ -152,7 +152,7 @@ func (b *KubeControllerManagerOptionsBuilder) BuildOptions(o interface{}) error kcm.ConfigureCloudRoutes = fi.Bool(true) } else if networking.External != nil { kcm.ConfigureCloudRoutes = fi.Bool(false) - } else if networking.CNI != nil || networking.Weave != nil || networking.Flannel != nil || networking.Calico != nil || networking.Canal != nil || networking.Kuberouter != nil || networking.Romana != nil || networking.AmazonVPC != nil || networking.Cilium != nil { + } else if networking.CNI != nil || networking.Weave != nil || networking.Flannel != nil || networking.Calico != nil || networking.Canal != nil || networking.Kuberouter != nil || networking.Romana != nil || networking.AmazonVPC != nil || networking.Cilium != nil || networking.LyftVPC != nil { kcm.ConfigureCloudRoutes = fi.Bool(false) } else if networking.Kopeio != nil { // Kopeio is based on kubenet / external diff --git a/pkg/model/components/networking.go b/pkg/model/components/networking.go index 6a86273a9a7d4..53fa4df626c7b 100644 --- a/pkg/model/components/networking.go +++ b/pkg/model/components/networking.go @@ -48,7 +48,8 @@ func (b *NetworkingOptionsBuilder) BuildOptions(o interface{}) error { if networking == nil { return fmt.Errorf("networking not set") } - if networking.CNI != nil || networking.Weave != nil || networking.Flannel != nil || networking.Calico != nil || networking.Canal != nil || networking.Kuberouter != nil || networking.Romana != nil || networking.AmazonVPC != nil || networking.Cilium != nil { + + if networking.CNI != nil || networking.Weave != nil || networking.Flannel != nil || networking.Calico != nil || networking.Canal != nil || networking.Kuberouter != nil || networking.Romana != nil || networking.AmazonVPC != nil || networking.Cilium != nil || networking.LyftVPC != nil { options.Kubelet.NetworkPluginName = "cni" if k8sVersion.Major == 1 && k8sVersion.Minor <= 4 { diff --git a/pkg/model/firewall.go b/pkg/model/firewall.go index 41e23f7917acb..dea08e12d6f39 100644 --- a/pkg/model/firewall.go +++ b/pkg/model/firewall.go @@ -483,8 +483,9 @@ func (b *KopsModelContext) GetSecurityGroups(role kops.InstanceGroupRole) ([]Sec } done[name] = true + sgName := fmt.Sprintf("%v-%v", fi.StringValue(ig.Spec.SecurityGroupOverride), role) t := &awstasks.SecurityGroup{ - Name: ig.Spec.SecurityGroupOverride, + Name: &sgName, ID: ig.Spec.SecurityGroupOverride, VPC: b.LinkToVPC(), Shared: fi.Bool(true), diff --git a/pkg/model/iam/iam_builder.go b/pkg/model/iam/iam_builder.go index 5c5c4407e32ae..e3a7cb4654978 100644 --- a/pkg/model/iam/iam_builder.go +++ b/pkg/model/iam/iam_builder.go @@ -188,6 +188,10 @@ func (b *PolicyBuilder) BuildAWSPolicyMaster() (*Policy, error) { addAmazonVPCCNIPermissions(p, resource, b.Cluster.Spec.IAM.Legacy, b.Cluster.GetName()) } + if b.Cluster.Spec.Networking != nil && b.Cluster.Spec.Networking.LyftVPC != nil { + addLyftVPCPermissions(p, resource, b.Cluster.Spec.IAM.Legacy, b.Cluster.GetName()) + } + return p, nil } @@ -221,6 +225,10 @@ func (b *PolicyBuilder) BuildAWSPolicyNode() (*Policy, error) { addAmazonVPCCNIPermissions(p, resource, b.Cluster.Spec.IAM.Legacy, b.Cluster.GetName()) } + if b.Cluster.Spec.Networking != nil && b.Cluster.Spec.Networking.LyftVPC != nil { + addLyftVPCPermissions(p, resource, b.Cluster.Spec.IAM.Legacy, b.Cluster.GetName()) + } + return p, nil } @@ -314,7 +322,7 @@ func (b *PolicyBuilder) AddS3Permissions(p *Policy) (*Policy, error) { p.Statement = append(p.Statement, &Statement{ Effect: StatementEffectAllow, - Action: stringorslice.Of("s3:GetBucketLocation", "s3:ListBucket"), + Action: stringorslice.Of("s3:GetBucketLocation", "s3:GetEncryptionConfiguration", "s3:ListBucket"), Resource: stringorslice.Slice([]string{ strings.Join([]string{b.IAMPrefix(), ":s3:::", s3Path.Bucket()}, ""), }), @@ -820,6 +828,33 @@ func addRomanaCNIPermissions(p *Policy, resource stringorslice.StringOrSlice, le ) } +func addLyftVPCPermissions(p *Policy, resource stringorslice.StringOrSlice, legacyIAM bool, clusterName string) { + if legacyIAM { + // Legacy IAM provides ec2:*, so no additional permissions required + return + } + + p.Statement = append(p.Statement, + &Statement{ + Effect: StatementEffectAllow, + Action: stringorslice.Slice([]string{ + "ec2:DescribeSubnets", + "ec2:AttachNetworkInterface", + "ec2:AssignPrivateIpAddresses", + "ec2:UnassignPrivateIpAddresses", + "ec2:CreateNetworkInterface", + "ec2:DescribeNetworkInterfaces", + "ec2:DescribeVpcPeeringConnections", + "ec2:DescribeSecurityGroups", + "ec2:DetachNetworkInterface", + "ec2:DeleteNetworkInterface", + "ec2:ModifyNetworkInterfaceAttribute", + }), + Resource: resource, + }, + ) +} + func addAmazonVPCCNIPermissions(p *Policy, resource stringorslice.StringOrSlice, legacyIAM bool, clusterName string) { if legacyIAM { // Legacy IAM provides ec2:*, so no additional permissions required diff --git a/pkg/model/iam/tests/iam_builder_master_legacy.json b/pkg/model/iam/tests/iam_builder_master_legacy.json index c5d9728b38993..640fce7d63f88 100644 --- a/pkg/model/iam/tests/iam_builder_master_legacy.json +++ b/pkg/model/iam/tests/iam_builder_master_legacy.json @@ -48,6 +48,7 @@ "Effect": "Allow", "Action": [ "s3:GetBucketLocation", + "s3:GetEncryptionConfiguration", "s3:ListBucket" ], "Resource": [ diff --git a/pkg/model/iam/tests/iam_builder_master_strict.json b/pkg/model/iam/tests/iam_builder_master_strict.json index f5f595b4bfa6f..fbd0b74ca1016 100644 --- a/pkg/model/iam/tests/iam_builder_master_strict.json +++ b/pkg/model/iam/tests/iam_builder_master_strict.json @@ -138,6 +138,7 @@ "Effect": "Allow", "Action": [ "s3:GetBucketLocation", + "s3:GetEncryptionConfiguration", "s3:ListBucket" ], "Resource": [ diff --git a/pkg/model/iam/tests/iam_builder_master_strict_ecr.json b/pkg/model/iam/tests/iam_builder_master_strict_ecr.json index 0ebd4459010b1..a21d001eb9501 100644 --- a/pkg/model/iam/tests/iam_builder_master_strict_ecr.json +++ b/pkg/model/iam/tests/iam_builder_master_strict_ecr.json @@ -138,6 +138,7 @@ "Effect": "Allow", "Action": [ "s3:GetBucketLocation", + "s3:GetEncryptionConfiguration", "s3:ListBucket" ], "Resource": [ diff --git a/pkg/model/iam/tests/iam_builder_node_legacy.json b/pkg/model/iam/tests/iam_builder_node_legacy.json index 1c330cbf37094..5926662e64082 100644 --- a/pkg/model/iam/tests/iam_builder_node_legacy.json +++ b/pkg/model/iam/tests/iam_builder_node_legacy.json @@ -15,6 +15,7 @@ "Effect": "Allow", "Action": [ "s3:GetBucketLocation", + "s3:GetEncryptionConfiguration", "s3:ListBucket" ], "Resource": [ diff --git a/pkg/model/iam/tests/iam_builder_node_strict.json b/pkg/model/iam/tests/iam_builder_node_strict.json index 1d28f2718a119..aa7b32f2471f7 100644 --- a/pkg/model/iam/tests/iam_builder_node_strict.json +++ b/pkg/model/iam/tests/iam_builder_node_strict.json @@ -15,6 +15,7 @@ "Effect": "Allow", "Action": [ "s3:GetBucketLocation", + "s3:GetEncryptionConfiguration", "s3:ListBucket" ], "Resource": [ diff --git a/pkg/model/iam/tests/iam_builder_node_strict_ecr.json b/pkg/model/iam/tests/iam_builder_node_strict_ecr.json index 10053f1c0d519..61df952ce32bb 100644 --- a/pkg/model/iam/tests/iam_builder_node_strict_ecr.json +++ b/pkg/model/iam/tests/iam_builder_node_strict_ecr.json @@ -15,6 +15,7 @@ "Effect": "Allow", "Action": [ "s3:GetBucketLocation", + "s3:GetEncryptionConfiguration", "s3:ListBucket" ], "Resource": [ diff --git a/pkg/model/master_volumes.go b/pkg/model/master_volumes.go index 0fed42efe7f96..72ced7f2b184f 100644 --- a/pkg/model/master_volumes.go +++ b/pkg/model/master_volumes.go @@ -92,7 +92,10 @@ func (b *MasterVolumeBuilder) Build(c *fi.ModelBuilderContext) error { switch kops.CloudProviderID(b.Cluster.Spec.CloudProvider) { case kops.CloudProviderAWS: - b.addAWSVolume(c, name, volumeSize, zone, etcd, m, allMembers) + err = b.addAWSVolume(c, name, volumeSize, zone, etcd, m, allMembers) + if err != nil { + return err + } case kops.CloudProviderDO: b.addDOVolume(c, name, volumeSize, zone, etcd, m, allMembers) case kops.CloudProviderGCE: @@ -116,7 +119,7 @@ func (b *MasterVolumeBuilder) Build(c *fi.ModelBuilderContext) error { return nil } -func (b *MasterVolumeBuilder) addAWSVolume(c *fi.ModelBuilderContext, name string, volumeSize int32, zone string, etcd *kops.EtcdClusterSpec, m *kops.EtcdMemberSpec, allMembers []string) { +func (b *MasterVolumeBuilder) addAWSVolume(c *fi.ModelBuilderContext, name string, volumeSize int32, zone string, etcd *kops.EtcdClusterSpec, m *kops.EtcdMemberSpec, allMembers []string) error { volumeType := fi.StringValue(m.VolumeType) volumeIops := fi.Int32Value(m.VolumeIops) switch volumeType { @@ -160,9 +163,16 @@ func (b *MasterVolumeBuilder) addAWSVolume(c *fi.ModelBuilderContext, name strin } if volumeType == "io1" { t.VolumeIops = i64(int64(volumeIops)) + + // https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/EBSVolumeTypes.html + if float64(*t.VolumeIops)/float64(*t.SizeGB) > 50.0 { + return fmt.Errorf("volumeIops to volumeSize ratio must be lower than 50. For %s ratio is %f", *t.Name, float64(*t.VolumeIops)/float64(*t.SizeGB)) + } } c.AddTask(t) + + return nil } func (b *MasterVolumeBuilder) addDOVolume(c *fi.ModelBuilderContext, name string, volumeSize int32, zone string, etcd *kops.EtcdClusterSpec, m *kops.EtcdMemberSpec, allMembers []string) { diff --git a/pkg/model/network.go b/pkg/model/network.go index 4acdfe6a104f1..b605fa63a7762 100644 --- a/pkg/model/network.go +++ b/pkg/model/network.go @@ -184,21 +184,27 @@ func (b *NetworkModelBuilder) Build(c *fi.ModelBuilderContext) error { subnetSpec := &b.Cluster.Spec.Subnets[i] sharedSubnet := subnetSpec.ProviderID != "" subnetName := subnetSpec.Name + "." + b.ClusterName() - tags := b.CloudTags(subnetName, sharedSubnet) + tags := map[string]string{} // Apply tags so that Kubernetes knows which subnets should be used for internal/external ELBs - switch subnetSpec.Type { - case kops.SubnetTypePublic, kops.SubnetTypeUtility: - tags[aws.TagNameSubnetPublicELB] = "1" + if b.Cluster.Spec.DisableSubnetTags { + glog.V(2).Infof("skipping subnet tags. Ensure these are maintained externally.") + } else { + glog.V(2).Infof("applying subnet tags") + tags = b.CloudTags(subnetName, sharedSubnet) + tags["SubnetType"] = string(subnetSpec.Type) - case kops.SubnetTypePrivate: - tags[aws.TagNameSubnetInternalELB] = "1" + switch subnetSpec.Type { + case kops.SubnetTypePublic, kops.SubnetTypeUtility: + tags[aws.TagNameSubnetPublicELB] = "1" - default: - glog.V(2).Infof("unable to properly tag subnet %q because it has unknown type %q. Load balancers may be created in incorrect subnets", subnetSpec.Name, subnetSpec.Type) - } + case kops.SubnetTypePrivate: + tags[aws.TagNameSubnetInternalELB] = "1" - tags["SubnetType"] = string(subnetSpec.Type) + default: + glog.V(2).Infof("unable to properly tag subnet %q because it has unknown type %q. Load balancers may be created in incorrect subnets", subnetSpec.Name, subnetSpec.Type) + } + } subnet := &awstasks.Subnet{ Name: s(subnetName), diff --git a/pkg/model/pki.go b/pkg/model/pki.go index eb5c86e0d5900..2c0eec9bb3a6b 100644 --- a/pkg/model/pki.go +++ b/pkg/model/pki.go @@ -121,18 +121,48 @@ func (b *PKIModelBuilder) Build(c *fi.ModelBuilderContext) error { // that mutual tls used to verify between the peers we don't want certificates for kubernetes able to act as a peer. // For clients assuming we are using etcdv3 is can switch on user authentication and map the common names for auth. if b.UseEtcdTLS() { - alternativeNames := []string{fmt.Sprintf("*.internal.%s", b.ClusterName()), "localhost", "127.0.0.1"} + servingNames := []string{fmt.Sprintf("*.internal.%s", b.ClusterName()), "localhost", "127.0.0.1"} // @question should wildcard's be here instead of generating per node. If we ever provide the // ability to resize the master, this will become a blocker c.AddTask(&fitasks.Keypair{ - AlternateNames: alternativeNames, + AlternateNames: servingNames, Lifecycle: b.Lifecycle, Name: fi.String("etcd"), Subject: "cn=etcd", + // TODO: Can this be "server" now that we're not using it for peer connectivity? + Type: "clientServer", + Signer: defaultCA, + Format: format, + }) + + // For peer authentication, the same cert is used both as a client + // cert and as a server cert (which is unusual). Moreover, etcd + // 3.2 introduces some breaking changes to certificate validation + // where it tries to match any IP or DNS names to the client IP + // (including reverse DNS lookups!) We _could_ include a wildcard + // reverse DNS name e.g. *.ec2.internal for EC2, but it seems + // better just to list the names that we expect peer connectivity + // to happen on. + var peerNames []string + for _, etcdCluster := range b.Cluster.Spec.EtcdClusters { + prefix := "etcd-" + etcdCluster.Name + "-" + if prefix == "etcd-main-" { + prefix = "etcd-" + } + for _, m := range etcdCluster.Members { + peerNames = append(peerNames, prefix+m.Name+".internal."+b.ClusterName()) + } + } + c.AddTask(&fitasks.Keypair{ + AlternateNames: peerNames, + Lifecycle: b.Lifecycle, + Name: fi.String("etcd-peer"), + Subject: "cn=etcd-peer", Type: "clientServer", Signer: defaultCA, Format: format, }) + c.AddTask(&fitasks.Keypair{ Name: fi.String("etcd-client"), Lifecycle: b.Lifecycle, diff --git a/pkg/resources/aws/securitygroup.go b/pkg/resources/aws/securitygroup.go index 56e4cf31a5f73..0bb874e703246 100644 --- a/pkg/resources/aws/securitygroup.go +++ b/pkg/resources/aws/securitygroup.go @@ -51,7 +51,7 @@ func DeleteSecurityGroup(cloud fi.Cloud, t *resources.Resource) error { return nil } if len(response.SecurityGroups) != 1 { - return fmt.Errorf("found mutiple SecurityGroups with ID %q", id) + return fmt.Errorf("found multiple SecurityGroups with ID %q", id) } sg := response.SecurityGroups[0] diff --git a/pkg/resources/spotinst/resources.go b/pkg/resources/spotinst/resources.go index f5ad6011e53f5..d35d8f75a90c8 100644 --- a/pkg/resources/spotinst/resources.go +++ b/pkg/resources/spotinst/resources.go @@ -106,7 +106,7 @@ func GetCloudGroups(svc Service, cluster *kops.Cluster, instancegroups []*kops.I if instancegroup == nil { if warnUnmatched { - glog.Warningf("Found group with no corresponding instance group %q", group.Name()) + glog.V(2).Infof("Found group with no corresponding instance group %q", group.Name()) } continue } diff --git a/pkg/util/subnet/BUILD.bazel b/pkg/util/subnet/BUILD.bazel new file mode 100644 index 0000000000000..a34f27752eaa1 --- /dev/null +++ b/pkg/util/subnet/BUILD.bazel @@ -0,0 +1,14 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "go_default_library", + srcs = ["subnet.go"], + importpath = "k8s.io/kops/pkg/util/subnet", + visibility = ["//visibility:public"], +) + +go_test( + name = "go_default_test", + srcs = ["subnet_test.go"], + embed = [":go_default_library"], +) diff --git a/pkg/util/subnet/subnet.go b/pkg/util/subnet/subnet.go new file mode 100644 index 0000000000000..b15651ceacbd5 --- /dev/null +++ b/pkg/util/subnet/subnet.go @@ -0,0 +1,69 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package subnet + +import ( + "encoding/binary" + "fmt" + "net" +) + +// Overlap checks if two subnets overlap +func Overlap(l, r *net.IPNet) bool { + return l.Contains(r.IP) || r.Contains(l.IP) +} + +// BelongsTo checks if child is a subnet of parent +func BelongsTo(parent *net.IPNet, child *net.IPNet) bool { + parentOnes, parentBits := parent.Mask.Size() + childOnes, childBits := child.Mask.Size() + if childBits != parentBits { + return false + } + if parentOnes > childOnes { + return false + } + childMasked := child.IP.Mask(parent.Mask) + parentMasked := parent.IP.Mask(parent.Mask) + return childMasked.Equal(parentMasked) +} + +// SplitInto8 splits the parent IPNet into 8 subnets +func SplitInto8(parent *net.IPNet) ([]*net.IPNet, error) { + networkLength, _ := parent.Mask.Size() + networkLength += 3 + + var subnets []*net.IPNet + for i := 0; i < 8; i++ { + ip4 := parent.IP.To4() + if ip4 != nil { + n := binary.BigEndian.Uint32(ip4) + n += uint32(i) << uint(32-networkLength) + subnetIP := make(net.IP, len(ip4)) + binary.BigEndian.PutUint32(subnetIP, n) + + subnets = append(subnets, &net.IPNet{ + IP: subnetIP, + Mask: net.CIDRMask(networkLength, 32), + }) + } else { + return nil, fmt.Errorf("Unexpected IP address type: %s", parent) + } + } + + return subnets, nil +} diff --git a/pkg/util/subnet/subnet_test.go b/pkg/util/subnet/subnet_test.go new file mode 100644 index 0000000000000..1b3e58b8b6cfa --- /dev/null +++ b/pkg/util/subnet/subnet_test.go @@ -0,0 +1,111 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package subnet + +import ( + "net" + "reflect" + "testing" +) + +func Test_BelongsTo(t *testing.T) { + grid := []struct { + L string + R string + Belongs bool + }{ + { + L: "192.168.1.0/24", + R: "192.168.0.0/24", + Belongs: false, + }, + { + L: "192.168.0.0/16", + R: "192.168.0.0/24", + Belongs: true, + }, + { + L: "192.168.0.0/24", + R: "192.168.0.0/16", + Belongs: false, + }, + { + L: "192.168.0.0/16", + R: "192.168.0.0/16", + Belongs: true, // Not a strict subnet + }, + { + L: "192.168.0.1/16", + R: "192.168.0.0/24", + Belongs: true, + }, + { + L: "0.0.0.0/0", + R: "101.0.1.0/32", + Belongs: true, + }, + } + for _, g := range grid { + _, l, err := net.ParseCIDR(g.L) + if err != nil { + t.Fatalf("error parsing %q: %v", g.L, err) + } + _, r, err := net.ParseCIDR(g.R) + if err != nil { + t.Fatalf("error parsing %q: %v", g.R, err) + } + actual := BelongsTo(l, r) + if actual != g.Belongs { + t.Errorf("isSubnet(%q, %q) = %v, expected %v", g.L, g.R, actual, g.Belongs) + } + } +} + +func Test_SplitInto8(t *testing.T) { + tests := []struct { + parent string + expected []string + }{ + { + parent: "1.2.3.0/24", + expected: []string{"1.2.3.0/27", "1.2.3.32/27", "1.2.3.64/27", "1.2.3.96/27", "1.2.3.128/27", "1.2.3.160/27", "1.2.3.192/27", "1.2.3.224/27"}, + }, + { + parent: "1.2.3.0/27", + expected: []string{"1.2.3.0/30", "1.2.3.4/30", "1.2.3.8/30", "1.2.3.12/30", "1.2.3.16/30", "1.2.3.20/30", "1.2.3.24/30", "1.2.3.28/30"}, + }, + } + for _, test := range tests { + _, parent, err := net.ParseCIDR(test.parent) + if err != nil { + t.Fatalf("error parsing parent cidr %q: %v", test.parent, err) + } + + subnets, err := SplitInto8(parent) + if err != nil { + t.Fatalf("error splitting parent cidr %q: %v", parent, err) + } + + var actual []string + for _, subnet := range subnets { + actual = append(actual, subnet.String()) + } + if !reflect.DeepEqual(actual, test.expected) { + t.Fatalf("unexpected result of split: actual=%v, expected=%v", actual, test.expected) + } + } +} diff --git a/pkg/validation/validate_cluster.go b/pkg/validation/validate_cluster.go index a7c9cd09a16ed..8b8ed4010e654 100644 --- a/pkg/validation/validate_cluster.go +++ b/pkg/validation/validate_cluster.go @@ -66,6 +66,9 @@ func hasPlaceHolderIP(clusterName string) (bool, error) { config, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig( clientcmd.NewDefaultClientConfigLoadingRules(), &clientcmd.ConfigOverrides{CurrentContext: clusterName}).ClientConfig() + if err != nil { + return false, fmt.Errorf("error building configuration: %v", err) + } apiAddr, err := url.Parse(config.Host) if err != nil { diff --git a/tests/integration/create_cluster/minimal/expected-v1alpha1.yaml b/tests/integration/create_cluster/minimal/expected-v1alpha1.yaml index e1f63be799b36..b15bf1583f97a 100644 --- a/tests/integration/create_cluster/minimal/expected-v1alpha1.yaml +++ b/tests/integration/create_cluster/minimal/expected-v1alpha1.yaml @@ -25,7 +25,9 @@ spec: iam: allowContainerRegistry: true legacy: false - kubernetesVersion: v1.12.0 + kubelet: + anonymousAuth: false + kubernetesVersion: v1.11.0 masterPublicName: api.minimal.example.com networkCIDR: 172.20.0.0/16 networking: diff --git a/tests/integration/create_cluster/minimal/expected-v1alpha2.yaml b/tests/integration/create_cluster/minimal/expected-v1alpha2.yaml index 087fb09d4823f..c10c4b58d26df 100644 --- a/tests/integration/create_cluster/minimal/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/minimal/expected-v1alpha2.yaml @@ -23,9 +23,11 @@ spec: iam: allowContainerRegistry: true legacy: false + kubelet: + anonymousAuth: false kubernetesApiAccess: - 0.0.0.0/0 - kubernetesVersion: v1.12.0 + kubernetesVersion: v1.11.0 masterPublicName: api.minimal.example.com networkCIDR: 172.20.0.0/16 networking: diff --git a/tests/integration/create_cluster/minimal/options.yaml b/tests/integration/create_cluster/minimal/options.yaml index be32b6109f346..ffa0cbd53570e 100644 --- a/tests/integration/create_cluster/minimal/options.yaml +++ b/tests/integration/create_cluster/minimal/options.yaml @@ -2,4 +2,4 @@ ClusterName: minimal.example.com Zones: - us-test-1a Cloud: aws -KubernetesVersion: v1.12.0 +KubernetesVersion: v1.11.0 diff --git a/tests/integration/update_cluster/existing_iam_cloudformation/data/aws_launch_configuration_master-us-west-2a.masters.k8s-iam.us-west-2.td.priv_user_data b/tests/integration/update_cluster/existing_iam_cloudformation/data/aws_launch_configuration_master-us-west-2a.masters.k8s-iam.us-west-2.td.priv_user_data index 070f6cdfe36db..85089e632a711 100644 --- a/tests/integration/update_cluster/existing_iam_cloudformation/data/aws_launch_configuration_master-us-west-2a.masters.k8s-iam.us-west-2.td.priv_user_data +++ b/tests/integration/update_cluster/existing_iam_cloudformation/data/aws_launch_configuration_master-us-west-2a.masters.k8s-iam.us-west-2.td.priv_user_data @@ -171,7 +171,6 @@ kubeAPIServer: - Priority - ResourceQuota allowPrivileged: true - anonymousAuth: false apiServerCount: 1 authorizationMode: RBAC cloudProvider: aws diff --git a/upup/models/cloudup/resources/addons/coredns.addons.k8s.io/k8s-1.6.yaml.template b/upup/models/cloudup/resources/addons/coredns.addons.k8s.io/k8s-1.6.yaml.template index c3a7fea17efeb..5ad294aee5dad 100644 --- a/upup/models/cloudup/resources/addons/coredns.addons.k8s.io/k8s-1.6.yaml.template +++ b/upup/models/cloudup/resources/addons/coredns.addons.k8s.io/k8s-1.6.yaml.template @@ -25,6 +25,12 @@ rules: verbs: - list - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get --- apiVersion: rbac.authorization.k8s.io/v1beta1 kind: ClusterRoleBinding @@ -55,7 +61,6 @@ data: Corefile: | .:53 { errors - log health kubernetes {{ KubeDNS.Domain }}. in-addr.arpa ip6.arpa { pods insecure @@ -95,13 +100,13 @@ spec: spec: serviceAccountName: coredns tolerations: - - key: node-role.kubernetes.io/master - effect: NoSchedule - key: "CriticalAddonsOnly" operator: "Exists" + nodeSelector: + beta.kubernetes.io/os: linux containers: - name: coredns - image: k8s.gcr.io/coredns:1.2.2 + image: k8s.gcr.io/coredns:1.2.6 imagePullPolicy: IfNotPresent resources: limits: diff --git a/upup/models/cloudup/resources/addons/dns-controller.addons.k8s.io/k8s-1.6.yaml.template b/upup/models/cloudup/resources/addons/dns-controller.addons.k8s.io/k8s-1.6.yaml.template index 28104844801b6..f9d37b24c17dd 100644 --- a/upup/models/cloudup/resources/addons/dns-controller.addons.k8s.io/k8s-1.6.yaml.template +++ b/upup/models/cloudup/resources/addons/dns-controller.addons.k8s.io/k8s-1.6.yaml.template @@ -6,7 +6,7 @@ metadata: labels: k8s-addon: dns-controller.addons.k8s.io k8s-app: dns-controller - version: v1.11.0-alpha.1 + version: v1.11.0 spec: replicas: 1 selector: @@ -17,7 +17,7 @@ spec: labels: k8s-addon: dns-controller.addons.k8s.io k8s-app: dns-controller - version: v1.11.0-alpha.1 + version: v1.11.0 annotations: scheduler.alpha.kubernetes.io/critical-pod: '' # For 1.6, we keep the old tolerations in case of a downgrade to 1.5 @@ -33,7 +33,7 @@ spec: serviceAccount: dns-controller containers: - name: dns-controller - image: kope/dns-controller:1.11.0-alpha.1 + image: kope/dns-controller:1.11.0 command: {{ range $arg := DnsControllerArgv }} - "{{ $arg }}" diff --git a/upup/models/cloudup/resources/addons/dns-controller.addons.k8s.io/pre-k8s-1.6.yaml.template b/upup/models/cloudup/resources/addons/dns-controller.addons.k8s.io/pre-k8s-1.6.yaml.template index 0152cd5d15912..433a5c526fad5 100644 --- a/upup/models/cloudup/resources/addons/dns-controller.addons.k8s.io/pre-k8s-1.6.yaml.template +++ b/upup/models/cloudup/resources/addons/dns-controller.addons.k8s.io/pre-k8s-1.6.yaml.template @@ -6,7 +6,7 @@ metadata: labels: k8s-addon: dns-controller.addons.k8s.io k8s-app: dns-controller - version: v1.11.0-alpha.1 + version: v1.11.0 spec: replicas: 1 selector: @@ -17,7 +17,7 @@ spec: labels: k8s-addon: dns-controller.addons.k8s.io k8s-app: dns-controller - version: v1.11.0-alpha.1 + version: v1.11.0 annotations: scheduler.alpha.kubernetes.io/critical-pod: '' scheduler.alpha.kubernetes.io/tolerations: '[{"key": "dedicated", "value": "master"}]' @@ -28,7 +28,7 @@ spec: hostNetwork: true containers: - name: dns-controller - image: kope/dns-controller:1.11.0-alpha.1 + image: kope/dns-controller:1.11.0 command: {{ range $arg := DnsControllerArgv }} - "{{ $arg }}" diff --git a/upup/models/cloudup/resources/addons/networking.amazon-vpc-routed-eni/k8s-1.10.yaml.template b/upup/models/cloudup/resources/addons/networking.amazon-vpc-routed-eni/k8s-1.10.yaml.template index fdb59a22c5bde..689a234bcc9e1 100644 --- a/upup/models/cloudup/resources/addons/networking.amazon-vpc-routed-eni/k8s-1.10.yaml.template +++ b/upup/models/cloudup/resources/addons/networking.amazon-vpc-routed-eni/k8s-1.10.yaml.template @@ -1,28 +1,33 @@ -# Vendored from https://github.com/aws/amazon-vpc-cni-k8s/blob/release-1.0/config/v1.0/aws-k8s-cni.yaml +# Vendored from https://github.com/aws/amazon-vpc-cni-k8s/blob/v1.3.0/config/v1.3/aws-k8s-cni.yaml apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: aws-node rules: +- apiGroups: + - crd.k8s.amazonaws.com + resources: + - "*" + - namespaces + verbs: + - "*" - apiGroups: [""] resources: - pods + - nodes - namespaces verbs: ["list", "watch", "get"] - apiGroups: ["extensions"] resources: - daemonsets verbs: ["list", "watch"] - --- - apiVersion: v1 kind: ServiceAccount metadata: name: aws-node namespace: kube-system - --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding @@ -36,9 +41,7 @@ subjects: - kind: ServiceAccount name: aws-node namespace: kube-system - --- - kind: DaemonSet apiVersion: extensions/v1beta1 metadata: @@ -62,14 +65,12 @@ spec: serviceAccountName: aws-node hostNetwork: true tolerations: - - effect: NoSchedule - operator: Exists - - effect: NoExecute - operator: Exists - - key: CriticalAddonsOnly - operator: Exists + - operator: Exists containers: - - image: "{{- or .Networking.AmazonVPC.ImageName "602401143452.dkr.ecr.us-west-2.amazonaws.com/amazon-k8s-cni:1.0.0" }}" + - image: "{{- or .Networking.AmazonVPC.ImageName "602401143452.dkr.ecr.us-west-2.amazonaws.com/amazon-k8s-cni:1.3.0" }}" + ports: + - containerPort: 61678 + name: metrics name: aws-node env: - name: AWS_VPC_K8S_CNI_LOGLEVEL @@ -78,6 +79,10 @@ spec: valueFrom: fieldRef: fieldPath: spec.nodeName + - name: WATCH_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace resources: requests: cpu: 10m @@ -105,4 +110,17 @@ spec: - name: dockersock hostPath: path: /var/run/docker.sock - +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: eniconfigs.crd.k8s.amazonaws.com +spec: + scope: Cluster + group: crd.k8s.amazonaws.com + version: v1alpha1 + names: + scope: Cluster + plural: eniconfigs + singular: eniconfig + kind: ENIConfig diff --git a/upup/models/cloudup/resources/addons/networking.amazon-vpc-routed-eni/k8s-1.7.yaml.template b/upup/models/cloudup/resources/addons/networking.amazon-vpc-routed-eni/k8s-1.7.yaml.template index 6bd26b71561de..ef8ec8079ad14 100644 --- a/upup/models/cloudup/resources/addons/networking.amazon-vpc-routed-eni/k8s-1.7.yaml.template +++ b/upup/models/cloudup/resources/addons/networking.amazon-vpc-routed-eni/k8s-1.7.yaml.template @@ -1,28 +1,33 @@ -# Vendored from https://github.com/aws/amazon-vpc-cni-k8s/blob/release-1.0/config/v1.0/aws-k8s-cni.yaml +# Vendored from https://github.com/aws/amazon-vpc-cni-k8s/blob/v1.3.0/config/v1.3/aws-k8s-cni.yaml apiVersion: rbac.authorization.k8s.io/v1beta1 kind: ClusterRole metadata: name: aws-node rules: +- apiGroups: + - crd.k8s.amazonaws.com + resources: + - "*" + - namespaces + verbs: + - "*" - apiGroups: [""] resources: - pods + - nodes - namespaces verbs: ["list", "watch", "get"] - apiGroups: ["extensions"] resources: - daemonsets verbs: ["list", "watch"] - --- - apiVersion: v1 kind: ServiceAccount metadata: name: aws-node namespace: kube-system - --- apiVersion: rbac.authorization.k8s.io/v1beta1 kind: ClusterRoleBinding @@ -36,9 +41,7 @@ subjects: - kind: ServiceAccount name: aws-node namespace: kube-system - --- - kind: DaemonSet apiVersion: extensions/v1beta1 metadata: @@ -62,14 +65,12 @@ spec: serviceAccountName: aws-node hostNetwork: true tolerations: - - effect: NoSchedule - operator: Exists - - effect: NoExecute - operator: Exists - - key: CriticalAddonsOnly - operator: Exists + - operator: Exists containers: - - image: "{{- or .Networking.AmazonVPC.ImageName "602401143452.dkr.ecr.us-west-2.amazonaws.com/amazon-k8s-cni:1.0.0" }}" + - image: "{{- or .Networking.AmazonVPC.ImageName "602401143452.dkr.ecr.us-west-2.amazonaws.com/amazon-k8s-cni:1.3.0" }}" + ports: + - containerPort: 61678 + name: metrics name: aws-node env: - name: AWS_VPC_K8S_CNI_LOGLEVEL @@ -78,6 +79,10 @@ spec: valueFrom: fieldRef: fieldPath: spec.nodeName + - name: WATCH_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace resources: requests: cpu: 10m @@ -105,6 +110,20 @@ spec: - name: dockersock hostPath: path: /var/run/docker.sock +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: eniconfigs.crd.k8s.amazonaws.com +spec: + scope: Cluster + group: crd.k8s.amazonaws.com + version: v1alpha1 + names: + scope: Cluster + plural: eniconfigs + singular: eniconfig + kind: ENIConfig --- diff --git a/upup/models/cloudup/resources/addons/networking.amazon-vpc-routed-eni/k8s-1.8.yaml.template b/upup/models/cloudup/resources/addons/networking.amazon-vpc-routed-eni/k8s-1.8.yaml.template index d5802b705d0a0..c45ff28b86451 100644 --- a/upup/models/cloudup/resources/addons/networking.amazon-vpc-routed-eni/k8s-1.8.yaml.template +++ b/upup/models/cloudup/resources/addons/networking.amazon-vpc-routed-eni/k8s-1.8.yaml.template @@ -1,28 +1,33 @@ -# Vendored from https://github.com/aws/amazon-vpc-cni-k8s/blob/release-1.0/config/v1.0/aws-k8s-cni.yaml +# Vendored from https://github.com/aws/amazon-vpc-cni-k8s/blob/v1.3.0/config/v1.3/aws-k8s-cni.yaml apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: aws-node rules: +- apiGroups: + - crd.k8s.amazonaws.com + resources: + - "*" + - namespaces + verbs: + - "*" - apiGroups: [""] resources: - pods + - nodes - namespaces verbs: ["list", "watch", "get"] - apiGroups: ["extensions"] resources: - daemonsets verbs: ["list", "watch"] - --- - apiVersion: v1 kind: ServiceAccount metadata: name: aws-node namespace: kube-system - --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding @@ -36,9 +41,7 @@ subjects: - kind: ServiceAccount name: aws-node namespace: kube-system - --- - kind: DaemonSet apiVersion: extensions/v1beta1 metadata: @@ -62,14 +65,12 @@ spec: serviceAccountName: aws-node hostNetwork: true tolerations: - - effect: NoSchedule - operator: Exists - - effect: NoExecute - operator: Exists - - key: CriticalAddonsOnly - operator: Exists + - operator: Exists containers: - - image: "{{- or .Networking.AmazonVPC.ImageName "602401143452.dkr.ecr.us-west-2.amazonaws.com/amazon-k8s-cni:1.0.0" }}" + - image: "{{- or .Networking.AmazonVPC.ImageName "602401143452.dkr.ecr.us-west-2.amazonaws.com/amazon-k8s-cni:1.3.0" }}" + ports: + - containerPort: 61678 + name: metrics name: aws-node env: - name: AWS_VPC_K8S_CNI_LOGLEVEL @@ -78,6 +79,10 @@ spec: valueFrom: fieldRef: fieldPath: spec.nodeName + - name: WATCH_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace resources: requests: cpu: 10m @@ -105,6 +110,20 @@ spec: - name: dockersock hostPath: path: /var/run/docker.sock +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: eniconfigs.crd.k8s.amazonaws.com +spec: + scope: Cluster + group: crd.k8s.amazonaws.com + version: v1alpha1 + names: + scope: Cluster + plural: eniconfigs + singular: eniconfig + kind: ENIConfig --- diff --git a/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.7.yaml.template b/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.7.yaml.template index 6a10414e7830b..43e4d7c69dcec 100644 --- a/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.7.yaml.template +++ b/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.7.yaml.template @@ -243,7 +243,7 @@ spec: - "{{ .PrefilterDevice }}" {{ end }} {{ if ne .PrometheusServeAddr "" }} - - "--prefilter-device" + - "--prometheus-serve-addr" - "{{ .PrometheusServeAddr }}" {{ end }} {{ if .Restore}} diff --git a/upup/models/cloudup/resources/addons/networking.kope.io/k8s-1.6.yaml b/upup/models/cloudup/resources/addons/networking.kope.io/k8s-1.6.yaml index cde015ac5ee9f..5df99fb0462b1 100644 --- a/upup/models/cloudup/resources/addons/networking.kope.io/k8s-1.6.yaml +++ b/upup/models/cloudup/resources/addons/networking.kope.io/k8s-1.6.yaml @@ -28,12 +28,17 @@ spec: memory: 100Mi securityContext: privileged: true - image: kopeio/networking-agent:1.0.20180319 + image: kopeio/networking-agent:1.0.20181028 name: networking-agent volumeMounts: - name: lib-modules mountPath: /lib/modules readOnly: true + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName serviceAccountName: kopeio-networking-agent tolerations: - effect: NoSchedule @@ -74,6 +79,12 @@ rules: - list - watch - patch +- apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch --- diff --git a/upup/models/cloudup/resources/addons/networking.kope.io/pre-k8s-1.6.yaml b/upup/models/cloudup/resources/addons/networking.kope.io/pre-k8s-1.6.yaml index 116d2655b05ef..8babc915ae77f 100644 --- a/upup/models/cloudup/resources/addons/networking.kope.io/pre-k8s-1.6.yaml +++ b/upup/models/cloudup/resources/addons/networking.kope.io/pre-k8s-1.6.yaml @@ -28,7 +28,7 @@ spec: memory: 100Mi securityContext: privileged: true - image: kopeio/networking-agent:1.0.20180319 + image: kopeio/networking-agent:1.0.20181028 name: networking-agent volumeMounts: - name: lib-modules diff --git a/upup/models/cloudup/resources/addons/networking.projectcalico.org.canal/k8s-1.12.yaml.template b/upup/models/cloudup/resources/addons/networking.projectcalico.org.canal/k8s-1.12.yaml.template new file mode 100644 index 0000000000000..b9249de19d18a --- /dev/null +++ b/upup/models/cloudup/resources/addons/networking.projectcalico.org.canal/k8s-1.12.yaml.template @@ -0,0 +1,548 @@ +# Canal Version v3.3.0 +# https://docs.projectcalico.org/v3.3/releases#v3.3.0 +# This manifest includes the following component versions: +# calico/node:v3.3.0 +# calico/cni:v3.3.0 +# coreos/flannel:v0.9.0 + +# This ConfigMap is used to configure a self-hosted Canal installation. +kind: ConfigMap +apiVersion: v1 +metadata: + name: canal-config + namespace: kube-system +data: + # The interface used by canal for host <-> host communication. + # If left blank, then the interface is chosen using the node's + # default route. + canal_iface: "" + + # Whether or not to masquerade traffic to destinations not within + # the pod network. + masquerade: "true" + + # The CNI network configuration to install on each node. The special + # values in this config will be automatically populated. + cni_network_config: |- + { + "name": "k8s-pod-network", + "cniVersion": "0.3.0", + "plugins": [ + { + "type": "calico", + "log_level": "info", + "datastore_type": "kubernetes", + "nodename": "__KUBERNETES_NODE_NAME__", + "ipam": { + "type": "host-local", + "subnet": "usePodCidr" + }, + "policy": { + "type": "k8s" + }, + "kubernetes": { + "kubeconfig": "__KUBECONFIG_FILEPATH__" + } + }, + { + "type": "portmap", + "snat": true, + "capabilities": {"portMappings": true} + } + ] + } + + # Flannel network configuration. Mounted into the flannel container. + net-conf.json: | + { + "Network": "{{ .NonMasqueradeCIDR }}", + "Backend": { + "Type": "vxlan" + } + } + +--- + + + +# This manifest installs the calico/node container, as well +# as the Calico CNI plugins and network config on +# each master and worker node in a Kubernetes cluster. +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: canal + namespace: kube-system + labels: + k8s-app: canal +spec: + selector: + matchLabels: + k8s-app: canal + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + template: + metadata: + labels: + k8s-app: canal + annotations: + # This, along with the CriticalAddonsOnly toleration below, + # marks the pod as a critical add-on, ensuring it gets + # priority scheduling and that its resources are reserved + # if it ever gets evicted. + scheduler.alpha.kubernetes.io/critical-pod: '' + spec: + priorityClassName: system-node-critical + nodeSelector: + beta.kubernetes.io/os: linux + hostNetwork: true + tolerations: + # Make sure canal gets scheduled on all nodes. + - effect: NoSchedule + operator: Exists + # Mark the pod as a critical add-on for rescheduling. + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + serviceAccountName: canal + # Minimize downtime during a rolling upgrade or deletion; tell Kubernetes to do a "force + # deletion": https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods. + terminationGracePeriodSeconds: 0 + containers: + # Runs calico/node container on each Kubernetes node. This + # container programs network policy and routes on each + # host. + - name: calico-node + image: quay.io/calico/node:v3.3.0 + env: + # Use Kubernetes API as the backing datastore. + - name: DATASTORE_TYPE + value: "kubernetes" + # Wait for the datastore. + - name: WAIT_FOR_DATASTORE + value: "true" + # Set based on the k8s node name. + - name: NODENAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + # Don't enable BGP. + - name: CALICO_NETWORKING_BACKEND + value: "none" + # Cluster type to identify the deployment type + - name: CLUSTER_TYPE + value: "k8s,canal" + # Period, in seconds, at which felix re-applies all iptables state + - name: FELIX_IPTABLESREFRESHINTERVAL + value: "60" + # No IP address needed. + - name: IP + value: "" + # Disable file logging so `kubectl logs` works. + - name: CALICO_DISABLE_FILE_LOGGING + value: "true" + # Disable IPv6 on Kubernetes. + - name: FELIX_IPV6SUPPORT + value: "false" + # Set Felix logging to "info" + - name: FELIX_LOGSEVERITYSCREEN + value: "{{- or .Networking.Canal.LogSeveritySys "INFO" }}" + # Set Felix endpoint to host default action to ACCEPT. + - name: FELIX_DEFAULTENDPOINTTOHOSTACTION + value: "{{- or .Networking.Canal.DefaultEndpointToHostAction "ACCEPT" }}" + # Controls whether Felix inserts rules to the top of iptables chains, or appends to the bottom + - name: FELIX_CHAININSERTMODE + value: "{{- or .Networking.Canal.ChainInsertMode "insert" }}" + # Set to enable the experimental Prometheus metrics server + - name: FELIX_PROMETHEUSMETRICSENABLED + value: "{{- or .Networking.Canal.PrometheusMetricsEnabled "false" }}" + # TCP port that the Prometheus metrics server should bind to + - name: FELIX_PROMETHEUSMETRICSPORT + value: "{{- or .Networking.Canal.PrometheusMetricsPort "9091" }}" + # Enable Prometheus Go runtime metrics collection + - name: FELIX_PROMETHEUSGOMETRICSENABLED + value: "{{- or .Networking.Canal.PrometheusGoMetricsEnabled "true" }}" + # Enable Prometheus process metrics collection + - name: FELIX_PROMETHEUSPROCESSMETRICSENABLED + value: "{{- or .Networking.Canal.PrometheusProcessMetricsEnabled "true" }}" + - name: FELIX_HEALTHENABLED + value: "true" + securityContext: + privileged: true + resources: + requests: + cpu: 250m + livenessProbe: + httpGet: + path: /liveness + port: 9099 + host: localhost + periodSeconds: 10 + initialDelaySeconds: 10 + failureThreshold: 6 + readinessProbe: + httpGet: + path: /readiness + port: 9099 + host: localhost + periodSeconds: 10 + volumeMounts: + - mountPath: /lib/modules + name: lib-modules + readOnly: true + - mountPath: /run/xtables.lock + name: xtables-lock + readOnly: false + - mountPath: /var/run/calico + name: var-run-calico + readOnly: false + - mountPath: /var/lib/calico + name: var-lib-calico + readOnly: false + # This container installs the Calico CNI binaries + # and CNI network config file on each node. + - name: install-cni + image: quay.io/calico/cni:v3.3.0 + command: ["/install-cni.sh"] + env: + # Name of the CNI config file to create. + - name: CNI_CONF_NAME + value: "10-canal.conflist" + # Set the hostname based on the k8s node name. + - name: KUBERNETES_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + # The CNI network config to install on each node. + - name: CNI_NETWORK_CONFIG + valueFrom: + configMapKeyRef: + name: canal-config + key: cni_network_config + volumeMounts: + - mountPath: /host/opt/cni/bin + name: cni-bin-dir + - mountPath: /host/etc/cni/net.d + name: cni-net-dir + # This container runs flannel using the kube-subnet-mgr backend + # for allocating subnets. + - name: kube-flannel + image: quay.io/coreos/flannel:v0.9.0 + command: [ "/opt/bin/flanneld", "--ip-masq", "--kube-subnet-mgr" ] + securityContext: + privileged: true + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: FLANNELD_IFACE + valueFrom: + configMapKeyRef: + name: canal-config + key: canal_iface + - name: FLANNELD_IP_MASQ + valueFrom: + configMapKeyRef: + name: canal-config + key: masquerade + volumeMounts: + - mountPath: /run/xtables.lock + name: xtables-lock + readOnly: false + - name: flannel-cfg + mountPath: /etc/kube-flannel/ + volumes: + # Used by calico/node. + - name: lib-modules + hostPath: + path: /lib/modules + - name: var-run-calico + hostPath: + path: /var/run/calico + - name: var-lib-calico + hostPath: + path: /var/lib/calico + - name: xtables-lock + hostPath: + path: /run/xtables.lock + type: FileOrCreate + # Used by flannel. + - name: flannel-cfg + configMap: + name: canal-config + # Used to install CNI. + - name: cni-bin-dir + hostPath: + path: /opt/cni/bin + - name: cni-net-dir + hostPath: + path: /etc/cni/net.d +--- + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: canal + namespace: kube-system + +--- + +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: calico +rules: + - apiGroups: [""] + resources: + - namespaces + - serviceaccounts + verbs: + - get + - list + - watch + - apiGroups: [""] + resources: + - pods/status + verbs: + - patch + - apiGroups: [""] + resources: + - pods + verbs: + - get + - list + - watch + - apiGroups: [""] + resources: + - services + verbs: + - get + - apiGroups: [""] + resources: + - endpoints + verbs: + - get + - apiGroups: [""] + resources: + - nodes + verbs: + - get + - list + - update + - watch + - apiGroups: ["networking.k8s.io"] + resources: + - networkpolicies + verbs: + - get + - list + - watch + - apiGroups: ["crd.projectcalico.org"] + resources: + - globalfelixconfigs + - felixconfigurations + - bgppeers + - globalbgpconfigs + - globalnetworksets + - hostendpoints + - bgpconfigurations + - ippools + - globalnetworkpolicies + - networkpolicies + - clusterinformations + verbs: + - create + - get + - list + - update + - watch + +--- + +# Flannel roles +# Pulled from https://github.com/coreos/flannel/blob/master/Documentation/kube-flannel-rbac.yml +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: flannel +rules: + - apiGroups: + - "" + resources: + - pods + verbs: + - get + - apiGroups: + - "" + resources: + - nodes + verbs: + - list + - watch + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch +--- + +# Bind the flannel ClusterRole to the canal ServiceAccount. +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: canal-flannel +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: flannel +subjects: +- kind: ServiceAccount + name: canal + namespace: kube-system + +--- + +# Bind the ClusterRole to the canal ServiceAccount. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: canal-calico +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: calico +subjects: +- kind: ServiceAccount + name: canal + namespace: kube-system + +--- + +# Create all the CustomResourceDefinitions needed for +# Calico policy and networking mode. + +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: felixconfigurations.crd.projectcalico.org +spec: + scope: Cluster + group: crd.projectcalico.org + version: v1 + names: + kind: FelixConfiguration + plural: felixconfigurations + singular: felixconfiguration +--- + +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: bgpconfigurations.crd.projectcalico.org +spec: + scope: Cluster + group: crd.projectcalico.org + version: v1 + names: + kind: BGPConfiguration + plural: bgpconfigurations + singular: bgpconfiguration + +--- + +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: ippools.crd.projectcalico.org +spec: + scope: Cluster + group: crd.projectcalico.org + version: v1 + names: + kind: IPPool + plural: ippools + singular: ippool + +--- + +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: hostendpoints.crd.projectcalico.org +spec: + scope: Cluster + group: crd.projectcalico.org + version: v1 + names: + kind: HostEndpoint + plural: hostendpoints + singular: hostendpoint + +--- + +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: clusterinformations.crd.projectcalico.org +spec: + scope: Cluster + group: crd.projectcalico.org + version: v1 + names: + kind: ClusterInformation + plural: clusterinformations + singular: clusterinformation + +--- + +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: globalnetworkpolicies.crd.projectcalico.org +spec: + scope: Cluster + group: crd.projectcalico.org + version: v1 + names: + kind: GlobalNetworkPolicy + plural: globalnetworkpolicies + singular: globalnetworkpolicy + +--- + +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: globalnetworksets.crd.projectcalico.org +spec: + scope: Cluster + group: crd.projectcalico.org + version: v1 + names: + kind: GlobalNetworkSet + plural: globalnetworksets + singular: globalnetworkset + +--- + +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: networkpolicies.crd.projectcalico.org +spec: + scope: Namespaced + group: crd.projectcalico.org + version: v1 + names: + kind: NetworkPolicy + plural: networkpolicies + singular: networkpolicy diff --git a/upup/models/cloudup/resources/addons/networking.projectcalico.org/k8s-1.6.yaml.template b/upup/models/cloudup/resources/addons/networking.projectcalico.org/k8s-1.6.yaml.template index 0a641b3af031d..6c97cb9c5795e 100644 --- a/upup/models/cloudup/resources/addons/networking.projectcalico.org/k8s-1.6.yaml.template +++ b/upup/models/cloudup/resources/addons/networking.projectcalico.org/k8s-1.6.yaml.template @@ -141,7 +141,7 @@ spec: # container programs network policy and routes on each # host. - name: calico-node - image: quay.io/calico/node:v2.6.7 + image: quay.io/calico/node:v2.6.9 resources: requests: cpu: 10m @@ -226,7 +226,7 @@ spec: # This container installs the Calico CNI binaries # and CNI network config file on each node. - name: install-cni - image: quay.io/calico/cni:v1.11.2 + image: quay.io/calico/cni:v1.11.5 resources: requests: cpu: 10m @@ -379,7 +379,7 @@ spec: operator: Exists containers: - name: calico-kube-controllers - image: quay.io/calico/kube-controllers:v1.0.3 + image: quay.io/calico/kube-controllers:v1.0.4 resources: requests: cpu: 10m diff --git a/upup/models/cloudup/resources/addons/networking.projectcalico.org/k8s-1.7-v3.yaml.template b/upup/models/cloudup/resources/addons/networking.projectcalico.org/k8s-1.7-v3.yaml.template new file mode 100644 index 0000000000000..8848988370b1f --- /dev/null +++ b/upup/models/cloudup/resources/addons/networking.projectcalico.org/k8s-1.7-v3.yaml.template @@ -0,0 +1,749 @@ +{{- $etcd_scheme := EtcdScheme }} +# This ConfigMap is used to configure a self-hosted Calico installation. +kind: ConfigMap +apiVersion: v1 +metadata: + name: calico-config + namespace: kube-system +data: + # The calico-etcd PetSet service IP:port + etcd_endpoints: "{{ $cluster := index .EtcdClusters 0 -}} + {{- range $j, $member := $cluster.Members -}} + {{- if $j }},{{ end -}} + {{ $etcd_scheme }}://etcd-{{ $member.Name }}.internal.{{ ClusterName }}:4001 + {{- end }}" + + # Configure the Calico backend to use. + calico_backend: "bird" + + # The CNI network configuration to install on each node. + cni_network_config: |- + { + "name": "k8s-pod-network", + "cniVersion": "0.3.0", + "plugins": [ + { + "type": "calico", + "etcd_endpoints": "__ETCD_ENDPOINTS__", + {{- if eq $etcd_scheme "https" }} + "etcd_ca_cert_file": "/srv/kubernetes/calico/ca.pem", + "etcd_cert_file": "/srv/kubernetes/calico/calico-client.pem", + "etcd_key_file": "/srv/kubernetes/calico/calico-client-key.pem", + "etcd_scheme": "https", + {{- end }} + "log_level": "info", + "ipam": { + "type": "calico-ipam" + }, + "policy": { + "type": "k8s" + }, + "kubernetes": { + "kubeconfig": "/etc/cni/net.d/__KUBECONFIG_FILENAME__" + } + }, + { + "type": "portmap", + "snat": true, + "capabilities": {"portMappings": true} + } + ] + } + +--- + +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: calico-node + labels: + role.kubernetes.io/networking: "1" +rules: + - apiGroups: [""] + resources: + - pods + - nodes + - namespaces + verbs: + - get +--- + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: calico-node + namespace: kube-system + labels: + role.kubernetes.io/networking: "1" +--- + +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: calico-node + labels: + role.kubernetes.io/networking: "1" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: calico-node +subjects: +- kind: ServiceAccount + name: calico-node + namespace: kube-system +--- + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: calico-kube-controllers + namespace: kube-system + labels: + role.kubernetes.io/networking: "1" +--- + +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: calico-kube-controllers + labels: + role.kubernetes.io/networking: "1" +rules: + - apiGroups: + - "" + - extensions + resources: + - pods + - namespaces + - networkpolicies + - nodes + verbs: + - watch + - list + - apiGroups: + - networking.k8s.io + resources: + - networkpolicies + verbs: + - watch + - list +--- + +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: calico-kube-controllers + labels: + role.kubernetes.io/networking: "1" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: calico-kube-controllers +subjects: +- kind: ServiceAccount + name: calico-kube-controllers + namespace: kube-system + +--- + +# This manifest installs the calico/node container, as well +# as the Calico CNI plugins and network config on +# each master and worker node in a Kubernetes cluster. +kind: DaemonSet +apiVersion: extensions/v1beta1 +metadata: + name: calico-node + namespace: kube-system + labels: + k8s-app: calico-node + role.kubernetes.io/networking: "1" +spec: + selector: + matchLabels: + k8s-app: calico-node + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + template: + metadata: + labels: + k8s-app: calico-node + role.kubernetes.io/networking: "1" + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + spec: + hostNetwork: true + tolerations: + # Make sure calico/node gets scheduled on all nodes. + - effect: NoSchedule + operator: Exists + # Mark the pod as a critical add-on for rescheduling. + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + serviceAccountName: calico-node + # Minimize downtime during a rolling upgrade or deletion; tell Kubernetes to do a "force + # deletion": https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods. + terminationGracePeriodSeconds: 0 + containers: + # Runs calico/node container on each Kubernetes node. This + # container programs network policy and routes on each + # host. + - name: calico-node + image: quay.io/calico/node:v3.3.1 + env: + # The location of the Calico etcd cluster. + - name: ETCD_ENDPOINTS + valueFrom: + configMapKeyRef: + name: calico-config + key: etcd_endpoints + {{- if eq $etcd_scheme "https" }} + - name: ETCD_CERT_FILE + value: /certs/calico-client.pem + - name: ETCD_KEY_FILE + value: /certs/calico-client-key.pem + - name: ETCD_CA_CERT_FILE + value: /certs/ca.pem + {{- end }} + # Choose the backend to use. + - name: CALICO_NETWORKING_BACKEND + valueFrom: + configMapKeyRef: + name: calico-config + key: calico_backend + # Cluster type to identify the deployment type + - name: CLUSTER_TYPE + value: "kops,bgp" + # Disable file logging so `kubectl logs` works. + - name: CALICO_DISABLE_FILE_LOGGING + value: "true" + # Set noderef for node controller. + - name: CALICO_K8S_NODE_REF + valueFrom: + fieldRef: + fieldPath: spec.nodeName + # Set Felix endpoint to host default action to ACCEPT. + - name: FELIX_DEFAULTENDPOINTTOHOSTACTION + value: "ACCEPT" + # The default IPv4 pool to create on startup if none exists. Pod IPs will be + # chosen from this range. Changing this value after installation will have + # no effect. This should fall within `--cluster-cidr`. + # Configure the IP Pool from which Pod IPs will be chosen. + - name: CALICO_IPV4POOL_CIDR + value: "{{ .KubeControllerManager.ClusterCIDR }}" + - name: CALICO_IPV4POOL_IPIP + value: "{{- if and (eq .CloudProvider "aws") (.Networking.Calico.CrossSubnet) -}}cross-subnet{{- else -}}always{{- end -}}" + # Disable IPv6 on Kubernetes. + - name: FELIX_IPV6SUPPORT + value: "false" + # Set Felix logging to the desired level + - name: FELIX_LOGSEVERITYSCREEN + value: "{{- or .Networking.Calico.LogSeverityScreen "info" }}" + # Set to enable the experimental Prometheus metrics server + - name: FELIX_PROMETHEUSMETRICSENABLED + value: "{{- or .Networking.Calico.PrometheusMetricsEnabled "false" }}" + # TCP port that the Prometheus metrics server should bind to + - name: FELIX_PROMETHEUSMETRICSPORT + value: "{{- or .Networking.Calico.PrometheusMetricsPort "9091" }}" + # Enable Prometheus Go runtime metrics collection + - name: FELIX_PROMETHEUSGOMETRICSENABLED + value: "{{- or .Networking.Calico.PrometheusGoMetricsEnabled "true" }}" + # Enable Prometheus process metrics collection + - name: FELIX_PROMETHEUSPROCESSMETRICSENABLED + value: "{{- or .Networking.Calico.PrometheusProcessMetricsEnabled "true" }}" + # Auto-detect the BGP IP address. + - name: IP + value: "autodetect" + - name: FELIX_HEALTHENABLED + value: "true" + securityContext: + privileged: true + resources: + requests: + cpu: 10m + livenessProbe: + httpGet: + path: /liveness + port: 9099 + host: localhost + periodSeconds: 10 + initialDelaySeconds: 10 + failureThreshold: 6 + readinessProbe: + exec: + command: + - /bin/calico-node + - -bird-ready + - -felix-ready + periodSeconds: 10 + volumeMounts: + - mountPath: /lib/modules + name: lib-modules + readOnly: true + - mountPath: /var/run/calico + name: var-run-calico + readOnly: false + - mountPath: /var/lib/calico + name: var-lib-calico + readOnly: false + # Necessary for gossip based DNS + - mountPath: /etc/hosts + name: etc-hosts + readOnly: true + {{- if eq $etcd_scheme "https" }} + - mountPath: /certs + name: calico + readOnly: true + {{- end }} + # This container installs the Calico CNI binaries + # and CNI network config file on each node. + - name: install-cni + image: quay.io/calico/cni:v3.3.1 + command: ["/install-cni.sh"] + env: + # Name of the CNI config file to create. + - name: CNI_CONF_NAME + value: "10-calico.conflist" + # The location of the Calico etcd cluster. + - name: ETCD_ENDPOINTS + valueFrom: + configMapKeyRef: + name: calico-config + key: etcd_endpoints + # The CNI network config to install on each node. + - name: CNI_NETWORK_CONFIG + valueFrom: + configMapKeyRef: + name: calico-config + key: cni_network_config + volumeMounts: + - mountPath: /host/opt/cni/bin + name: cni-bin-dir + - mountPath: /host/etc/cni/net.d + name: cni-net-dir + # Necessary for gossip based DNS + - mountPath: /etc/hosts + name: etc-hosts + readOnly: true + resources: + requests: + cpu: 10m + initContainers: + - name: migrate + image: calico/upgrade:v1.0.5 + command: ['/bin/sh', '-c', '/node-init-container.sh'] + env: + - name: CALICO_ETCD_ENDPOINTS + valueFrom: + configMapKeyRef: + name: calico-config + key: etcd_endpoints + - name: CALICO_APIV1_DATASTORE_TYPE + value: "etcdv2" + - name: CALICO_APIV1_ETCD_ENDPOINTS + valueFrom: + configMapKeyRef: + name: calico-config + key: etcd_endpoints + {{- if eq $etcd_scheme "https" }} + - name: CALICO_ETCD_CERT_FILE + value: /certs/calico-client.pem + - name: CALICO_ETCD_KEY_FILE + value: /certs/calico-client-key.pem + - name: CALICO_ETCD_CA_CERT_FILE + value: /certs/ca.pem + - name: CALICO_APIV1_ETCD_CERT_FILE + value: /certs/calico-client.pem + - name: CALICO_APIV1_ETCD_KEY_FILE + value: /certs/calico-client-key.pem + - name: CALICO_APIV1_ETCD_CA_CERT_FILE + value: /certs/ca.pem + {{- end }} + volumeMounts: + # Necessary for gossip based DNS + - mountPath: /etc/hosts + name: etc-hosts + readOnly: true + {{- if eq $etcd_scheme "https" }} + - mountPath: /certs + name: calico + readOnly: true + {{- end }} + volumes: + # Used by calico/node. + - name: lib-modules + hostPath: + path: /lib/modules + - name: var-run-calico + hostPath: + path: /var/run/calico + - name: var-lib-calico + hostPath: + path: /var/lib/calico + # Used to install CNI. + - name: cni-bin-dir + hostPath: + path: /opt/cni/bin + - name: cni-net-dir + hostPath: + path: /etc/cni/net.d + # Necessary for gossip based DNS + - name: etc-hosts + hostPath: + path: /etc/hosts + {{- if eq $etcd_scheme "https" }} + - name: calico + hostPath: + path: /srv/kubernetes/calico + {{- end }} + +--- + +# This manifest deploys the Calico Kubernetes controllers. +# See https://github.com/projectcalico/kube-controllers +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: calico-kube-controllers + namespace: kube-system + labels: + k8s-app: calico-kube-controllers + role.kubernetes.io/networking: "1" + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' +spec: + # The controllers can only have a single active instance. + replicas: 1 + strategy: + type: Recreate + template: + metadata: + name: calico-kube-controllers + namespace: kube-system + labels: + k8s-app: calico-kube-controllers + role.kubernetes.io/networking: "1" + spec: + # The controllers must run in the host network namespace so that + # it isn't governed by policy that would prevent it from working. + hostNetwork: true + tolerations: + # Mark the pod as a critical add-on for rescheduling. + - key: CriticalAddonsOnly + operator: Exists + - key: node-role.kubernetes.io/master + effect: NoSchedule + serviceAccountName: calico-kube-controllers + containers: + - name: calico-kube-controllers + image: quay.io/calico/kube-controllers:v3.3.1 + resources: + requests: + cpu: 10m + env: + # The location of the Calico etcd cluster. + - name: ETCD_ENDPOINTS + valueFrom: + configMapKeyRef: + name: calico-config + key: etcd_endpoints + # Choose which controllers to run. + - name: ENABLED_CONTROLLERS + value: policy,profile,workloadendpoint,node + {{- if eq $etcd_scheme "https" }} + - name: ETCD_CERT_FILE + value: /certs/calico-client.pem + - name: ETCD_KEY_FILE + value: /certs/calico-client-key.pem + - name: ETCD_CA_CERT_FILE + value: /certs/ca.pem + volumeMounts: + - mountPath: /certs + name: calico + readOnly: true + {{- end }} + readinessProbe: + exec: + command: + - /usr/bin/check-status + - -r + initContainers: + - name: migrate + image: calico/upgrade:v1.0.5 + command: ['/bin/sh', '-c', '/controller-init.sh'] + env: + - name: CALICO_ETCD_ENDPOINTS + valueFrom: + configMapKeyRef: + name: calico-config + key: etcd_endpoints + - name: CALICO_APIV1_DATASTORE_TYPE + value: "etcdv2" + - name: CALICO_APIV1_ETCD_ENDPOINTS + valueFrom: + configMapKeyRef: + name: calico-config + key: etcd_endpoints + {{- if eq $etcd_scheme "https" }} + - name: CALICO_ETCD_CERT_FILE + value: /certs/calico-client.pem + - name: CALICO_ETCD_KEY_FILE + value: /certs/calico-client-key.pem + - name: CALICO_ETCD_CA_CERT_FILE + value: /certs/ca.pem + - name: CALICO_APIV1_ETCD_CERT_FILE + value: /certs/calico-client.pem + - name: CALICO_APIV1_ETCD_KEY_FILE + value: /certs/calico-client-key.pem + - name: CALICO_APIV1_ETCD_CA_CERT_FILE + value: /certs/ca.pem + {{- end }} + volumeMounts: + # Necessary for gossip based DNS + - mountPath: /etc/hosts + name: etc-hosts + readOnly: true + {{- if eq $etcd_scheme "https" }} + - mountPath: /certs + name: calico + readOnly: true + {{- end }} + volumes: + # Necessary for gossip based DNS + - name: etc-hosts + hostPath: + path: /etc/hosts + {{- if eq $etcd_scheme "https" }} + - name: calico + hostPath: + path: /srv/kubernetes/calico + {{- end }} + +# This manifest runs the Migration complete container that monitors for the +# completion of the calico-node Daemonset rollout and when it finishes +# successfully rolling out it will mark the migration complete and allow pods +# to be created again. +--- + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: calico-upgrade-job + namespace: kube-system + labels: + role.kubernetes.io/networking: "1" +--- + +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: calico-upgrade-job + labels: + role.kubernetes.io/networking: "1" +rules: + - apiGroups: + - extensions + resources: + - daemonsets + - daemonsets/status + verbs: + - get + - list + - watch +--- + +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: calico-upgrade-job + labels: + role.kubernetes.io/networking: "1" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: calico-upgrade-job +subjects: +- kind: ServiceAccount + name: calico-upgrade-job + namespace: kube-system +--- +# If anything in this job is changed then the name of the job +# should be changed because Jobs cannot be updated, so changing +# the name would run a different Job if the previous version had been +# created before and it does not hurt to rerun this job. + +apiVersion: batch/v1 +kind: Job +metadata: + name: calico-complete-upgrade-v331 + namespace: kube-system + labels: + role.kubernetes.io/networking: "1" +spec: + template: + metadata: + labels: + role.kubernetes.io/networking: "1" + spec: + hostNetwork: true + serviceAccountName: calico-upgrade-job + restartPolicy: OnFailure + containers: + - name: migrate-completion + image: calico/upgrade:v1.0.5 + command: ['/bin/sh', '-c', '/completion-job.sh'] + env: + - name: EXPECTED_NODE_IMAGE + value: quay.io/calico/node:v3.3.1 + # The location of the Calico etcd cluster. + - name: CALICO_ETCD_ENDPOINTS + valueFrom: + configMapKeyRef: + name: calico-config + key: etcd_endpoints + - name: CALICO_APIV1_DATASTORE_TYPE + value: "etcdv2" + - name: CALICO_APIV1_ETCD_ENDPOINTS + valueFrom: + configMapKeyRef: + name: calico-config + key: etcd_endpoints + {{- if eq $etcd_scheme "https" }} + - name: CALICO_ETCD_CERT_FILE + value: /certs/calico-client.pem + - name: CALICO_ETCD_KEY_FILE + value: /certs/calico-client-key.pem + - name: CALICO_ETCD_CA_CERT_FILE + value: /certs/ca.pem + - name: CALICO_APIV1_ETCD_CERT_FILE + value: /certs/calico-client.pem + - name: CALICO_APIV1_ETCD_KEY_FILE + value: /certs/calico-client-key.pem + - name: CALICO_APIV1_ETCD_CA_CERT_FILE + value: /certs/ca.pem + {{- end }} + volumeMounts: + # Necessary for gossip based DNS + - mountPath: /etc/hosts + name: etc-hosts + readOnly: true + {{- if eq $etcd_scheme "https" }} + - mountPath: /certs + name: calico + readOnly: true + {{- end }} + volumes: + - name: etc-hosts + hostPath: + path: /etc/hosts + {{- if eq $etcd_scheme "https" }} + - name: calico + hostPath: + path: /srv/kubernetes/calico + {{- end }} + +{{ if and (eq .CloudProvider "aws") (.Networking.Calico.CrossSubnet) -}} +# This manifest installs the k8s-ec2-srcdst container, which disables +# src/dst ip checks to allow BGP to function for calico for hosts within subnets +# This only applies for AWS environments. +--- + +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: k8s-ec2-srcdst + labels: + role.kubernetes.io/networking: "1" +rules: +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch + - update + - patch + +--- + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: k8s-ec2-srcdst + namespace: kube-system + labels: + role.kubernetes.io/networking: "1" +--- + +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: k8s-ec2-srcdst + labels: + role.kubernetes.io/networking: "1" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: k8s-ec2-srcdst +subjects: +- kind: ServiceAccount + name: k8s-ec2-srcdst + namespace: kube-system + +--- + +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: k8s-ec2-srcdst + namespace: kube-system + labels: + k8s-app: k8s-ec2-srcdst + role.kubernetes.io/networking: "1" +spec: + replicas: 1 + selector: + matchLabels: + k8s-app: k8s-ec2-srcdst + template: + metadata: + labels: + k8s-app: k8s-ec2-srcdst + role.kubernetes.io/networking: "1" + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + spec: + hostNetwork: true + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + - key: CriticalAddonsOnly + operator: Exists + serviceAccountName: k8s-ec2-srcdst + containers: + - image: ottoyiu/k8s-ec2-srcdst:v0.2.1 + name: k8s-ec2-srcdst + resources: + requests: + cpu: 10m + memory: 64Mi + env: + - name: AWS_REGION + value: {{ Region }} + volumeMounts: + - name: ssl-certs + mountPath: "/etc/ssl/certs/ca-certificates.crt" + readOnly: true + imagePullPolicy: "Always" + volumes: + - name: ssl-certs + hostPath: + path: "/etc/ssl/certs/ca-certificates.crt" + nodeSelector: + node-role.kubernetes.io/master: "" +{{- end -}} diff --git a/upup/models/cloudup/resources/addons/networking.projectcalico.org/k8s-1.7.yaml.template b/upup/models/cloudup/resources/addons/networking.projectcalico.org/k8s-1.7.yaml.template index 9af6738bf0b43..6b1f3fc6f6433 100644 --- a/upup/models/cloudup/resources/addons/networking.projectcalico.org/k8s-1.7.yaml.template +++ b/upup/models/cloudup/resources/addons/networking.projectcalico.org/k8s-1.7.yaml.template @@ -155,7 +155,7 @@ spec: # container programs network policy and routes on each # host. - name: calico-node - image: quay.io/calico/node:v2.6.7 + image: quay.io/calico/node:v2.6.9 resources: requests: cpu: 10m @@ -244,7 +244,7 @@ spec: # This container installs the Calico CNI binaries # and CNI network config file on each node. - name: install-cni - image: quay.io/calico/cni:v1.11.2 + image: quay.io/calico/cni:v1.11.5 resources: requests: cpu: 10m @@ -314,6 +314,8 @@ metadata: spec: # The controllers can only have a single active instance. replicas: 1 + strategy: + type: Recreate template: metadata: name: calico-kube-controllers @@ -335,7 +337,7 @@ spec: operator: Exists containers: - name: calico-kube-controllers - image: quay.io/calico/kube-controllers:v1.0.3 + image: quay.io/calico/kube-controllers:v1.0.4 resources: requests: cpu: 10m @@ -360,7 +362,13 @@ spec: value: /certs/calico-client-key.pem - name: ETCD_CA_CERT_FILE value: /certs/ca.pem + {{- end }} volumeMounts: + # Necessary for gossip based DNS + - mountPath: /etc/hosts + name: etc-hosts + readOnly: true + {{- if eq $etcd_scheme "https" }} - mountPath: /certs name: calico readOnly: true diff --git a/upup/models/cloudup/resources/addons/networking.weave/k8s-1.7.yaml.template b/upup/models/cloudup/resources/addons/networking.weave/k8s-1.7.yaml.template index e1a0fe384bda1..9fe75e06bda1c 100644 --- a/upup/models/cloudup/resources/addons/networking.weave/k8s-1.7.yaml.template +++ b/upup/models/cloudup/resources/addons/networking.weave/k8s-1.7.yaml.template @@ -155,7 +155,7 @@ spec: name: weave-net key: network-password {{- end }} - image: 'weaveworks/weave-kube:2.4.1' + image: 'weaveworks/weave-kube:2.5.0' livenessProbe: httpGet: host: 127.0.0.1 @@ -193,7 +193,7 @@ spec: fieldRef: apiVersion: v1 fieldPath: spec.nodeName - image: 'weaveworks/weave-npc:2.4.1' + image: 'weaveworks/weave-npc:2.5.0' resources: requests: cpu: 50m diff --git a/upup/models/cloudup/resources/addons/networking.weave/k8s-1.8.yaml.template b/upup/models/cloudup/resources/addons/networking.weave/k8s-1.8.yaml.template index e52c78e2b7140..8a94036650306 100644 --- a/upup/models/cloudup/resources/addons/networking.weave/k8s-1.8.yaml.template +++ b/upup/models/cloudup/resources/addons/networking.weave/k8s-1.8.yaml.template @@ -159,7 +159,7 @@ spec: name: weave-net key: network-password {{- end }} - image: 'weaveworks/weave-kube:2.4.1' + image: 'weaveworks/weave-kube:2.5.0' livenessProbe: httpGet: host: 127.0.0.1 @@ -197,7 +197,7 @@ spec: fieldRef: apiVersion: v1 fieldPath: spec.nodeName - image: 'weaveworks/weave-npc:2.4.1' + image: 'weaveworks/weave-npc:2.5.0' resources: requests: cpu: 50m diff --git a/upup/models/cloudup/resources/addons/spotinst-kubernetes-cluster-controller.addons.k8s.io/v1.8.0.yaml.template b/upup/models/cloudup/resources/addons/spotinst-kubernetes-cluster-controller.addons.k8s.io/v1.8.0.yaml.template index f8146d93c236e..4f2cbb0838226 100644 --- a/upup/models/cloudup/resources/addons/spotinst-kubernetes-cluster-controller.addons.k8s.io/v1.8.0.yaml.template +++ b/upup/models/cloudup/resources/addons/spotinst-kubernetes-cluster-controller.addons.k8s.io/v1.8.0.yaml.template @@ -97,7 +97,7 @@ spec: spec: containers: - name: spotinst-kubernetes-cluster-controller - image: spotinst/kubernetes-cluster-controller:1.0.16 + image: spotinst/kubernetes-cluster-controller:1.0.18 imagePullPolicy: Always env: - name: SPOTINST_TOKEN diff --git a/upup/models/cloudup/resources/addons/spotinst-kubernetes-cluster-controller.addons.k8s.io/v1.9.0.yaml.template b/upup/models/cloudup/resources/addons/spotinst-kubernetes-cluster-controller.addons.k8s.io/v1.9.0.yaml.template index 832a4e60e03bc..539315499dbd7 100644 --- a/upup/models/cloudup/resources/addons/spotinst-kubernetes-cluster-controller.addons.k8s.io/v1.9.0.yaml.template +++ b/upup/models/cloudup/resources/addons/spotinst-kubernetes-cluster-controller.addons.k8s.io/v1.9.0.yaml.template @@ -97,7 +97,7 @@ spec: spec: containers: - name: spotinst-kubernetes-cluster-controller - image: spotinst/kubernetes-cluster-controller:1.0.16 + image: spotinst/kubernetes-cluster-controller:1.0.18 imagePullPolicy: Always env: - name: SPOTINST_TOKEN diff --git a/upup/models/nodeup/docker/_systemd/_debian_family/files/opt/kubernetes/helpers/docker-healthcheck b/upup/models/nodeup/docker/_systemd/_debian_family/files/opt/kubernetes/helpers/docker-healthcheck index c21bf3ee2c3fc..87ae0413b8943 100644 --- a/upup/models/nodeup/docker/_systemd/_debian_family/files/opt/kubernetes/helpers/docker-healthcheck +++ b/upup/models/nodeup/docker/_systemd/_debian_family/files/opt/kubernetes/helpers/docker-healthcheck @@ -17,7 +17,7 @@ # This script is intended to be run periodically, to check the health # of docker. If it detects a failure, it will restart docker using systemctl. -if timeout 10 docker ps > /dev/null; then +if timeout 60 docker ps > /dev/null; then echo "docker healthy" exit 0 fi @@ -26,20 +26,26 @@ echo "docker failed" echo "Giving docker 30 seconds grace before restarting" sleep 30 -if timeout 10 docker ps > /dev/null; then +if timeout 60 docker ps > /dev/null; then echo "docker recovered" exit 0 fi -echo "docker still down; triggering docker restart" -systemctl restart docker +echo "docker still unresposive; triggering docker restart" +systemctl stop docker -echo "Waiting 60 seconds to give docker time to start" +echo "wait all tcp sockets to close" +sleep `cat /proc/sys/net/ipv4/tcp_fin_timeout` + +sleep 10 +systemctl start docker + +echo "Waiting 120 seconds to give docker time to start" sleep 60 -if timeout 10 docker ps > /dev/null; then +if timeout 60 docker ps > /dev/null; then echo "docker recovered" exit 0 fi -echo "docker still failing" \ No newline at end of file +echo "docker still failing" diff --git a/upup/models/nodeup/resources/_lyft_vpc_cni/files/etc/cni/net.d/10-cni-ipvlan-vpc-k8s.conflist.template b/upup/models/nodeup/resources/_lyft_vpc_cni/files/etc/cni/net.d/10-cni-ipvlan-vpc-k8s.conflist.template new file mode 100644 index 0000000000000..7b2077fa8b813 --- /dev/null +++ b/upup/models/nodeup/resources/_lyft_vpc_cni/files/etc/cni/net.d/10-cni-ipvlan-vpc-k8s.conflist.template @@ -0,0 +1,26 @@ +{ + "cniVersion": "0.3.1", + "name": "cni-ipvlan-vpc-k8s", + "plugins": [ + { + "cniVersion": "0.3.1", + "type": "cni-ipvlan-vpc-k8s-ipam", + "interfaceIndex": 1, + "skipDeallocation" : true, + "subnetTags": {{ SubnetTags }}, + "secGroupIds": {{ NodeSecurityGroups }} + }, + { + "cniVersion": "0.3.1", + "type": "cni-ipvlan-vpc-k8s-ipvlan", + "mode": "l2" + }, + { + "cniVersion": "0.3.1", + "type": "cni-ipvlan-vpc-k8s-unnumbered-ptp", + "hostInterface": "eth0", + "containerInterface": "veth0", + "ipMasq": true + } + ] +} \ No newline at end of file diff --git a/upup/pkg/fi/cloud.go b/upup/pkg/fi/cloud.go index 5fb051799d83c..428380887b4d3 100644 --- a/upup/pkg/fi/cloud.go +++ b/upup/pkg/fi/cloud.go @@ -177,6 +177,10 @@ var zonesToCloud = map[string]kops.CloudProviderID{ "australia-southeast1-b": kops.CloudProviderGCE, "australia-southeast1-c": kops.CloudProviderGCE, + "europe-north1-a": kops.CloudProviderGCE, + "europe-north1-b": kops.CloudProviderGCE, + "europe-north1-c": kops.CloudProviderGCE, + "europe-west1-a": kops.CloudProviderGCE, "europe-west1-b": kops.CloudProviderGCE, "europe-west1-c": kops.CloudProviderGCE, diff --git a/upup/pkg/fi/cloudup/BUILD.bazel b/upup/pkg/fi/cloudup/BUILD.bazel index 8d70ce10b60d9..9a6d4a9696451 100644 --- a/upup/pkg/fi/cloudup/BUILD.bazel +++ b/upup/pkg/fi/cloudup/BUILD.bazel @@ -39,6 +39,7 @@ go_library( "//pkg/client/simple/vfsclientset:go_default_library", "//pkg/dns:go_default_library", "//pkg/featureflag:go_default_library", + "//pkg/k8sversion:go_default_library", "//pkg/model:go_default_library", "//pkg/model/alimodel:go_default_library", "//pkg/model/awsmodel:go_default_library", @@ -53,6 +54,7 @@ go_library( "//pkg/resources/digitalocean:go_default_library", "//pkg/resources/spotinst:go_default_library", "//pkg/templates:go_default_library", + "//pkg/util/subnet:go_default_library", "//upup/models:go_default_library", "//upup/pkg/fi:go_default_library", "//upup/pkg/fi/assettasks:go_default_library", diff --git a/upup/pkg/fi/cloudup/alitasks/disk.go b/upup/pkg/fi/cloudup/alitasks/disk.go index 399d73dfee63e..d4b2b54b0306f 100644 --- a/upup/pkg/fi/cloudup/alitasks/disk.go +++ b/upup/pkg/fi/cloudup/alitasks/disk.go @@ -143,7 +143,7 @@ func (_ *Disk) RenderALI(t *aliup.ALIAPITarget, a, e, changes *Disk) error { } if changes != nil && changes.Tags != nil { - glog.V(2).Infof("Modifing tags of disk with Name:%q", fi.StringValue(e.Name)) + glog.V(2).Infof("Modifying tags of disk with Name:%q", fi.StringValue(e.Name)) if err := t.Cloud.CreateTags(*e.DiskId, DiskResource, e.Tags); err != nil { return fmt.Errorf("error adding Tags to ALI YunPan: %v", err) } diff --git a/upup/pkg/fi/cloudup/alitasks/launchconfiguration.go b/upup/pkg/fi/cloudup/alitasks/launchconfiguration.go index aef675a8c90aa..234068d872b01 100644 --- a/upup/pkg/fi/cloudup/alitasks/launchconfiguration.go +++ b/upup/pkg/fi/cloudup/alitasks/launchconfiguration.go @@ -201,8 +201,8 @@ func (_ *LaunchConfiguration) RenderALI(t *aliup.ALIAPITarget, a, e, changes *La } e.ConfigurationId = fi.String(createScalingConfigurationResponse.ScalingConfigurationId) - // Disable ScalingGroup, used to bind scalingConfig, we should excute EnableScalingGroup in the task LaunchConfiguration - // If the ScalingGroup is active, we can not excute EnableScalingGroup. + // Disable ScalingGroup, used to bind scalingConfig, we should execute EnableScalingGroup in the task LaunchConfiguration + // If the ScalingGroup is active, we can not execute EnableScalingGroup. if e.ScalingGroup.Active != nil && fi.BoolValue(e.ScalingGroup.Active) { glog.V(2).Infof("Disabling LoadBalancer with id:%q", fi.StringValue(e.ScalingGroup.ScalingGroupId)) diff --git a/upup/pkg/fi/cloudup/alitasks/loadbalancer.go b/upup/pkg/fi/cloudup/alitasks/loadbalancer.go index b3ac7fe5e6b37..be397b911e1b5 100644 --- a/upup/pkg/fi/cloudup/alitasks/loadbalancer.go +++ b/upup/pkg/fi/cloudup/alitasks/loadbalancer.go @@ -188,7 +188,7 @@ func (_ *LoadBalancer) RenderALI(t *aliup.ALIAPITarget, a, e, changes *LoadBalan } if a != nil && (len(a.Tags) > 0) { - glog.V(2).Infof("Modifing LoadBalancer with Name:%q, update LoadBalancer tags", fi.StringValue(e.Name)) + glog.V(2).Infof("Modifying LoadBalancer with Name:%q, update LoadBalancer tags", fi.StringValue(e.Name)) tagsToDelete := e.getLoadBalancerTagsToDelete(a.Tags) if len(tagsToDelete) > 0 { diff --git a/upup/pkg/fi/cloudup/alitasks/scalinggroup.go b/upup/pkg/fi/cloudup/alitasks/scalinggroup.go index 828c9f933b49d..4773bfd8b283c 100644 --- a/upup/pkg/fi/cloudup/alitasks/scalinggroup.go +++ b/upup/pkg/fi/cloudup/alitasks/scalinggroup.go @@ -157,7 +157,7 @@ func (_ *ScalingGroup) RenderALI(t *aliup.ALIAPITarget, a, e, changes *ScalingGr } else { //only support to update size if changes.MinSize != nil || changes.MaxSize != nil { - glog.V(2).Infof("Modifing AutoscalingGroup with Name:%q", fi.StringValue(e.Name)) + glog.V(2).Infof("Modifying AutoscalingGroup with Name:%q", fi.StringValue(e.Name)) modifyScalingGroupArgs := &ess.ModifyScalingGroupArgs{ ScalingGroupId: fi.StringValue(a.ScalingGroupId), @@ -166,7 +166,7 @@ func (_ *ScalingGroup) RenderALI(t *aliup.ALIAPITarget, a, e, changes *ScalingGr } _, err := t.Cloud.EssClient().ModifyScalingGroup(modifyScalingGroupArgs) if err != nil { - return fmt.Errorf("error modifing autoscalingGroup: %v", err) + return fmt.Errorf("error modifying autoscalingGroup: %v", err) } } } diff --git a/upup/pkg/fi/cloudup/alitasks/securitygroup.go b/upup/pkg/fi/cloudup/alitasks/securitygroup.go index 940408fbaedd2..1f76572a1d9d2 100644 --- a/upup/pkg/fi/cloudup/alitasks/securitygroup.go +++ b/upup/pkg/fi/cloudup/alitasks/securitygroup.go @@ -159,7 +159,7 @@ func (_ *SecurityGroup) RenderALI(t *aliup.ALIAPITarget, a, e, changes *Security } if a != nil && (len(a.Tags) > 0) { - glog.V(2).Infof("Modifing SecurityGroup with Name:%q", fi.StringValue(e.Name)) + glog.V(2).Infof("Modifying SecurityGroup with Name:%q", fi.StringValue(e.Name)) tagsToDelete := e.getGroupTagsToDelete(a.Tags) if len(tagsToDelete) > 0 { diff --git a/upup/pkg/fi/cloudup/apply_cluster.go b/upup/pkg/fi/cloudup/apply_cluster.go index c0ffffc8687a6..5c233199a6d62 100644 --- a/upup/pkg/fi/cloudup/apply_cluster.go +++ b/upup/pkg/fi/cloudup/apply_cluster.go @@ -23,6 +23,8 @@ import ( "path" "strings" + "k8s.io/kops/pkg/k8sversion" + "github.com/blang/semver" "github.com/golang/glog" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -278,6 +280,36 @@ func (c *ApplyClusterCmd) Run() error { cluster.Spec.KubernetesVersion = versionWithoutV } + kv, err := k8sversion.Parse(cluster.Spec.KubernetesVersion) + if err != nil { + return err + } + + // check if we should recommend turning off anonymousAuth on k8s versions gte than 1.10 + // we do 1.10 since this is a really critical issues and 1.10 has it + if kv.IsGTE("1.10") { + // we do a check here because setting modifying the kubelet object messes with the output + warn := false + if cluster.Spec.Kubelet == nil { + warn = true + } else if cluster.Spec.Kubelet.AnonymousAuth == nil { + warn = true + } + + if warn { + fmt.Println("") + fmt.Printf(starline) + fmt.Println("") + fmt.Println("Kubelet anonymousAuth is currently turned on. This allows RBAC escalation and remote code execution possibilites.") + fmt.Println("It is highly recommended you turn it off by setting 'spec.kubelet.anonymousAuth' to 'false' via 'kops edit cluster'") + fmt.Println("") + fmt.Println("See https://github.com/kubernetes/kops/blob/master/docs/security.md#kubelet-api") + fmt.Println("") + fmt.Printf(starline) + fmt.Println("") + } + } + if err := c.AddFileAssets(assetBuilder); err != nil { return err } @@ -1092,6 +1124,17 @@ func (c *ApplyClusterCmd) AddFileAssets(assetBuilder *assets.AssetBuilder) error c.Assets = append(c.Assets, cniAssetHashString+"@"+cniAsset.String()) } + if c.Cluster.Spec.Networking.LyftVPC != nil { + lyftVPCDownloadURL := os.Getenv("LYFT_VPC_DOWNLOAD_URL") + if lyftVPCDownloadURL == "" { + lyftVPCDownloadURL = "bfdc65028a3bf8ffe14388fca28ede3600e7e2dee4e781908b6a23f9e79f86ad@https://github.com/lyft/cni-ipvlan-vpc-k8s/releases/download/v0.4.2/cni-ipvlan-vpc-k8s-v0.4.2.tar.gz" + } else { + glog.Warningf("Using url from LYFT_VPC_DOWNLOAD_URL env var: %q", lyftVPCDownloadURL) + } + + c.Assets = append(c.Assets, lyftVPCDownloadURL) + } + // TODO figure out if we can only do this for CoreOS only and GCE Container OS // TODO It is very difficult to pre-determine what OS an ami is, and if that OS needs socat // At this time we just copy the socat and conntrack binaries to all distros. diff --git a/upup/pkg/fi/cloudup/awstasks/load_balancer.go b/upup/pkg/fi/cloudup/awstasks/load_balancer.go index 257c2b066d572..86489e2910aaf 100644 --- a/upup/pkg/fi/cloudup/awstasks/load_balancer.go +++ b/upup/pkg/fi/cloudup/awstasks/load_balancer.go @@ -657,6 +657,7 @@ type terraformLoadBalancerListener struct { InstanceProtocol string `json:"instance_protocol"` LBPort int64 `json:"lb_port"` LBProtocol string `json:"lb_protocol"` + SSLCertificateID string `json:"ssl_certificate_id,omitempty"` } type terraformLoadBalancerHealthCheck struct { @@ -697,12 +698,23 @@ func (_ *LoadBalancer) RenderTerraform(t *terraform.TerraformTarget, a, e, chang return fmt.Errorf("error parsing load balancer listener port: %q", loadBalancerPort) } - tf.Listener = append(tf.Listener, &terraformLoadBalancerListener{ - InstanceProtocol: "TCP", - InstancePort: listener.InstancePort, - LBPort: loadBalancerPortInt, - LBProtocol: "TCP", - }) + if listener.SSLCertificateID != "" { + tf.Listener = append(tf.Listener, &terraformLoadBalancerListener{ + InstanceProtocol: "SSL", + InstancePort: listener.InstancePort, + LBPort: loadBalancerPortInt, + LBProtocol: "SSL", + SSLCertificateID: listener.SSLCertificateID, + }) + } else { + tf.Listener = append(tf.Listener, &terraformLoadBalancerListener{ + InstanceProtocol: "TCP", + InstancePort: listener.InstancePort, + LBPort: loadBalancerPortInt, + LBProtocol: "TCP", + }) + } + } if e.HealthCheck != nil { diff --git a/upup/pkg/fi/cloudup/awstasks/securitygroup.go b/upup/pkg/fi/cloudup/awstasks/securitygroup.go index 739067eb697c8..dbc92401d2dd3 100644 --- a/upup/pkg/fi/cloudup/awstasks/securitygroup.go +++ b/upup/pkg/fi/cloudup/awstasks/securitygroup.go @@ -96,26 +96,22 @@ func (e *SecurityGroup) Find(c *fi.Context) (*SecurityGroup, error) { func (e *SecurityGroup) findEc2(c *fi.Context) (*ec2.SecurityGroup, error) { cloud := c.Cloud.(awsup.AWSCloud) - - var vpcID *string - if e.VPC != nil { - vpcID = e.VPC.ID - } - - if vpcID == nil { - return nil, nil - } - request := &ec2.DescribeSecurityGroupsInput{} if fi.StringValue(e.ID) != "" { + // Find by ID. request.GroupIds = []*string{e.ID} - } else { + + } else if fi.StringValue(e.Name) != "" && e.VPC != nil && e.VPC.ID != nil { + // Find by filters (name and VPC ID). filters := cloud.BuildFilters(e.Name) - filters = append(filters, awsup.NewEC2Filter("vpc-id", *vpcID)) + filters = append(filters, awsup.NewEC2Filter("vpc-id", *e.VPC.ID)) filters = append(filters, awsup.NewEC2Filter("group-name", *e.Name)) - request.Filters = filters + + } else { + // No reason to try. + return nil, nil } response, err := cloud.EC2().DescribeSecurityGroups(request) diff --git a/upup/pkg/fi/cloudup/awstasks/vpccidrblock.go b/upup/pkg/fi/cloudup/awstasks/vpccidrblock.go index 60a40e9312497..a6226bd923124 100644 --- a/upup/pkg/fi/cloudup/awstasks/vpccidrblock.go +++ b/upup/pkg/fi/cloudup/awstasks/vpccidrblock.go @@ -122,7 +122,7 @@ func (_ *VPCCIDRBlock) RenderTerraform(t *terraform.TerraformTarget, a, e, chang return fmt.Errorf("terraform does not support AdditionalCIDRs on VPCs") // The code below is based on https://github.com/terraform-providers/terraform-provider-aws/pull/1568 - // and can be un-comented once it is landed. + // and can be un-commented once it is landed. // When this has been enabled please fix test TestAdditionalCIDR in integration_test.go to run runTestAWS. // tf := &terraformVPCCIDRBlock{ // VPCID: e.VPC.TerraformLink(), diff --git a/upup/pkg/fi/cloudup/awsup/aws_cloud.go b/upup/pkg/fi/cloudup/awsup/aws_cloud.go index 1f2e15f530139..a15a41d62dd12 100644 --- a/upup/pkg/fi/cloudup/awsup/aws_cloud.go +++ b/upup/pkg/fi/cloudup/awsup/aws_cloud.go @@ -491,26 +491,29 @@ func FindAutoscalingGroups(c AWSCloud, tags map[string]string) ([]*autoscaling.G } if len(asgNames) != 0 { - request := &autoscaling.DescribeAutoScalingGroupsInput{ - AutoScalingGroupNames: asgNames, - } - err := c.Autoscaling().DescribeAutoScalingGroupsPages(request, func(p *autoscaling.DescribeAutoScalingGroupsOutput, lastPage bool) bool { - for _, asg := range p.AutoScalingGroups { - if !matchesAsgTags(tags, asg.Tags) { - // We used an inexact filter above - continue - } - // Check for "Delete in progress" (the only use of .Status) - if asg.Status != nil { - glog.Warningf("Skipping ASG %v (which matches tags): %v", *asg.AutoScalingGroupARN, *asg.Status) - continue + for i := 0; i < len(asgNames); i += 50 { + batch := asgNames[i:minInt(i+50, len(asgNames))] + request := &autoscaling.DescribeAutoScalingGroupsInput{ + AutoScalingGroupNames: batch, + } + err := c.Autoscaling().DescribeAutoScalingGroupsPages(request, func(p *autoscaling.DescribeAutoScalingGroupsOutput, lastPage bool) bool { + for _, asg := range p.AutoScalingGroups { + if !matchesAsgTags(tags, asg.Tags) { + // We used an inexact filter above + continue + } + // Check for "Delete in progress" (the only use of .Status) + if asg.Status != nil { + glog.Warningf("Skipping ASG %v (which matches tags): %v", *asg.AutoScalingGroupARN, *asg.Status) + continue + } + asgs = append(asgs, asg) } - asgs = append(asgs, asg) + return true + }) + if err != nil { + return nil, fmt.Errorf("error listing autoscaling groups: %v", err) } - return true - }) - if err != nil { - return nil, fmt.Errorf("error listing autoscaling groups: %v", err) } } @@ -518,6 +521,14 @@ func FindAutoscalingGroups(c AWSCloud, tags map[string]string) ([]*autoscaling.G return asgs, nil } +// Returns the minimum of two ints +func minInt(a int, b int) int { + if a < b { + return a + } + return b +} + // matchesAsgTags is used to filter an asg by tags func matchesAsgTags(tags map[string]string, actual []*autoscaling.TagDescription) bool { for k, v := range tags { diff --git a/upup/pkg/fi/cloudup/awsup/machine_types.go b/upup/pkg/fi/cloudup/awsup/machine_types.go index 49f40a5f0f216..de0bea0de9a63 100644 --- a/upup/pkg/fi/cloudup/awsup/machine_types.go +++ b/upup/pkg/fi/cloudup/awsup/machine_types.go @@ -26,13 +26,16 @@ import ( const BurstableCreditsToECUS float32 = 3.0 / 60.0 type AWSMachineTypeInfo struct { - Name string - MemoryGB float32 - ECU float32 - Cores int - EphemeralDisks []int - Burstable bool - GPU bool + Name string + MemoryGB float32 + ECU float32 + Cores int + EphemeralDisks []int + Burstable bool + GPU bool + MaxPods int + InstanceENIs int + InstanceIPsPerENI int } type EphemeralDevice struct { @@ -77,1191 +80,1781 @@ var MachineTypes []AWSMachineTypeInfo = []AWSMachineTypeInfo{ // NOTE: Content below is auto generated by `make update-machine-types` // BEGIN GENERATED CONTENT + // a1 family + { + Name: "a1.medium", + MemoryGB: 2, + ECU: 0, + Cores: 1, + InstanceENIs: 2, + InstanceIPsPerENI: 4, + EphemeralDisks: nil, + }, + + { + Name: "a1.large", + MemoryGB: 4, + ECU: 0, + Cores: 2, + InstanceENIs: 3, + InstanceIPsPerENI: 10, + EphemeralDisks: nil, + }, + + { + Name: "a1.xlarge", + MemoryGB: 8, + ECU: 0, + Cores: 4, + InstanceENIs: 4, + InstanceIPsPerENI: 15, + EphemeralDisks: nil, + }, + + { + Name: "a1.2xlarge", + MemoryGB: 16, + ECU: 0, + Cores: 8, + InstanceENIs: 4, + InstanceIPsPerENI: 15, + EphemeralDisks: nil, + }, + + { + Name: "a1.4xlarge", + MemoryGB: 32, + ECU: 0, + Cores: 16, + InstanceENIs: 8, + InstanceIPsPerENI: 30, + EphemeralDisks: nil, + }, + // c1 family { - Name: "c1.medium", - MemoryGB: 1.7, - ECU: 5, - Cores: 2, - EphemeralDisks: []int{350}, + Name: "c1.medium", + MemoryGB: 1.7, + ECU: 5, + Cores: 2, + InstanceENIs: 2, + InstanceIPsPerENI: 6, + EphemeralDisks: []int{350}, }, { - Name: "c1.xlarge", - MemoryGB: 7, - ECU: 20, - Cores: 8, - EphemeralDisks: []int{420, 420, 420, 420}, + Name: "c1.xlarge", + MemoryGB: 7, + ECU: 20, + Cores: 8, + InstanceENIs: 4, + InstanceIPsPerENI: 15, + EphemeralDisks: []int{420, 420, 420, 420}, }, // c3 family { - Name: "c3.large", - MemoryGB: 3.75, - ECU: 7, - Cores: 2, - EphemeralDisks: []int{16, 16}, + Name: "c3.large", + MemoryGB: 3.75, + ECU: 7, + Cores: 2, + InstanceENIs: 3, + InstanceIPsPerENI: 10, + EphemeralDisks: []int{16, 16}, }, { - Name: "c3.xlarge", - MemoryGB: 7.5, - ECU: 14, - Cores: 4, - EphemeralDisks: []int{40, 40}, + Name: "c3.xlarge", + MemoryGB: 7.5, + ECU: 14, + Cores: 4, + InstanceENIs: 4, + InstanceIPsPerENI: 15, + EphemeralDisks: []int{40, 40}, }, { - Name: "c3.2xlarge", - MemoryGB: 15, - ECU: 28, - Cores: 8, - EphemeralDisks: []int{80, 80}, + Name: "c3.2xlarge", + MemoryGB: 15, + ECU: 28, + Cores: 8, + InstanceENIs: 4, + InstanceIPsPerENI: 15, + EphemeralDisks: []int{80, 80}, }, { - Name: "c3.4xlarge", - MemoryGB: 30, - ECU: 55, - Cores: 16, - EphemeralDisks: []int{160, 160}, + Name: "c3.4xlarge", + MemoryGB: 30, + ECU: 55, + Cores: 16, + InstanceENIs: 8, + InstanceIPsPerENI: 30, + EphemeralDisks: []int{160, 160}, }, { - Name: "c3.8xlarge", - MemoryGB: 60, - ECU: 108, - Cores: 32, - EphemeralDisks: []int{320, 320}, + Name: "c3.8xlarge", + MemoryGB: 60, + ECU: 108, + Cores: 32, + InstanceENIs: 8, + InstanceIPsPerENI: 30, + EphemeralDisks: []int{320, 320}, }, // c4 family { - Name: "c4.large", - MemoryGB: 3.75, - ECU: 8, - Cores: 2, - EphemeralDisks: nil, + Name: "c4.large", + MemoryGB: 3.75, + ECU: 8, + Cores: 2, + InstanceENIs: 3, + InstanceIPsPerENI: 10, + EphemeralDisks: nil, }, { - Name: "c4.xlarge", - MemoryGB: 7.5, - ECU: 16, - Cores: 4, - EphemeralDisks: nil, + Name: "c4.xlarge", + MemoryGB: 7.5, + ECU: 16, + Cores: 4, + InstanceENIs: 4, + InstanceIPsPerENI: 15, + EphemeralDisks: nil, }, { - Name: "c4.2xlarge", - MemoryGB: 15, - ECU: 31, - Cores: 8, - EphemeralDisks: nil, + Name: "c4.2xlarge", + MemoryGB: 15, + ECU: 31, + Cores: 8, + InstanceENIs: 4, + InstanceIPsPerENI: 15, + EphemeralDisks: nil, }, { - Name: "c4.4xlarge", - MemoryGB: 30, - ECU: 62, - Cores: 16, - EphemeralDisks: nil, + Name: "c4.4xlarge", + MemoryGB: 30, + ECU: 62, + Cores: 16, + InstanceENIs: 8, + InstanceIPsPerENI: 30, + EphemeralDisks: nil, }, { - Name: "c4.8xlarge", - MemoryGB: 60, - ECU: 132, - Cores: 36, - EphemeralDisks: nil, + Name: "c4.8xlarge", + MemoryGB: 60, + ECU: 132, + Cores: 36, + InstanceENIs: 8, + InstanceIPsPerENI: 30, + EphemeralDisks: nil, }, // c5 family { - Name: "c5.large", - MemoryGB: 4, - ECU: 9, - Cores: 2, - EphemeralDisks: nil, + Name: "c5.large", + MemoryGB: 4, + ECU: 9, + Cores: 2, + InstanceENIs: 3, + InstanceIPsPerENI: 10, + EphemeralDisks: nil, }, { - Name: "c5.xlarge", - MemoryGB: 8, - ECU: 17, - Cores: 4, - EphemeralDisks: nil, + Name: "c5.xlarge", + MemoryGB: 8, + ECU: 17, + Cores: 4, + InstanceENIs: 4, + InstanceIPsPerENI: 15, + EphemeralDisks: nil, }, { - Name: "c5.2xlarge", - MemoryGB: 16, - ECU: 34, - Cores: 8, - EphemeralDisks: nil, + Name: "c5.2xlarge", + MemoryGB: 16, + ECU: 34, + Cores: 8, + InstanceENIs: 4, + InstanceIPsPerENI: 15, + EphemeralDisks: nil, }, { - Name: "c5.4xlarge", - MemoryGB: 32, - ECU: 68, - Cores: 16, - EphemeralDisks: nil, + Name: "c5.4xlarge", + MemoryGB: 32, + ECU: 68, + Cores: 16, + InstanceENIs: 8, + InstanceIPsPerENI: 30, + EphemeralDisks: nil, }, { - Name: "c5.9xlarge", - MemoryGB: 72, - ECU: 141, - Cores: 36, - EphemeralDisks: nil, + Name: "c5.9xlarge", + MemoryGB: 72, + ECU: 141, + Cores: 36, + InstanceENIs: 8, + InstanceIPsPerENI: 30, + EphemeralDisks: nil, }, { - Name: "c5.18xlarge", - MemoryGB: 144, - ECU: 281, - Cores: 72, - EphemeralDisks: nil, + Name: "c5.18xlarge", + MemoryGB: 144, + ECU: 281, + Cores: 72, + InstanceENIs: 15, + InstanceIPsPerENI: 50, + EphemeralDisks: nil, }, // c5d family { - Name: "c5d.large", - MemoryGB: 4, - ECU: 9, - Cores: 2, - EphemeralDisks: []int{50}, + Name: "c5d.large", + MemoryGB: 4, + ECU: 9, + Cores: 2, + InstanceENIs: 3, + InstanceIPsPerENI: 10, + EphemeralDisks: []int{50}, + }, + + { + Name: "c5d.xlarge", + MemoryGB: 8, + ECU: 17, + Cores: 4, + InstanceENIs: 4, + InstanceIPsPerENI: 15, + EphemeralDisks: []int{100}, }, { - Name: "c5d.xlarge", - MemoryGB: 8, - ECU: 17, - Cores: 4, - EphemeralDisks: []int{100}, + Name: "c5d.2xlarge", + MemoryGB: 16, + ECU: 34, + Cores: 8, + InstanceENIs: 4, + InstanceIPsPerENI: 15, + EphemeralDisks: []int{200}, }, { - Name: "c5d.2xlarge", - MemoryGB: 16, - ECU: 34, - Cores: 8, - EphemeralDisks: []int{200}, + Name: "c5d.4xlarge", + MemoryGB: 32, + ECU: 68, + Cores: 16, + InstanceENIs: 8, + InstanceIPsPerENI: 30, + EphemeralDisks: []int{400}, }, { - Name: "c5d.4xlarge", - MemoryGB: 32, - ECU: 68, - Cores: 16, - EphemeralDisks: []int{400}, + Name: "c5d.9xlarge", + MemoryGB: 72, + ECU: 141, + Cores: 36, + InstanceENIs: 8, + InstanceIPsPerENI: 30, + EphemeralDisks: []int{900}, }, { - Name: "c5d.9xlarge", - MemoryGB: 72, - ECU: 141, - Cores: 36, - EphemeralDisks: []int{900}, + Name: "c5d.18xlarge", + MemoryGB: 144, + ECU: 281, + Cores: 72, + InstanceENIs: 15, + InstanceIPsPerENI: 50, + EphemeralDisks: []int{900, 900}, }, + // c5n family { - Name: "c5d.18xlarge", - MemoryGB: 144, - ECU: 281, - Cores: 72, - EphemeralDisks: []int{1800}, + Name: "c5n.large", + MemoryGB: 5.25, + ECU: 0, + Cores: 2, + InstanceENIs: 3, + InstanceIPsPerENI: 10, + EphemeralDisks: nil, + }, + + { + Name: "c5n.xlarge", + MemoryGB: 10.5, + ECU: 0, + Cores: 4, + InstanceENIs: 4, + InstanceIPsPerENI: 15, + EphemeralDisks: nil, + }, + + { + Name: "c5n.2xlarge", + MemoryGB: 21, + ECU: 0, + Cores: 8, + InstanceENIs: 4, + InstanceIPsPerENI: 15, + EphemeralDisks: nil, + }, + + { + Name: "c5n.4xlarge", + MemoryGB: 42, + ECU: 0, + Cores: 16, + InstanceENIs: 8, + InstanceIPsPerENI: 30, + EphemeralDisks: nil, + }, + + { + Name: "c5n.9xlarge", + MemoryGB: 96, + ECU: 0, + Cores: 36, + InstanceENIs: 8, + InstanceIPsPerENI: 30, + EphemeralDisks: nil, + }, + + { + Name: "c5n.18xlarge", + MemoryGB: 192, + ECU: 0, + Cores: 72, + InstanceENIs: 15, + InstanceIPsPerENI: 50, + EphemeralDisks: nil, }, // cc2 family { - Name: "cc2.8xlarge", - MemoryGB: 60.5, - ECU: 88, - Cores: 32, - EphemeralDisks: []int{840, 840, 840, 840}, + Name: "cc2.8xlarge", + MemoryGB: 60.5, + ECU: 88, + Cores: 32, + InstanceENIs: 8, + InstanceIPsPerENI: 30, + EphemeralDisks: []int{840, 840, 840, 840}, }, // cr1 family { - Name: "cr1.8xlarge", - MemoryGB: 244, - ECU: 88, - Cores: 32, - EphemeralDisks: []int{120, 120}, + Name: "cr1.8xlarge", + MemoryGB: 244, + ECU: 88, + Cores: 32, + InstanceENIs: 8, + InstanceIPsPerENI: 30, + EphemeralDisks: []int{120, 120}, }, // d2 family { - Name: "d2.xlarge", - MemoryGB: 30.5, - ECU: 14, - Cores: 4, - EphemeralDisks: []int{2000, 2000, 2000}, + Name: "d2.xlarge", + MemoryGB: 30.5, + ECU: 14, + Cores: 4, + InstanceENIs: 4, + InstanceIPsPerENI: 15, + EphemeralDisks: []int{2000, 2000, 2000}, }, { - Name: "d2.2xlarge", - MemoryGB: 61, - ECU: 28, - Cores: 8, - EphemeralDisks: []int{2000, 2000, 2000, 2000, 2000, 2000}, + Name: "d2.2xlarge", + MemoryGB: 61, + ECU: 28, + Cores: 8, + InstanceENIs: 4, + InstanceIPsPerENI: 15, + EphemeralDisks: []int{2000, 2000, 2000, 2000, 2000, 2000}, }, { - Name: "d2.4xlarge", - MemoryGB: 122, - ECU: 56, - Cores: 16, - EphemeralDisks: []int{2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000}, + Name: "d2.4xlarge", + MemoryGB: 122, + ECU: 56, + Cores: 16, + InstanceENIs: 8, + InstanceIPsPerENI: 30, + EphemeralDisks: []int{2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000}, }, { - Name: "d2.8xlarge", - MemoryGB: 244, - ECU: 116, - Cores: 36, - EphemeralDisks: []int{2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000}, + Name: "d2.8xlarge", + MemoryGB: 244, + ECU: 116, + Cores: 36, + InstanceENIs: 8, + InstanceIPsPerENI: 30, + EphemeralDisks: []int{2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000}, }, // f1 family { - Name: "f1.2xlarge", - MemoryGB: 122, - ECU: 26, - Cores: 8, - EphemeralDisks: nil, + Name: "f1.2xlarge", + MemoryGB: 122, + ECU: 26, + Cores: 8, + InstanceENIs: 4, + InstanceIPsPerENI: 15, + EphemeralDisks: nil, }, { - Name: "f1.4xlarge", - MemoryGB: 244, - ECU: 52, - Cores: 16, - EphemeralDisks: []int{940}, + Name: "f1.4xlarge", + MemoryGB: 244, + ECU: 52, + Cores: 16, + InstanceENIs: 8, + InstanceIPsPerENI: 30, + EphemeralDisks: []int{940}, }, { - Name: "f1.16xlarge", - MemoryGB: 976, - ECU: 188, - Cores: 64, - EphemeralDisks: nil, + Name: "f1.16xlarge", + MemoryGB: 976, + ECU: 188, + Cores: 64, + InstanceENIs: 8, + InstanceIPsPerENI: 50, + EphemeralDisks: nil, }, // g2 family { - Name: "g2.2xlarge", - MemoryGB: 15, - ECU: 26, - Cores: 8, - EphemeralDisks: []int{60}, - GPU: true, + Name: "g2.2xlarge", + MemoryGB: 15, + ECU: 26, + Cores: 8, + InstanceENIs: 4, + InstanceIPsPerENI: 15, + EphemeralDisks: []int{60}, + GPU: true, }, { - Name: "g2.8xlarge", - MemoryGB: 60, - ECU: 104, - Cores: 32, - EphemeralDisks: []int{120, 120}, - GPU: true, + Name: "g2.8xlarge", + MemoryGB: 60, + ECU: 104, + Cores: 32, + InstanceENIs: 8, + InstanceIPsPerENI: 30, + EphemeralDisks: []int{120, 120}, + GPU: true, }, // g3 family { - Name: "g3.4xlarge", - MemoryGB: 122, - ECU: 47, - Cores: 16, - EphemeralDisks: nil, - GPU: true, + Name: "g3.4xlarge", + MemoryGB: 122, + ECU: 47, + Cores: 16, + InstanceENIs: 8, + InstanceIPsPerENI: 30, + EphemeralDisks: nil, + GPU: true, + }, + + { + Name: "g3.8xlarge", + MemoryGB: 244, + ECU: 94, + Cores: 32, + InstanceENIs: 8, + InstanceIPsPerENI: 30, + EphemeralDisks: nil, + GPU: true, }, { - Name: "g3.8xlarge", - MemoryGB: 244, - ECU: 94, - Cores: 32, - EphemeralDisks: nil, - GPU: true, + Name: "g3.16xlarge", + MemoryGB: 488, + ECU: 188, + Cores: 64, + InstanceENIs: 15, + InstanceIPsPerENI: 50, + EphemeralDisks: nil, + GPU: true, }, + // g3s family { - Name: "g3.16xlarge", - MemoryGB: 488, - ECU: 188, - Cores: 64, - EphemeralDisks: nil, - GPU: true, + Name: "g3s.xlarge", + MemoryGB: 30.5, + ECU: 13, + Cores: 4, + InstanceENIs: 4, + InstanceIPsPerENI: 15, + EphemeralDisks: nil, + GPU: true, }, // h1 family { - Name: "h1.2xlarge", - MemoryGB: 32, - ECU: 26, - Cores: 8, - EphemeralDisks: []int{2000}, + Name: "h1.2xlarge", + MemoryGB: 32, + ECU: 26, + Cores: 8, + InstanceENIs: 4, + InstanceIPsPerENI: 15, + EphemeralDisks: []int{2000}, }, { - Name: "h1.4xlarge", - MemoryGB: 64, - ECU: 53.5, - Cores: 16, - EphemeralDisks: []int{2000, 2000}, + Name: "h1.4xlarge", + MemoryGB: 64, + ECU: 53.5, + Cores: 16, + InstanceENIs: 8, + InstanceIPsPerENI: 30, + EphemeralDisks: []int{2000, 2000}, }, { - Name: "h1.8xlarge", - MemoryGB: 128, - ECU: 99, - Cores: 32, - EphemeralDisks: []int{2000, 2000, 2000, 2000}, + Name: "h1.8xlarge", + MemoryGB: 128, + ECU: 99, + Cores: 32, + InstanceENIs: 8, + InstanceIPsPerENI: 30, + EphemeralDisks: []int{2000, 2000, 2000, 2000}, }, { - Name: "h1.16xlarge", - MemoryGB: 256, - ECU: 188, - Cores: 64, - EphemeralDisks: []int{2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000}, + Name: "h1.16xlarge", + MemoryGB: 256, + ECU: 188, + Cores: 64, + InstanceENIs: 15, + InstanceIPsPerENI: 50, + EphemeralDisks: []int{2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000}, }, // hs1 family { - Name: "hs1.8xlarge", - MemoryGB: 117, - ECU: 35, - Cores: 17, - EphemeralDisks: []int{2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000}, + Name: "hs1.8xlarge", + MemoryGB: 117, + ECU: 35, + Cores: 17, + InstanceENIs: 8, + InstanceIPsPerENI: 30, + EphemeralDisks: []int{2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000}, }, // i2 family { - Name: "i2.xlarge", - MemoryGB: 30.5, - ECU: 14, - Cores: 4, - EphemeralDisks: []int{800}, + Name: "i2.xlarge", + MemoryGB: 30.5, + ECU: 14, + Cores: 4, + InstanceENIs: 4, + InstanceIPsPerENI: 15, + EphemeralDisks: []int{800}, }, { - Name: "i2.2xlarge", - MemoryGB: 61, - ECU: 27, - Cores: 8, - EphemeralDisks: []int{800, 800}, + Name: "i2.2xlarge", + MemoryGB: 61, + ECU: 27, + Cores: 8, + InstanceENIs: 4, + InstanceIPsPerENI: 15, + EphemeralDisks: []int{800, 800}, }, { - Name: "i2.4xlarge", - MemoryGB: 122, - ECU: 53, - Cores: 16, - EphemeralDisks: []int{800, 800, 800, 800}, + Name: "i2.4xlarge", + MemoryGB: 122, + ECU: 53, + Cores: 16, + InstanceENIs: 8, + InstanceIPsPerENI: 30, + EphemeralDisks: []int{800, 800, 800, 800}, }, { - Name: "i2.8xlarge", - MemoryGB: 244, - ECU: 104, - Cores: 32, - EphemeralDisks: []int{800, 800, 800, 800, 800, 800, 800, 800}, + Name: "i2.8xlarge", + MemoryGB: 244, + ECU: 104, + Cores: 32, + InstanceENIs: 8, + InstanceIPsPerENI: 30, + EphemeralDisks: []int{800, 800, 800, 800, 800, 800, 800, 800}, }, // i3 family { - Name: "i3.large", - MemoryGB: 15.25, - ECU: 7, - Cores: 2, - EphemeralDisks: []int{475}, + Name: "i3.large", + MemoryGB: 15.25, + ECU: 7, + Cores: 2, + InstanceENIs: 3, + InstanceIPsPerENI: 10, + EphemeralDisks: []int{475}, }, { - Name: "i3.xlarge", - MemoryGB: 30.5, - ECU: 13, - Cores: 4, - EphemeralDisks: []int{950}, + Name: "i3.xlarge", + MemoryGB: 30.5, + ECU: 13, + Cores: 4, + InstanceENIs: 4, + InstanceIPsPerENI: 15, + EphemeralDisks: []int{950}, }, { - Name: "i3.2xlarge", - MemoryGB: 61, - ECU: 27, - Cores: 8, - EphemeralDisks: []int{1900}, + Name: "i3.2xlarge", + MemoryGB: 61, + ECU: 27, + Cores: 8, + InstanceENIs: 4, + InstanceIPsPerENI: 15, + EphemeralDisks: []int{1900}, }, { - Name: "i3.4xlarge", - MemoryGB: 122, - ECU: 53, - Cores: 16, - EphemeralDisks: []int{1900, 1900}, + Name: "i3.4xlarge", + MemoryGB: 122, + ECU: 53, + Cores: 16, + InstanceENIs: 8, + InstanceIPsPerENI: 30, + EphemeralDisks: []int{1900, 1900}, }, { - Name: "i3.8xlarge", - MemoryGB: 244, - ECU: 99, - Cores: 32, - EphemeralDisks: []int{1900, 1900, 1900, 1900}, + Name: "i3.8xlarge", + MemoryGB: 244, + ECU: 99, + Cores: 32, + InstanceENIs: 8, + InstanceIPsPerENI: 30, + EphemeralDisks: []int{1900, 1900, 1900, 1900}, }, { - Name: "i3.16xlarge", - MemoryGB: 488, - ECU: 200, - Cores: 64, - EphemeralDisks: []int{1900, 1900, 1900, 1900, 1900, 1900, 1900, 1900}, + Name: "i3.16xlarge", + MemoryGB: 488, + ECU: 200, + Cores: 64, + InstanceENIs: 15, + InstanceIPsPerENI: 50, + EphemeralDisks: []int{1900, 1900, 1900, 1900, 1900, 1900, 1900, 1900}, }, { - Name: "i3.metal", - MemoryGB: 512, - ECU: 208, - Cores: 72, - EphemeralDisks: []int{1900, 1900, 1900, 1900, 1900, 1900, 1900, 1900}, + Name: "i3.metal", + MemoryGB: 512, + ECU: 208, + Cores: 72, + InstanceENIs: 15, + InstanceIPsPerENI: 50, + EphemeralDisks: []int{1900, 1900, 1900, 1900, 1900, 1900, 1900, 1900}, }, // m1 family { - Name: "m1.small", - MemoryGB: 1.7, - ECU: 1, - Cores: 1, - EphemeralDisks: []int{160}, + Name: "m1.small", + MemoryGB: 1.7, + ECU: 1, + Cores: 1, + InstanceENIs: 2, + InstanceIPsPerENI: 4, + EphemeralDisks: []int{160}, }, { - Name: "m1.medium", - MemoryGB: 3.75, - ECU: 2, - Cores: 1, - EphemeralDisks: []int{410}, + Name: "m1.medium", + MemoryGB: 3.75, + ECU: 2, + Cores: 1, + InstanceENIs: 2, + InstanceIPsPerENI: 6, + EphemeralDisks: []int{410}, }, { - Name: "m1.large", - MemoryGB: 7.5, - ECU: 4, - Cores: 2, - EphemeralDisks: []int{420, 420}, + Name: "m1.large", + MemoryGB: 7.5, + ECU: 4, + Cores: 2, + InstanceENIs: 3, + InstanceIPsPerENI: 10, + EphemeralDisks: []int{420, 420}, }, { - Name: "m1.xlarge", - MemoryGB: 15, - ECU: 8, - Cores: 4, - EphemeralDisks: []int{420, 420, 420, 420}, + Name: "m1.xlarge", + MemoryGB: 15, + ECU: 8, + Cores: 4, + InstanceENIs: 4, + InstanceIPsPerENI: 15, + EphemeralDisks: []int{420, 420, 420, 420}, }, // m2 family { - Name: "m2.xlarge", - MemoryGB: 17.1, - ECU: 6.5, - Cores: 2, - EphemeralDisks: []int{420}, + Name: "m2.xlarge", + MemoryGB: 17.1, + ECU: 6.5, + Cores: 2, + InstanceENIs: 4, + InstanceIPsPerENI: 15, + EphemeralDisks: []int{420}, }, { - Name: "m2.2xlarge", - MemoryGB: 34.2, - ECU: 13, - Cores: 4, - EphemeralDisks: []int{850}, + Name: "m2.2xlarge", + MemoryGB: 34.2, + ECU: 13, + Cores: 4, + InstanceENIs: 4, + InstanceIPsPerENI: 30, + EphemeralDisks: []int{850}, }, { - Name: "m2.4xlarge", - MemoryGB: 68.4, - ECU: 26, - Cores: 8, - EphemeralDisks: []int{840, 840}, + Name: "m2.4xlarge", + MemoryGB: 68.4, + ECU: 26, + Cores: 8, + InstanceENIs: 8, + InstanceIPsPerENI: 30, + EphemeralDisks: []int{840, 840}, }, // m3 family { - Name: "m3.medium", - MemoryGB: 3.75, - ECU: 3, - Cores: 1, - EphemeralDisks: []int{4}, + Name: "m3.medium", + MemoryGB: 3.75, + ECU: 3, + Cores: 1, + InstanceENIs: 2, + InstanceIPsPerENI: 6, + EphemeralDisks: []int{4}, }, { - Name: "m3.large", - MemoryGB: 7.5, - ECU: 6.5, - Cores: 2, - EphemeralDisks: []int{32}, + Name: "m3.large", + MemoryGB: 7.5, + ECU: 6.5, + Cores: 2, + InstanceENIs: 3, + InstanceIPsPerENI: 10, + EphemeralDisks: []int{32}, }, { - Name: "m3.xlarge", - MemoryGB: 15, - ECU: 13, - Cores: 4, - EphemeralDisks: []int{40, 40}, + Name: "m3.xlarge", + MemoryGB: 15, + ECU: 13, + Cores: 4, + InstanceENIs: 4, + InstanceIPsPerENI: 15, + EphemeralDisks: []int{40, 40}, }, { - Name: "m3.2xlarge", - MemoryGB: 30, - ECU: 26, - Cores: 8, - EphemeralDisks: []int{80, 80}, + Name: "m3.2xlarge", + MemoryGB: 30, + ECU: 26, + Cores: 8, + InstanceENIs: 4, + InstanceIPsPerENI: 30, + EphemeralDisks: []int{80, 80}, }, // m4 family { - Name: "m4.large", - MemoryGB: 8, - ECU: 6.5, - Cores: 2, - EphemeralDisks: nil, + Name: "m4.large", + MemoryGB: 8, + ECU: 6.5, + Cores: 2, + InstanceENIs: 2, + InstanceIPsPerENI: 10, + EphemeralDisks: nil, }, { - Name: "m4.xlarge", - MemoryGB: 16, - ECU: 13, - Cores: 4, - EphemeralDisks: nil, + Name: "m4.xlarge", + MemoryGB: 16, + ECU: 13, + Cores: 4, + InstanceENIs: 4, + InstanceIPsPerENI: 15, + EphemeralDisks: nil, }, { - Name: "m4.2xlarge", - MemoryGB: 32, - ECU: 26, - Cores: 8, - EphemeralDisks: nil, + Name: "m4.2xlarge", + MemoryGB: 32, + ECU: 26, + Cores: 8, + InstanceENIs: 4, + InstanceIPsPerENI: 15, + EphemeralDisks: nil, }, { - Name: "m4.4xlarge", - MemoryGB: 64, - ECU: 53.5, - Cores: 16, - EphemeralDisks: nil, + Name: "m4.4xlarge", + MemoryGB: 64, + ECU: 53.5, + Cores: 16, + InstanceENIs: 8, + InstanceIPsPerENI: 30, + EphemeralDisks: nil, }, { - Name: "m4.10xlarge", - MemoryGB: 160, - ECU: 124.5, - Cores: 40, - EphemeralDisks: nil, + Name: "m4.10xlarge", + MemoryGB: 160, + ECU: 124.5, + Cores: 40, + InstanceENIs: 8, + InstanceIPsPerENI: 30, + EphemeralDisks: nil, }, { - Name: "m4.16xlarge", - MemoryGB: 256, - ECU: 188, - Cores: 64, - EphemeralDisks: nil, + Name: "m4.16xlarge", + MemoryGB: 256, + ECU: 188, + Cores: 64, + InstanceENIs: 8, + InstanceIPsPerENI: 30, + EphemeralDisks: nil, }, // m5 family { - Name: "m5.large", - MemoryGB: 8, - ECU: 8, - Cores: 2, - EphemeralDisks: nil, + Name: "m5.large", + MemoryGB: 8, + ECU: 8, + Cores: 2, + InstanceENIs: 3, + InstanceIPsPerENI: 10, + EphemeralDisks: nil, + }, + + { + Name: "m5.xlarge", + MemoryGB: 16, + ECU: 16, + Cores: 4, + InstanceENIs: 4, + InstanceIPsPerENI: 15, + EphemeralDisks: nil, + }, + + { + Name: "m5.2xlarge", + MemoryGB: 32, + ECU: 31, + Cores: 8, + InstanceENIs: 4, + InstanceIPsPerENI: 15, + EphemeralDisks: nil, + }, + + { + Name: "m5.4xlarge", + MemoryGB: 64, + ECU: 60, + Cores: 16, + InstanceENIs: 8, + InstanceIPsPerENI: 30, + EphemeralDisks: nil, }, { - Name: "m5.xlarge", - MemoryGB: 16, - ECU: 16, - Cores: 4, - EphemeralDisks: nil, + Name: "m5.12xlarge", + MemoryGB: 192, + ECU: 173, + Cores: 48, + InstanceENIs: 8, + InstanceIPsPerENI: 30, + EphemeralDisks: nil, }, { - Name: "m5.2xlarge", - MemoryGB: 32, - ECU: 31, - Cores: 8, - EphemeralDisks: nil, + Name: "m5.24xlarge", + MemoryGB: 384, + ECU: 345, + Cores: 96, + InstanceENIs: 15, + InstanceIPsPerENI: 50, + EphemeralDisks: nil, }, { - Name: "m5.4xlarge", - MemoryGB: 64, - ECU: 60, - Cores: 16, - EphemeralDisks: nil, + Name: "m5.metal", + MemoryGB: 384, + ECU: 345, + Cores: 96, + InstanceENIs: 15, + InstanceIPsPerENI: 50, + EphemeralDisks: nil, }, + // m5a family { - Name: "m5.12xlarge", - MemoryGB: 192, - ECU: 173, - Cores: 48, - EphemeralDisks: nil, + Name: "m5a.large", + MemoryGB: 8, + ECU: 0, + Cores: 2, + InstanceENIs: 3, + InstanceIPsPerENI: 10, + EphemeralDisks: nil, }, { - Name: "m5.24xlarge", - MemoryGB: 384, - ECU: 345, - Cores: 96, - EphemeralDisks: nil, + Name: "m5a.xlarge", + MemoryGB: 16, + ECU: 0, + Cores: 4, + InstanceENIs: 4, + InstanceIPsPerENI: 15, + EphemeralDisks: nil, + }, + + { + Name: "m5a.2xlarge", + MemoryGB: 32, + ECU: 0, + Cores: 8, + InstanceENIs: 4, + InstanceIPsPerENI: 15, + EphemeralDisks: nil, + }, + + { + Name: "m5a.4xlarge", + MemoryGB: 64, + ECU: 0, + Cores: 16, + InstanceENIs: 8, + InstanceIPsPerENI: 30, + EphemeralDisks: nil, + }, + + { + Name: "m5a.12xlarge", + MemoryGB: 192, + ECU: 0, + Cores: 48, + InstanceENIs: 8, + InstanceIPsPerENI: 30, + EphemeralDisks: nil, + }, + + { + Name: "m5a.24xlarge", + MemoryGB: 384, + ECU: 0, + Cores: 96, + InstanceENIs: 15, + InstanceIPsPerENI: 50, + EphemeralDisks: nil, }, // m5d family { - Name: "m5d.large", - MemoryGB: 8, - ECU: 8, - Cores: 2, - EphemeralDisks: []int{75}, + Name: "m5d.large", + MemoryGB: 8, + ECU: 8, + Cores: 2, + InstanceENIs: 3, + InstanceIPsPerENI: 10, + EphemeralDisks: []int{75}, }, { - Name: "m5d.xlarge", - MemoryGB: 16, - ECU: 16, - Cores: 4, - EphemeralDisks: []int{150}, + Name: "m5d.xlarge", + MemoryGB: 16, + ECU: 16, + Cores: 4, + InstanceENIs: 4, + InstanceIPsPerENI: 15, + EphemeralDisks: []int{150}, }, { - Name: "m5d.2xlarge", - MemoryGB: 32, - ECU: 31, - Cores: 8, - EphemeralDisks: []int{300}, + Name: "m5d.2xlarge", + MemoryGB: 32, + ECU: 31, + Cores: 8, + InstanceENIs: 4, + InstanceIPsPerENI: 15, + EphemeralDisks: []int{300}, }, { - Name: "m5d.4xlarge", - MemoryGB: 64, - ECU: 60, - Cores: 16, - EphemeralDisks: []int{300, 300}, + Name: "m5d.4xlarge", + MemoryGB: 64, + ECU: 60, + Cores: 16, + InstanceENIs: 8, + InstanceIPsPerENI: 30, + EphemeralDisks: []int{300, 300}, }, { - Name: "m5d.12xlarge", - MemoryGB: 192, - ECU: 173, - Cores: 48, - EphemeralDisks: []int{900, 900}, + Name: "m5d.12xlarge", + MemoryGB: 192, + ECU: 173, + Cores: 48, + InstanceENIs: 8, + InstanceIPsPerENI: 30, + EphemeralDisks: []int{900, 900}, }, { - Name: "m5d.24xlarge", - MemoryGB: 384, - ECU: 345, - Cores: 96, - EphemeralDisks: []int{900, 900, 900, 900}, + Name: "m5d.24xlarge", + MemoryGB: 384, + ECU: 345, + Cores: 96, + InstanceENIs: 15, + InstanceIPsPerENI: 50, + EphemeralDisks: []int{900, 900, 900, 900}, + }, + + { + Name: "m5d.metal", + MemoryGB: 384, + ECU: 345, + Cores: 96, + InstanceENIs: 15, + InstanceIPsPerENI: 50, + EphemeralDisks: []int{900, 900, 900, 900}, }, // p2 family { - Name: "p2.xlarge", - MemoryGB: 61, - ECU: 12, - Cores: 4, - EphemeralDisks: nil, - GPU: true, + Name: "p2.xlarge", + MemoryGB: 61, + ECU: 12, + Cores: 4, + InstanceENIs: 4, + InstanceIPsPerENI: 15, + EphemeralDisks: nil, + GPU: true, }, { - Name: "p2.8xlarge", - MemoryGB: 488, - ECU: 94, - Cores: 32, - EphemeralDisks: nil, - GPU: true, + Name: "p2.8xlarge", + MemoryGB: 488, + ECU: 94, + Cores: 32, + InstanceENIs: 8, + InstanceIPsPerENI: 30, + EphemeralDisks: nil, + GPU: true, }, { - Name: "p2.16xlarge", - MemoryGB: 768, - ECU: 188, - Cores: 64, - EphemeralDisks: nil, - GPU: true, + Name: "p2.16xlarge", + MemoryGB: 768, + ECU: 188, + Cores: 64, + InstanceENIs: 8, + InstanceIPsPerENI: 30, + EphemeralDisks: nil, + GPU: true, }, // p3 family { - Name: "p3.2xlarge", - MemoryGB: 61, - ECU: 26, - Cores: 8, - EphemeralDisks: nil, - GPU: true, + Name: "p3.2xlarge", + MemoryGB: 61, + ECU: 26, + Cores: 8, + InstanceENIs: 4, + InstanceIPsPerENI: 15, + EphemeralDisks: nil, + GPU: true, + }, + + { + Name: "p3.8xlarge", + MemoryGB: 244, + ECU: 94, + Cores: 32, + InstanceENIs: 8, + InstanceIPsPerENI: 30, + EphemeralDisks: nil, + GPU: true, }, { - Name: "p3.8xlarge", - MemoryGB: 244, - ECU: 94, - Cores: 32, - EphemeralDisks: nil, - GPU: true, + Name: "p3.16xlarge", + MemoryGB: 488, + ECU: 188, + Cores: 64, + InstanceENIs: 8, + InstanceIPsPerENI: 30, + EphemeralDisks: nil, + GPU: true, }, + // p3dn family { - Name: "p3.16xlarge", - MemoryGB: 488, - ECU: 188, - Cores: 64, - EphemeralDisks: nil, - GPU: true, + Name: "p3dn.24xlarge", + MemoryGB: 768, + ECU: 345, + Cores: 96, + InstanceENIs: 15, + InstanceIPsPerENI: 50, + EphemeralDisks: []int{900, 900}, + GPU: true, }, // r3 family { - Name: "r3.large", - MemoryGB: 15.25, - ECU: 6.5, - Cores: 2, - EphemeralDisks: []int{32}, + Name: "r3.large", + MemoryGB: 15.25, + ECU: 6.5, + Cores: 2, + InstanceENIs: 3, + InstanceIPsPerENI: 10, + EphemeralDisks: []int{32}, }, { - Name: "r3.xlarge", - MemoryGB: 30.5, - ECU: 13, - Cores: 4, - EphemeralDisks: []int{80}, + Name: "r3.xlarge", + MemoryGB: 30.5, + ECU: 13, + Cores: 4, + InstanceENIs: 4, + InstanceIPsPerENI: 15, + EphemeralDisks: []int{80}, }, { - Name: "r3.2xlarge", - MemoryGB: 61, - ECU: 26, - Cores: 8, - EphemeralDisks: []int{160}, + Name: "r3.2xlarge", + MemoryGB: 61, + ECU: 26, + Cores: 8, + InstanceENIs: 4, + InstanceIPsPerENI: 15, + EphemeralDisks: []int{160}, }, { - Name: "r3.4xlarge", - MemoryGB: 122, - ECU: 52, - Cores: 16, - EphemeralDisks: []int{320}, + Name: "r3.4xlarge", + MemoryGB: 122, + ECU: 52, + Cores: 16, + InstanceENIs: 8, + InstanceIPsPerENI: 30, + EphemeralDisks: []int{320}, }, { - Name: "r3.8xlarge", - MemoryGB: 244, - ECU: 104, - Cores: 32, - EphemeralDisks: []int{320, 320}, + Name: "r3.8xlarge", + MemoryGB: 244, + ECU: 104, + Cores: 32, + InstanceENIs: 8, + InstanceIPsPerENI: 30, + EphemeralDisks: []int{320, 320}, }, // r4 family { - Name: "r4.large", - MemoryGB: 15.25, - ECU: 7, - Cores: 2, - EphemeralDisks: nil, + Name: "r4.large", + MemoryGB: 15.25, + ECU: 7, + Cores: 2, + InstanceENIs: 3, + InstanceIPsPerENI: 10, + EphemeralDisks: nil, }, { - Name: "r4.xlarge", - MemoryGB: 30.5, - ECU: 13.5, - Cores: 4, - EphemeralDisks: nil, + Name: "r4.xlarge", + MemoryGB: 30.5, + ECU: 13.5, + Cores: 4, + InstanceENIs: 4, + InstanceIPsPerENI: 15, + EphemeralDisks: nil, }, { - Name: "r4.2xlarge", - MemoryGB: 61, - ECU: 27, - Cores: 8, - EphemeralDisks: nil, + Name: "r4.2xlarge", + MemoryGB: 61, + ECU: 27, + Cores: 8, + InstanceENIs: 4, + InstanceIPsPerENI: 15, + EphemeralDisks: nil, }, { - Name: "r4.4xlarge", - MemoryGB: 122, - ECU: 53, - Cores: 16, - EphemeralDisks: nil, + Name: "r4.4xlarge", + MemoryGB: 122, + ECU: 53, + Cores: 16, + InstanceENIs: 8, + InstanceIPsPerENI: 30, + EphemeralDisks: nil, }, { - Name: "r4.8xlarge", - MemoryGB: 244, - ECU: 99, - Cores: 32, - EphemeralDisks: nil, + Name: "r4.8xlarge", + MemoryGB: 244, + ECU: 99, + Cores: 32, + InstanceENIs: 8, + InstanceIPsPerENI: 30, + EphemeralDisks: nil, }, { - Name: "r4.16xlarge", - MemoryGB: 488, - ECU: 195, - Cores: 64, - EphemeralDisks: nil, + Name: "r4.16xlarge", + MemoryGB: 488, + ECU: 195, + Cores: 64, + InstanceENIs: 15, + InstanceIPsPerENI: 50, + EphemeralDisks: nil, }, // r5 family { - Name: "r5.large", - MemoryGB: 16, - ECU: 10, - Cores: 2, - EphemeralDisks: nil, + Name: "r5.large", + MemoryGB: 16, + ECU: 10, + Cores: 2, + InstanceENIs: 3, + InstanceIPsPerENI: 10, + EphemeralDisks: nil, }, { - Name: "r5.xlarge", - MemoryGB: 32, - ECU: 19, - Cores: 4, - EphemeralDisks: nil, + Name: "r5.xlarge", + MemoryGB: 32, + ECU: 19, + Cores: 4, + InstanceENIs: 4, + InstanceIPsPerENI: 15, + EphemeralDisks: nil, }, { - Name: "r5.2xlarge", - MemoryGB: 64, - ECU: 38, - Cores: 8, - EphemeralDisks: nil, + Name: "r5.2xlarge", + MemoryGB: 64, + ECU: 38, + Cores: 8, + InstanceENIs: 4, + InstanceIPsPerENI: 15, + EphemeralDisks: nil, }, { - Name: "r5.4xlarge", - MemoryGB: 128, - ECU: 71, - Cores: 16, - EphemeralDisks: nil, + Name: "r5.4xlarge", + MemoryGB: 128, + ECU: 71, + Cores: 16, + InstanceENIs: 8, + InstanceIPsPerENI: 30, + EphemeralDisks: nil, }, { - Name: "r5.12xlarge", - MemoryGB: 384, - ECU: 173, - Cores: 48, - EphemeralDisks: nil, + Name: "r5.12xlarge", + MemoryGB: 384, + ECU: 173, + Cores: 48, + InstanceENIs: 8, + InstanceIPsPerENI: 30, + EphemeralDisks: nil, }, { - Name: "r5.24xlarge", - MemoryGB: 768, - ECU: 347, - Cores: 96, - EphemeralDisks: nil, + Name: "r5.24xlarge", + MemoryGB: 768, + ECU: 347, + Cores: 96, + InstanceENIs: 15, + InstanceIPsPerENI: 50, + EphemeralDisks: nil, + }, + + { + Name: "r5.metal", + MemoryGB: 768, + ECU: 347, + Cores: 96, + InstanceENIs: 15, + InstanceIPsPerENI: 50, + EphemeralDisks: nil, + }, + + // r5a family + { + Name: "r5a.large", + MemoryGB: 16, + ECU: 0, + Cores: 2, + InstanceENIs: 3, + InstanceIPsPerENI: 10, + EphemeralDisks: nil, + }, + + { + Name: "r5a.xlarge", + MemoryGB: 32, + ECU: 0, + Cores: 4, + InstanceENIs: 4, + InstanceIPsPerENI: 15, + EphemeralDisks: nil, + }, + + { + Name: "r5a.2xlarge", + MemoryGB: 64, + ECU: 0, + Cores: 8, + InstanceENIs: 4, + InstanceIPsPerENI: 15, + EphemeralDisks: nil, + }, + + { + Name: "r5a.4xlarge", + MemoryGB: 128, + ECU: 0, + Cores: 16, + InstanceENIs: 8, + InstanceIPsPerENI: 30, + EphemeralDisks: nil, + }, + + { + Name: "r5a.12xlarge", + MemoryGB: 384, + ECU: 0, + Cores: 48, + InstanceENIs: 8, + InstanceIPsPerENI: 30, + EphemeralDisks: nil, + }, + + { + Name: "r5a.24xlarge", + MemoryGB: 768, + ECU: 0, + Cores: 96, + InstanceENIs: 15, + InstanceIPsPerENI: 50, + EphemeralDisks: nil, }, // r5d family { - Name: "r5d.large", - MemoryGB: 16, - ECU: 10, - Cores: 2, - EphemeralDisks: []int{75}, + Name: "r5d.large", + MemoryGB: 16, + ECU: 10, + Cores: 2, + InstanceENIs: 3, + InstanceIPsPerENI: 10, + EphemeralDisks: []int{75}, }, { - Name: "r5d.xlarge", - MemoryGB: 32, - ECU: 19, - Cores: 4, - EphemeralDisks: []int{150}, + Name: "r5d.xlarge", + MemoryGB: 32, + ECU: 19, + Cores: 4, + InstanceENIs: 4, + InstanceIPsPerENI: 15, + EphemeralDisks: []int{150}, }, { - Name: "r5d.2xlarge", - MemoryGB: 64, - ECU: 38, - Cores: 8, - EphemeralDisks: []int{300}, + Name: "r5d.2xlarge", + MemoryGB: 64, + ECU: 38, + Cores: 8, + InstanceENIs: 4, + InstanceIPsPerENI: 15, + EphemeralDisks: []int{300}, }, { - Name: "r5d.4xlarge", - MemoryGB: 128, - ECU: 71, - Cores: 16, - EphemeralDisks: []int{300, 300}, + Name: "r5d.4xlarge", + MemoryGB: 128, + ECU: 71, + Cores: 16, + InstanceENIs: 8, + InstanceIPsPerENI: 30, + EphemeralDisks: []int{300, 300}, }, { - Name: "r5d.12xlarge", - MemoryGB: 384, - ECU: 173, - Cores: 48, - EphemeralDisks: []int{900, 900}, + Name: "r5d.12xlarge", + MemoryGB: 384, + ECU: 173, + Cores: 48, + InstanceENIs: 8, + InstanceIPsPerENI: 30, + EphemeralDisks: []int{900, 900}, }, { - Name: "r5d.24xlarge", - MemoryGB: 768, - ECU: 347, - Cores: 96, - EphemeralDisks: []int{900, 900, 900, 900}, + Name: "r5d.24xlarge", + MemoryGB: 768, + ECU: 347, + Cores: 96, + InstanceENIs: 15, + InstanceIPsPerENI: 50, + EphemeralDisks: []int{900, 900, 900, 900}, + }, + + { + Name: "r5d.metal", + MemoryGB: 768, + ECU: 347, + Cores: 96, + InstanceENIs: 15, + InstanceIPsPerENI: 50, + EphemeralDisks: []int{900, 900, 900, 900}, }, // t1 family { - Name: "t1.micro", - MemoryGB: 0.613, - ECU: 1 * BurstableCreditsToECUS, - Cores: 1, - EphemeralDisks: nil, - Burstable: true, + Name: "t1.micro", + MemoryGB: 0.613, + ECU: 1 * BurstableCreditsToECUS, + Cores: 1, + InstanceENIs: 2, + InstanceIPsPerENI: 2, + EphemeralDisks: nil, + Burstable: true, }, // t2 family { - Name: "t2.nano", - MemoryGB: 0.5, - ECU: 3 * BurstableCreditsToECUS, - Cores: 1, - EphemeralDisks: nil, - Burstable: true, + Name: "t2.nano", + MemoryGB: 0.5, + ECU: 3 * BurstableCreditsToECUS, + Cores: 1, + InstanceENIs: 2, + InstanceIPsPerENI: 2, + EphemeralDisks: nil, + Burstable: true, }, { - Name: "t2.micro", - MemoryGB: 1, - ECU: 6 * BurstableCreditsToECUS, - Cores: 1, - EphemeralDisks: nil, - Burstable: true, + Name: "t2.micro", + MemoryGB: 1, + ECU: 6 * BurstableCreditsToECUS, + Cores: 1, + InstanceENIs: 2, + InstanceIPsPerENI: 2, + EphemeralDisks: nil, + Burstable: true, }, { - Name: "t2.small", - MemoryGB: 2, - ECU: 12 * BurstableCreditsToECUS, - Cores: 1, - EphemeralDisks: nil, - Burstable: true, + Name: "t2.small", + MemoryGB: 2, + ECU: 12 * BurstableCreditsToECUS, + Cores: 1, + InstanceENIs: 3, + InstanceIPsPerENI: 4, + EphemeralDisks: nil, + Burstable: true, }, { - Name: "t2.medium", - MemoryGB: 4, - ECU: 24 * BurstableCreditsToECUS, - Cores: 2, - EphemeralDisks: nil, - Burstable: true, + Name: "t2.medium", + MemoryGB: 4, + ECU: 24 * BurstableCreditsToECUS, + Cores: 2, + InstanceENIs: 3, + InstanceIPsPerENI: 6, + EphemeralDisks: nil, + Burstable: true, }, { - Name: "t2.large", - MemoryGB: 8, - ECU: 36 * BurstableCreditsToECUS, - Cores: 2, - EphemeralDisks: nil, - Burstable: true, + Name: "t2.large", + MemoryGB: 8, + ECU: 36 * BurstableCreditsToECUS, + Cores: 2, + InstanceENIs: 3, + InstanceIPsPerENI: 12, + EphemeralDisks: nil, + Burstable: true, }, { - Name: "t2.xlarge", - MemoryGB: 16, - ECU: 54 * BurstableCreditsToECUS, - Cores: 4, - EphemeralDisks: nil, - Burstable: true, + Name: "t2.xlarge", + MemoryGB: 16, + ECU: 54 * BurstableCreditsToECUS, + Cores: 4, + InstanceENIs: 3, + InstanceIPsPerENI: 15, + EphemeralDisks: nil, + Burstable: true, }, { - Name: "t2.2xlarge", - MemoryGB: 32, - ECU: 81.6 * BurstableCreditsToECUS, - Cores: 8, - EphemeralDisks: nil, - Burstable: true, + Name: "t2.2xlarge", + MemoryGB: 32, + ECU: 81.6 * BurstableCreditsToECUS, + Cores: 8, + InstanceENIs: 3, + InstanceIPsPerENI: 15, + EphemeralDisks: nil, + Burstable: true, }, // t3 family { - Name: "t3.nano", - MemoryGB: 0.5, - ECU: 6 * BurstableCreditsToECUS, - Cores: 2, - EphemeralDisks: nil, - Burstable: true, + Name: "t3.nano", + MemoryGB: 0.5, + ECU: 6 * BurstableCreditsToECUS, + Cores: 2, + InstanceENIs: 2, + InstanceIPsPerENI: 2, + EphemeralDisks: nil, + Burstable: true, }, { - Name: "t3.micro", - MemoryGB: 1, - ECU: 12 * BurstableCreditsToECUS, - Cores: 2, - EphemeralDisks: nil, - Burstable: true, + Name: "t3.micro", + MemoryGB: 1, + ECU: 12 * BurstableCreditsToECUS, + Cores: 2, + InstanceENIs: 2, + InstanceIPsPerENI: 2, + EphemeralDisks: nil, + Burstable: true, }, { - Name: "t3.small", - MemoryGB: 2, - ECU: 24 * BurstableCreditsToECUS, - Cores: 2, - EphemeralDisks: nil, - Burstable: true, + Name: "t3.small", + MemoryGB: 2, + ECU: 24 * BurstableCreditsToECUS, + Cores: 2, + InstanceENIs: 3, + InstanceIPsPerENI: 4, + EphemeralDisks: nil, + Burstable: true, }, { - Name: "t3.medium", - MemoryGB: 4, - ECU: 24 * BurstableCreditsToECUS, - Cores: 2, - EphemeralDisks: nil, - Burstable: true, + Name: "t3.medium", + MemoryGB: 4, + ECU: 24 * BurstableCreditsToECUS, + Cores: 2, + InstanceENIs: 3, + InstanceIPsPerENI: 6, + EphemeralDisks: nil, + Burstable: true, }, { - Name: "t3.large", - MemoryGB: 8, - ECU: 36 * BurstableCreditsToECUS, - Cores: 2, - EphemeralDisks: nil, - Burstable: true, + Name: "t3.large", + MemoryGB: 8, + ECU: 36 * BurstableCreditsToECUS, + Cores: 2, + InstanceENIs: 3, + InstanceIPsPerENI: 12, + EphemeralDisks: nil, + Burstable: true, }, { - Name: "t3.xlarge", - MemoryGB: 16, - ECU: 96 * BurstableCreditsToECUS, - Cores: 4, - EphemeralDisks: nil, - Burstable: true, + Name: "t3.xlarge", + MemoryGB: 16, + ECU: 96 * BurstableCreditsToECUS, + Cores: 4, + InstanceENIs: 4, + InstanceIPsPerENI: 15, + EphemeralDisks: nil, + Burstable: true, }, { - Name: "t3.2xlarge", - MemoryGB: 32, - ECU: 192 * BurstableCreditsToECUS, - Cores: 8, - EphemeralDisks: nil, - Burstable: true, + Name: "t3.2xlarge", + MemoryGB: 32, + ECU: 192 * BurstableCreditsToECUS, + Cores: 8, + InstanceENIs: 4, + InstanceIPsPerENI: 15, + EphemeralDisks: nil, + Burstable: true, }, // x1 family { - Name: "x1.16xlarge", - MemoryGB: 976, - ECU: 174.5, - Cores: 64, - EphemeralDisks: []int{1920}, + Name: "x1.16xlarge", + MemoryGB: 976, + ECU: 174.5, + Cores: 64, + InstanceENIs: 8, + InstanceIPsPerENI: 30, + EphemeralDisks: []int{1920}, }, { - Name: "x1.32xlarge", - MemoryGB: 1952, - ECU: 349, - Cores: 128, - EphemeralDisks: []int{1920, 1920}, + Name: "x1.32xlarge", + MemoryGB: 1952, + ECU: 349, + Cores: 128, + InstanceENIs: 8, + InstanceIPsPerENI: 30, + EphemeralDisks: []int{1920, 1920}, }, // x1e family { - Name: "x1e.xlarge", - MemoryGB: 122, - ECU: 12, - Cores: 4, - EphemeralDisks: []int{120}, + Name: "x1e.xlarge", + MemoryGB: 122, + ECU: 12, + Cores: 4, + InstanceENIs: 3, + InstanceIPsPerENI: 10, + EphemeralDisks: []int{120}, }, { - Name: "x1e.2xlarge", - MemoryGB: 244, - ECU: 23, - Cores: 8, - EphemeralDisks: []int{240}, + Name: "x1e.2xlarge", + MemoryGB: 244, + ECU: 23, + Cores: 8, + InstanceENIs: 4, + InstanceIPsPerENI: 15, + EphemeralDisks: []int{240}, }, { - Name: "x1e.4xlarge", - MemoryGB: 488, - ECU: 47, - Cores: 16, - EphemeralDisks: []int{480}, + Name: "x1e.4xlarge", + MemoryGB: 488, + ECU: 47, + Cores: 16, + InstanceENIs: 4, + InstanceIPsPerENI: 15, + EphemeralDisks: []int{480}, }, { - Name: "x1e.8xlarge", - MemoryGB: 976, - ECU: 91, - Cores: 32, - EphemeralDisks: []int{960}, + Name: "x1e.8xlarge", + MemoryGB: 976, + ECU: 91, + Cores: 32, + InstanceENIs: 4, + InstanceIPsPerENI: 15, + EphemeralDisks: []int{960}, }, { - Name: "x1e.16xlarge", - MemoryGB: 1952, - ECU: 179, - Cores: 64, - EphemeralDisks: []int{1920}, + Name: "x1e.16xlarge", + MemoryGB: 1952, + ECU: 179, + Cores: 64, + InstanceENIs: 8, + InstanceIPsPerENI: 30, + EphemeralDisks: []int{1920}, }, { - Name: "x1e.32xlarge", - MemoryGB: 3904, - ECU: 340, - Cores: 128, - EphemeralDisks: []int{1920, 1920}, + Name: "x1e.32xlarge", + MemoryGB: 3904, + ECU: 340, + Cores: 128, + InstanceENIs: 8, + InstanceIPsPerENI: 30, + EphemeralDisks: []int{1920, 1920}, }, // z1d family { - Name: "z1d.large", - MemoryGB: 16, - ECU: 15, - Cores: 2, - EphemeralDisks: []int{75}, + Name: "z1d.large", + MemoryGB: 16, + ECU: 15, + Cores: 2, + InstanceENIs: 3, + InstanceIPsPerENI: 10, + EphemeralDisks: []int{75}, + }, + + { + Name: "z1d.xlarge", + MemoryGB: 32, + ECU: 28, + Cores: 4, + InstanceENIs: 4, + InstanceIPsPerENI: 15, + EphemeralDisks: []int{150}, }, { - Name: "z1d.xlarge", - MemoryGB: 32, - ECU: 28, - Cores: 4, - EphemeralDisks: []int{150}, + Name: "z1d.2xlarge", + MemoryGB: 64, + ECU: 53, + Cores: 8, + InstanceENIs: 4, + InstanceIPsPerENI: 15, + EphemeralDisks: []int{300}, }, { - Name: "z1d.2xlarge", - MemoryGB: 64, - ECU: 53, - Cores: 8, - EphemeralDisks: []int{300}, + Name: "z1d.3xlarge", + MemoryGB: 96, + ECU: 75, + Cores: 12, + InstanceENIs: 8, + InstanceIPsPerENI: 30, + EphemeralDisks: []int{450}, }, { - Name: "z1d.3xlarge", - MemoryGB: 96, - ECU: 75, - Cores: 12, - EphemeralDisks: []int{450}, + Name: "z1d.6xlarge", + MemoryGB: 192, + ECU: 134, + Cores: 24, + InstanceENIs: 8, + InstanceIPsPerENI: 30, + EphemeralDisks: []int{900}, }, { - Name: "z1d.6xlarge", - MemoryGB: 192, - ECU: 134, - Cores: 24, - EphemeralDisks: []int{900}, + Name: "z1d.12xlarge", + MemoryGB: 384, + ECU: 271, + Cores: 48, + InstanceENIs: 15, + InstanceIPsPerENI: 50, + EphemeralDisks: []int{900, 900}, }, { - Name: "z1d.12xlarge", - MemoryGB: 384, - ECU: 271, - Cores: 48, - EphemeralDisks: []int{900, 900}, + Name: "z1d.metal", + MemoryGB: 384, + ECU: 271, + Cores: 48, + InstanceENIs: 15, + InstanceIPsPerENI: 50, + EphemeralDisks: []int{900, 900}, }, // END GENERATED CONTENT diff --git a/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go b/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go index 8f339237737f4..51737e3fd2a24 100644 --- a/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go +++ b/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go @@ -211,7 +211,7 @@ func (b *BootstrapChannelBuilder) buildManifest() (*channelsapi.Addons, map[stri if kubeDNS.Provider == "CoreDNS" { { key := "coredns.addons.k8s.io" - version := "1.2.2-kops.1" + version := "1.2.6-kops.1" { location := key + "/k8s-1.6.yaml" @@ -284,7 +284,7 @@ func (b *BootstrapChannelBuilder) buildManifest() (*channelsapi.Addons, map[stri if externalDNS == nil || !externalDNS.Disable { { key := "dns-controller.addons.k8s.io" - version := "1.11.0-alpha.1" + version := "1.11.0" { location := key + "/pre-k8s-1.6.yaml" @@ -447,7 +447,7 @@ func (b *BootstrapChannelBuilder) buildManifest() (*channelsapi.Addons, map[stri if featureflag.Spotinst.Enabled() { key := "spotinst-kubernetes-cluster-controller.addons.k8s.io" - version := "1.0.16" + version := "1.0.18" { id := "v1.8.0" @@ -501,7 +501,7 @@ func (b *BootstrapChannelBuilder) buildManifest() (*channelsapi.Addons, map[stri if b.cluster.Spec.Networking.Kopeio != nil { key := "networking.kope.io" - version := "1.0.20180319-kops.2" + version := "1.0.20181028-kops.1" { location := key + "/pre-k8s-1.6.yaml" @@ -539,8 +539,8 @@ func (b *BootstrapChannelBuilder) buildManifest() (*channelsapi.Addons, map[stri versions := map[string]string{ "pre-k8s-1.6": "2.3.0-kops.2", "k8s-1.6": "2.3.0-kops.2", - "k8s-1.7": "2.4.1-kops.1", - "k8s-1.8": "2.4.1-kops.1", + "k8s-1.7": "2.5.0-kops.1", + "k8s-1.8": "2.5.0-kops.1", } { @@ -643,53 +643,71 @@ func (b *BootstrapChannelBuilder) buildManifest() (*channelsapi.Addons, map[stri key := "networking.projectcalico.org" versions := map[string]string{ "pre-k8s-1.6": "2.4.2-kops.1", - "k8s-1.6": "2.6.7-kops.2", - "k8s-1.7": "2.6.7-kops.3", + "k8s-1.6": "2.6.9-kops.1", + "k8s-1.7": "2.6.9-kops.1", + "k8s-1.7-v3": "3.3.1-kops.3", } - { - id := "pre-k8s-1.6" - location := key + "/" + id + ".yaml" + if b.cluster.Spec.Networking.Calico.MajorVersion == "v3" { + { + id := "k8s-1.7-v3" + location := key + "/" + id + ".yaml" - addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{ - Name: fi.String(key), - Version: fi.String(versions[id]), - Selector: networkingSelector, - Manifest: fi.String(location), - KubernetesVersion: "<1.6.0", - Id: id, - }) - manifests[key+"-"+id] = "addons/" + location - } + addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{ + Name: fi.String(key), + Version: fi.String(versions[id]), + Selector: networkingSelector, + Manifest: fi.String(location), + KubernetesVersion: ">=1.7.0", + Id: id, + }) + manifests[key+"-"+id] = "addons/" + location + } + } else { + { + id := "pre-k8s-1.6" + location := key + "/" + id + ".yaml" - { - id := "k8s-1.6" - location := key + "/" + id + ".yaml" + addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{ + Name: fi.String(key), + Version: fi.String(versions[id]), + Selector: networkingSelector, + Manifest: fi.String(location), + KubernetesVersion: "<1.6.0", + Id: id, + }) + manifests[key+"-"+id] = "addons/" + location + } - addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{ - Name: fi.String(key), - Version: fi.String(versions[id]), - Selector: networkingSelector, - Manifest: fi.String(location), - KubernetesVersion: ">=1.6.0 <1.7.0", - Id: id, - }) - manifests[key+"-"+id] = "addons/" + location - } + { + id := "k8s-1.6" + location := key + "/" + id + ".yaml" - { - id := "k8s-1.7" - location := key + "/" + id + ".yaml" + addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{ + Name: fi.String(key), + Version: fi.String(versions[id]), + Selector: networkingSelector, + Manifest: fi.String(location), + KubernetesVersion: ">=1.6.0 <1.7.0", + Id: id, + }) + manifests[key+"-"+id] = "addons/" + location + } - addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{ - Name: fi.String(key), - Version: fi.String(versions[id]), - Selector: networkingSelector, - Manifest: fi.String(location), - KubernetesVersion: ">=1.7.0", - Id: id, - }) - manifests[key+"-"+id] = "addons/" + location + { + id := "k8s-1.7" + location := key + "/" + id + ".yaml" + + addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{ + Name: fi.String(key), + Version: fi.String(versions[id]), + Selector: networkingSelector, + Manifest: fi.String(location), + KubernetesVersion: ">=1.7.0", + Id: id, + }) + manifests[key+"-"+id] = "addons/" + location + } } } @@ -701,6 +719,7 @@ func (b *BootstrapChannelBuilder) buildManifest() (*channelsapi.Addons, map[stri "k8s-1.6": "2.4.2-kops.2", "k8s-1.8": "2.6.7-kops.3", "k8s-1.9": "3.2.3-kops.1", + "k8s-1.12": "3.3.0-kops.1", } { id := "pre-k8s-1.6" @@ -760,6 +779,20 @@ func (b *BootstrapChannelBuilder) buildManifest() (*channelsapi.Addons, map[stri }) manifests[key+"-"+id] = "addons/" + location } + { + id := "k8s-1.12" + location := key + "/" + id + ".yaml" + + addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{ + Name: fi.String(key), + Version: fi.String(versions[id]), + Selector: networkingSelector, + Manifest: fi.String(location), + KubernetesVersion: ">=1.12.0", + Id: id, + }) + manifests[key+"-"+id] = "addons/" + location + } } if b.cluster.Spec.Networking.Kuberouter != nil { @@ -804,7 +837,7 @@ func (b *BootstrapChannelBuilder) buildManifest() (*channelsapi.Addons, map[stri if b.cluster.Spec.Networking.AmazonVPC != nil { key := "networking.amazon-vpc-routed-eni" - version := "1.0.0-kops.3" + version := "1.3.0-kops.1" { id := "k8s-1.7" diff --git a/upup/pkg/fi/cloudup/networking.go b/upup/pkg/fi/cloudup/networking.go index a158b59c92038..8dc90e7d16f87 100644 --- a/upup/pkg/fi/cloudup/networking.go +++ b/upup/pkg/fi/cloudup/networking.go @@ -21,7 +21,6 @@ import ( "net/url" "os" - "github.com/blang/semver" "github.com/golang/glog" api "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/pkg/apis/kops/util" @@ -95,6 +94,11 @@ func usesCNI(c *api.Cluster) bool { return true } + if networkConfig.LyftVPC != nil { + // LyftVPC uses CNI + return true + } + // Assume other modes also use CNI glog.Warningf("Unknown networking mode configured") return true @@ -118,6 +122,10 @@ const ( defaultCNIAssetK8s1_9 = "https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.6.0.tgz" defaultCNIAssetHashStringK8s1_9 = "d595d3ded6499a64e8dac02466e2f5f2ce257c9f" + // defaultCNIAssetK8s1_11 is the CNI tarball for k8s >= 1.11 + defaultCNIAssetK8s1_11 = "https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.7.5.tgz" + defaultCNIAssetHashStringK8s1_11 = "52e9d2de8a5f927307d9397308735658ee44ab8d" + // Environment variable for overriding CNI url ENV_VAR_CNI_VERSION_URL = "CNI_VERSION_URL" ENV_VAR_CNI_ASSET_HASH_STRING = "CNI_ASSET_HASH_STRING" @@ -148,22 +156,23 @@ func findCNIAssets(c *api.Cluster, assetBuilder *assets.AssetBuilder) (*url.URL, return nil, "", fmt.Errorf("failed to lookup kubernetes version: %v", err) } - sv.Pre = nil - sv.Build = nil - var cniAsset, cniAssetHash string - if sv.GTE(semver.Version{Major: 1, Minor: 9, Patch: 0, Pre: nil, Build: nil}) { + if util.IsKubernetesGTE("1.11", *sv) { + cniAsset = defaultCNIAssetK8s1_11 + cniAssetHash = defaultCNIAssetHashStringK8s1_11 + glog.V(2).Infof("Adding default CNI asset for k8s >= 1.11: %s", defaultCNIAssetK8s1_9) + } else if util.IsKubernetesGTE("1.9", *sv) { cniAsset = defaultCNIAssetK8s1_9 cniAssetHash = defaultCNIAssetHashStringK8s1_9 - glog.V(2).Infof("Adding default CNI asset for k8s 1.9.x and higher: %s", defaultCNIAssetK8s1_9) - } else if sv.GTE(semver.Version{Major: 1, Minor: 6, Patch: 0, Pre: nil, Build: nil}) { + glog.V(2).Infof("Adding default CNI asset for 1.11 > k8s >= 1.9: %s", defaultCNIAssetK8s1_9) + } else if util.IsKubernetesGTE("1.6", *sv) { cniAsset = defaultCNIAssetK8s1_6 cniAssetHash = defaultCNIAssetHashStringK8s1_6 - glog.V(2).Infof("Adding default CNI asset for k8s 1.6.x and higher: %s", defaultCNIAssetK8s1_6) + glog.V(2).Infof("Adding default CNI asset for 1.9 > k8s >= 1.6: %s", defaultCNIAssetK8s1_6) } else { cniAsset = defaultCNIAssetK8s1_5 cniAssetHash = defaultCNIAssetHashStringK8s1_5 - glog.V(2).Infof("Adding default CNI asset for k8s 1.5: %s", defaultCNIAssetK8s1_5) + glog.V(2).Infof("Adding default CNI asset for 1.6 > k8s >= 1.5: %s", defaultCNIAssetK8s1_5) } u, err := url.Parse(cniAsset) diff --git a/upup/pkg/fi/cloudup/subnets.go b/upup/pkg/fi/cloudup/subnets.go index 36278bb563db5..4ca4f01eed68e 100644 --- a/upup/pkg/fi/cloudup/subnets.go +++ b/upup/pkg/fi/cloudup/subnets.go @@ -17,13 +17,13 @@ limitations under the License. package cloudup import ( - "encoding/binary" "fmt" "net" "sort" "github.com/golang/glog" "k8s.io/kops/pkg/apis/kops" + "k8s.io/kops/pkg/util/subnet" "k8s.io/kops/upup/pkg/fi" ) @@ -109,7 +109,7 @@ func assignCIDRsToSubnets(c *kops.Cluster) error { // TODO: Does this make sense on GCE? // TODO: Should we limit this to say 1000 IPs per subnet? (any reason to?) - bigCIDRs, err := splitInto8Subnets(cidr) + bigCIDRs, err := subnet.SplitInto8(cidr) if err != nil { return err } @@ -147,7 +147,7 @@ func assignCIDRsToSubnets(c *kops.Cluster) error { for _, c := range bigCIDRs { overlapped := false for _, r := range reserved { - if cidrsOverlap(r, c) { + if subnet.Overlap(r, c) { overlapped = true } } @@ -162,7 +162,7 @@ func assignCIDRsToSubnets(c *kops.Cluster) error { return fmt.Errorf("could not find any non-overlapping CIDRs in parent NetworkCIDR; cannot automatically assign CIDR to subnet") } - littleCIDRs, err := splitInto8Subnets(bigCIDRs[0]) + littleCIDRs, err := subnet.SplitInto8(bigCIDRs[0]) if err != nil { return err } @@ -203,32 +203,6 @@ func assignCIDRsToSubnets(c *kops.Cluster) error { return nil } -// splitInto8Subnets splits the parent IPNet into 8 subnets -func splitInto8Subnets(parent *net.IPNet) ([]*net.IPNet, error) { - networkLength, _ := parent.Mask.Size() - networkLength += 3 - - var subnets []*net.IPNet - for i := 0; i < 8; i++ { - ip4 := parent.IP.To4() - if ip4 != nil { - n := binary.BigEndian.Uint32(ip4) - n += uint32(i) << uint(32-networkLength) - subnetIP := make(net.IP, len(ip4)) - binary.BigEndian.PutUint32(subnetIP, n) - - subnets = append(subnets, &net.IPNet{ - IP: subnetIP, - Mask: net.CIDRMask(networkLength, 32), - }) - } else { - return nil, fmt.Errorf("Unexpected IP address type: %s", parent) - } - } - - return subnets, nil -} - // allSubnetsHaveCIDRs returns true iff each subnet in the cluster has a non-empty CIDR func allSubnetsHaveCIDRs(c *kops.Cluster) bool { for i := range c.Spec.Subnets { @@ -240,8 +214,3 @@ func allSubnetsHaveCIDRs(c *kops.Cluster) bool { return true } - -// cidrsOverlap returns true iff the two CIDRs are non-disjoint -func cidrsOverlap(l, r *net.IPNet) bool { - return l.Contains(r.IP) || r.Contains(l.IP) -} diff --git a/upup/pkg/fi/cloudup/subnets_test.go b/upup/pkg/fi/cloudup/subnets_test.go index c41e1ce08138c..27e496de0ba6d 100644 --- a/upup/pkg/fi/cloudup/subnets_test.go +++ b/upup/pkg/fi/cloudup/subnets_test.go @@ -17,48 +17,12 @@ limitations under the License. package cloudup import ( - "net" "reflect" "testing" "k8s.io/kops/pkg/apis/kops" ) -func Test_Split_Subnet(t *testing.T) { - tests := []struct { - parent string - expected []string - }{ - { - parent: "1.2.3.0/24", - expected: []string{"1.2.3.0/27", "1.2.3.32/27", "1.2.3.64/27", "1.2.3.96/27", "1.2.3.128/27", "1.2.3.160/27", "1.2.3.192/27", "1.2.3.224/27"}, - }, - { - parent: "1.2.3.0/27", - expected: []string{"1.2.3.0/30", "1.2.3.4/30", "1.2.3.8/30", "1.2.3.12/30", "1.2.3.16/30", "1.2.3.20/30", "1.2.3.24/30", "1.2.3.28/30"}, - }, - } - for _, test := range tests { - _, parent, err := net.ParseCIDR(test.parent) - if err != nil { - t.Fatalf("error parsing parent cidr %q: %v", test.parent, err) - } - - subnets, err := splitInto8Subnets(parent) - if err != nil { - t.Fatalf("error splitting parent cidr %q: %v", parent, err) - } - - var actual []string - for _, subnet := range subnets { - actual = append(actual, subnet.String()) - } - if !reflect.DeepEqual(actual, test.expected) { - t.Fatalf("unexpected result of split: actual=%v, expected=%v", actual, test.expected) - } - } -} - func Test_AssignSubnets(t *testing.T) { tests := []struct { subnets []kops.ClusterSubnetSpec diff --git a/upup/pkg/fi/cloudup/tagbuilder.go b/upup/pkg/fi/cloudup/tagbuilder.go index d5b745801a514..84462d32057f7 100644 --- a/upup/pkg/fi/cloudup/tagbuilder.go +++ b/upup/pkg/fi/cloudup/tagbuilder.go @@ -96,6 +96,16 @@ func buildCloudupTags(cluster *api.Cluster) (sets.String, error) { func buildNodeupTags(role api.InstanceGroupRole, cluster *api.Cluster, clusterTags sets.String) (sets.String, error) { tags := sets.NewString() + networking := cluster.Spec.Networking + + if networking == nil { + return nil, fmt.Errorf("Networking is not set, and should not be nil here") + } + + if networking.LyftVPC != nil { + tags.Insert("_lyft_vpc_cni") + } + switch fi.StringValue(cluster.Spec.UpdatePolicy) { case "": // default tags.Insert("_automatic_upgrades") diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/manifest.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/manifest.yaml index eb4d052a07221..cc6a57b112d99 100644 --- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/manifest.yaml +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/manifest.yaml @@ -41,14 +41,14 @@ spec: name: dns-controller.addons.k8s.io selector: k8s-addon: dns-controller.addons.k8s.io - version: 1.11.0-alpha.1 + version: 1.11.0 - id: k8s-1.6 kubernetesVersion: '>=1.6.0' manifest: dns-controller.addons.k8s.io/k8s-1.6.yaml name: dns-controller.addons.k8s.io selector: k8s-addon: dns-controller.addons.k8s.io - version: 1.11.0-alpha.1 + version: 1.11.0 - id: v1.7.0 kubernetesVersion: '>=1.7.0' manifest: storage-aws.addons.k8s.io/v1.7.0.yaml diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/kopeio-vxlan/manifest.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/kopeio-vxlan/manifest.yaml index bf8129bfc4dd5..627792c5c7c37 100644 --- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/kopeio-vxlan/manifest.yaml +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/kopeio-vxlan/manifest.yaml @@ -41,14 +41,14 @@ spec: name: dns-controller.addons.k8s.io selector: k8s-addon: dns-controller.addons.k8s.io - version: 1.11.0-alpha.1 + version: 1.11.0 - id: k8s-1.6 kubernetesVersion: '>=1.6.0' manifest: dns-controller.addons.k8s.io/k8s-1.6.yaml name: dns-controller.addons.k8s.io selector: k8s-addon: dns-controller.addons.k8s.io - version: 1.11.0-alpha.1 + version: 1.11.0 - id: v1.7.0 kubernetesVersion: '>=1.7.0' manifest: storage-aws.addons.k8s.io/v1.7.0.yaml @@ -69,11 +69,11 @@ spec: name: networking.kope.io selector: role.kubernetes.io/networking: "1" - version: 1.0.20180319-kops.2 + version: 1.0.20181028-kops.1 - id: k8s-1.6 kubernetesVersion: '>=1.6.0' manifest: networking.kope.io/k8s-1.6.yaml name: networking.kope.io selector: role.kubernetes.io/networking: "1" - version: 1.0.20180319-kops.2 + version: 1.0.20181028-kops.1 diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/simple/manifest.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/simple/manifest.yaml index 86002b53a2e7b..3fb96ffe3ba1e 100644 --- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/simple/manifest.yaml +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/simple/manifest.yaml @@ -41,14 +41,14 @@ spec: name: dns-controller.addons.k8s.io selector: k8s-addon: dns-controller.addons.k8s.io - version: 1.11.0-alpha.1 + version: 1.11.0 - id: k8s-1.6 kubernetesVersion: '>=1.6.0' manifest: dns-controller.addons.k8s.io/k8s-1.6.yaml name: dns-controller.addons.k8s.io selector: k8s-addon: dns-controller.addons.k8s.io - version: 1.11.0-alpha.1 + version: 1.11.0 - id: v1.7.0 kubernetesVersion: '>=1.7.0' manifest: storage-aws.addons.k8s.io/v1.7.0.yaml diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/weave/manifest.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/weave/manifest.yaml index 18f88d3286ccc..ba5401db099a1 100644 --- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/weave/manifest.yaml +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/weave/manifest.yaml @@ -41,14 +41,14 @@ spec: name: dns-controller.addons.k8s.io selector: k8s-addon: dns-controller.addons.k8s.io - version: 1.11.0-alpha.1 + version: 1.11.0 - id: k8s-1.6 kubernetesVersion: '>=1.6.0' manifest: dns-controller.addons.k8s.io/k8s-1.6.yaml name: dns-controller.addons.k8s.io selector: k8s-addon: dns-controller.addons.k8s.io - version: 1.11.0-alpha.1 + version: 1.11.0 - id: v1.7.0 kubernetesVersion: '>=1.7.0' manifest: storage-aws.addons.k8s.io/v1.7.0.yaml @@ -83,11 +83,11 @@ spec: name: networking.weave selector: role.kubernetes.io/networking: "1" - version: 2.4.1-kops.1 + version: 2.5.0-kops.1 - id: k8s-1.8 kubernetesVersion: '>=1.8.0' manifest: networking.weave/k8s-1.8.yaml name: networking.weave selector: role.kubernetes.io/networking: "1" - version: 2.4.1-kops.1 + version: 2.5.0-kops.1 diff --git a/upup/pkg/fi/fitasks/mirrorsecrets.go b/upup/pkg/fi/fitasks/mirrorsecrets.go index 407788a06b61e..afbf2339c82f4 100644 --- a/upup/pkg/fi/fitasks/mirrorsecrets.go +++ b/upup/pkg/fi/fitasks/mirrorsecrets.go @@ -57,7 +57,7 @@ func (e *MirrorSecrets) Find(c *fi.Context) (*MirrorSecrets, error) { return nil, nil } -// Run implemements fi.Task::Run +// Run implements fi.Task::Run func (e *MirrorSecrets) Run(c *fi.Context) error { return fi.DefaultDeltaRunMethod(e, c) } diff --git a/upup/pkg/fi/nodeup/BUILD.bazel b/upup/pkg/fi/nodeup/BUILD.bazel index 5fc32601304bd..ec9cbaa7eaa7e 100644 --- a/upup/pkg/fi/nodeup/BUILD.bazel +++ b/upup/pkg/fi/nodeup/BUILD.bazel @@ -23,6 +23,11 @@ go_library( "//upup/pkg/fi/secrets:go_default_library", "//upup/pkg/fi/utils:go_default_library", "//util/pkg/vfs:go_default_library", + "//vendor/github.com/aws/aws-sdk-go/aws:go_default_library", + "//vendor/github.com/aws/aws-sdk-go/aws/ec2metadata:go_default_library", + "//vendor/github.com/aws/aws-sdk-go/aws/request:go_default_library", + "//vendor/github.com/aws/aws-sdk-go/aws/session:go_default_library", + "//vendor/github.com/aws/aws-sdk-go/service/ec2:go_default_library", "//vendor/github.com/golang/glog:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library", ], diff --git a/upup/pkg/fi/nodeup/command.go b/upup/pkg/fi/nodeup/command.go index d3ecc2ab26ba9..c364608e0958e 100644 --- a/upup/pkg/fi/nodeup/command.go +++ b/upup/pkg/fi/nodeup/command.go @@ -17,6 +17,7 @@ limitations under the License. package nodeup import ( + "encoding/json" "errors" "fmt" "io" @@ -27,8 +28,12 @@ import ( "strings" "time" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/ec2metadata" + "github.com/aws/aws-sdk-go/aws/request" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/ec2" "github.com/golang/glog" - "k8s.io/apimachinery/pkg/util/sets" "k8s.io/kops/nodeup/pkg/distros" "k8s.io/kops/nodeup/pkg/model" @@ -238,6 +243,7 @@ func (c *NodeUpCommand) Run(out io.Writer) error { loader.Builders = append(loader.Builders, &model.KubeAPIServerBuilder{NodeupModelContext: modelContext}) loader.Builders = append(loader.Builders, &model.KubeControllerManagerBuilder{NodeupModelContext: modelContext}) loader.Builders = append(loader.Builders, &model.KubeSchedulerBuilder{NodeupModelContext: modelContext}) + loader.Builders = append(loader.Builders, &model.EtcdManagerTLSBuilder{NodeupModelContext: modelContext}) if c.cluster.Spec.Networking.Kuberouter == nil { loader.Builders = append(loader.Builders, &model.KubeProxyBuilder{NodeupModelContext: modelContext}) } else { @@ -247,6 +253,37 @@ func (c *NodeUpCommand) Run(out io.Writer) error { loader.Builders = append(loader.Builders, &model.EtcdTLSBuilder{NodeupModelContext: modelContext}) } + if c.cluster.Spec.Networking.LyftVPC != nil { + + loader.TemplateFunctions["SubnetTags"] = func() (string, error) { + tags := map[string]string{ + "Type": "pod", + } + if len(c.cluster.Spec.Networking.LyftVPC.SubnetTags) > 0 { + tags = c.cluster.Spec.Networking.LyftVPC.SubnetTags + } + + bytes, err := json.Marshal(tags) + if err != nil { + return "", err + } + return string(bytes), nil + } + + loader.TemplateFunctions["NodeSecurityGroups"] = func() (string, error) { + // use the same security groups as the node + ids, err := evaluateSecurityGroups(c.cluster.Spec.NetworkID) + if err != nil { + return "", err + } + bytes, err := json.Marshal(ids) + if err != nil { + return "", err + } + return string(bytes), nil + } + } + taskMap, err := loader.Build(c.ModelDir) if err != nil { return fmt.Errorf("error building loader: %v", err) @@ -343,6 +380,57 @@ func evaluateSpec(c *api.Cluster) error { return nil } +func evaluateSecurityGroups(vpcId string) ([]string, error) { + config := aws.NewConfig() + config = config.WithCredentialsChainVerboseErrors(true) + + s, err := session.NewSession(config) + if err != nil { + return nil, fmt.Errorf("error starting new AWS session: %v", err) + } + s.Handlers.Send.PushFront(func(r *request.Request) { + // Log requests + glog.V(4).Infof("AWS API Request: %s/%s", r.ClientInfo.ServiceName, r.Operation.Name) + }) + + metadata := ec2metadata.New(s, config) + + region, err := metadata.Region() + if err != nil { + return nil, fmt.Errorf("error querying ec2 metadata service (for az/region): %v", err) + } + + sgNames, err := metadata.GetMetadata("security-groups") + if err != nil { + return nil, fmt.Errorf("error querying ec2 metadata service (for security-groups): %v", err) + } + svc := ec2.New(s, config.WithRegion(region)) + + result, err := svc.DescribeSecurityGroups(&ec2.DescribeSecurityGroupsInput{ + Filters: []*ec2.Filter{ + { + Name: aws.String("group-name"), + Values: aws.StringSlice(strings.Fields(sgNames)), + }, + { + Name: aws.String("vpc-id"), + Values: []*string{aws.String(vpcId)}, + }, + }, + }) + + if err != nil { + return nil, fmt.Errorf("error looking up instance security group ids: %v", err) + } + var sgIds []string + for _, group := range result.SecurityGroups { + sgIds = append(sgIds, *group.GroupId) + } + + return sgIds, nil + +} + func evaluateHostnameOverride(hostnameOverride string) (string, error) { if hostnameOverride == "" || hostnameOverride == "@hostname" { return "", nil diff --git a/upup/pkg/fi/nodeup/nodetasks/BUILD.bazel b/upup/pkg/fi/nodeup/nodetasks/BUILD.bazel index 2f9eb96f4f0e7..5c7e4dfb5901c 100644 --- a/upup/pkg/fi/nodeup/nodetasks/BUILD.bazel +++ b/upup/pkg/fi/nodeup/nodetasks/BUILD.bazel @@ -6,6 +6,7 @@ go_library( "archive.go", "asset.go", "bindmount.go", + "chattr.go", "createsdir.go", "file.go", "group.go", diff --git a/upup/pkg/fi/nodeup/nodetasks/chattr.go b/upup/pkg/fi/nodeup/nodetasks/chattr.go new file mode 100644 index 0000000000000..0e2551eb32073 --- /dev/null +++ b/upup/pkg/fi/nodeup/nodetasks/chattr.go @@ -0,0 +1,90 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package nodetasks + +import ( + "fmt" + "strings" + + "github.com/golang/glog" + "k8s.io/kops/upup/pkg/fi" + "k8s.io/kops/upup/pkg/fi/nodeup/cloudinit" + "k8s.io/kops/upup/pkg/fi/nodeup/local" +) + +// Chattr performs a chattr command, in particular to set a file as immutable +type Chattr struct { + File string `json:"file"` + Mode string `json:"mode"` + + Deps []fi.Task `json:"-"` +} + +var _ fi.Task = &Chattr{} + +func (s *Chattr) String() string { + return fmt.Sprintf("Chattr: chattr %s %s", s.Mode, s.File) +} + +var _ fi.HasName = &Archive{} + +func (e *Chattr) GetName() *string { + return fi.String("Chattr-" + e.File) +} + +func (e *Chattr) SetName(name string) { + glog.Fatalf("SetName not supported for Chattr task") +} + +var _ fi.HasDependencies = &Chattr{} + +// GetDependencies implements HasDependencies::GetDependencies +func (e *Chattr) GetDependencies(tasks map[string]fi.Task) []fi.Task { + return e.Deps +} + +func (e *Chattr) Find(c *fi.Context) (*Chattr, error) { + // We always re-run the chattr command + return nil, nil +} + +func (e *Chattr) Run(c *fi.Context) error { + return fi.DefaultDeltaRunMethod(e, c) +} + +func (s *Chattr) CheckChanges(a, e, changes *Chattr) error { + return nil +} + +func (_ *Chattr) RenderLocal(t *local.LocalTarget, a, e, changes *Chattr) error { + return e.execute(t) +} + +func (e *Chattr) execute(t Executor) error { + chattrCommand := []string{"chattr", e.Mode, e.File} + + glog.Infof("running chattr command chattr %s", chattrCommand) + if output, err := t.CombinedOutput(chattrCommand); err != nil { + return fmt.Errorf("error doing %q: %v: %s", strings.Join(chattrCommand, " "), err, string(output)) + } + + return nil +} + +func (_ *Chattr) RenderCloudInit(t *cloudinit.CloudInitTarget, a, e, changes *Chattr) error { + return fmt.Errorf("Chattr::RenderCloudInit not implemented") +} diff --git a/upup/pkg/fi/nodeup/nodetasks/service.go b/upup/pkg/fi/nodeup/nodetasks/service.go index 1a0f4e5bcf4d0..3939a0869df76 100644 --- a/upup/pkg/fi/nodeup/nodetasks/service.go +++ b/upup/pkg/fi/nodeup/nodetasks/service.go @@ -70,7 +70,7 @@ func (p *Service) GetDependencies(tasks map[string]fi.Task) []fi.Task { // launching a custom Kubernetes build), they all depend on // the "docker.service" Service task. switch v.(type) { - case *File, *Package, *UpdatePackages, *UserTask, *GroupTask, *MountDiskTask: + case *File, *Package, *UpdatePackages, *UserTask, *GroupTask, *MountDiskTask, *Chattr: deps = append(deps, v) case *Service, *LoadImageTask: // ignore diff --git a/upup/pkg/fi/secrets/clientset_secretstore.go b/upup/pkg/fi/secrets/clientset_secretstore.go index 59fb11d4abd1d..67f3841cd5594 100644 --- a/upup/pkg/fi/secrets/clientset_secretstore.go +++ b/upup/pkg/fi/secrets/clientset_secretstore.go @@ -194,7 +194,7 @@ func (c *ClientsetSecretStore) GetOrCreateSecret(name string, secret *fi.Secret) // Make double-sure it round-trips s, err := c.loadSecret(name) if err != nil { - glog.Fatalf("unable to load secret immmediately after creation %v: %v", name, err) + glog.Fatalf("unable to load secret immediately after creation %v: %v", name, err) return nil, false, err } return s, true, nil @@ -210,7 +210,7 @@ func (c *ClientsetSecretStore) ReplaceSecret(name string, secret *fi.Secret) (*f // Confirm the secret exists s, err := c.loadSecret(name) if err != nil { - return nil, fmt.Errorf("unable to load secret immmediately after creation: %v", err) + return nil, fmt.Errorf("unable to load secret immediately after creation: %v", err) } return s, nil } diff --git a/upup/pkg/fi/secrets/vfs_secretstore.go b/upup/pkg/fi/secrets/vfs_secretstore.go index f2c7b02cd8333..918b0a783d9e8 100644 --- a/upup/pkg/fi/secrets/vfs_secretstore.go +++ b/upup/pkg/fi/secrets/vfs_secretstore.go @@ -171,7 +171,7 @@ func (c *VFSSecretStore) GetOrCreateSecret(id string, secret *fi.Secret) (*fi.Se // Make double-sure it round-trips s, err := c.loadSecret(p) if err != nil { - glog.Fatalf("unable to load secret immmediately after creation %v: %v", p, err) + glog.Fatalf("unable to load secret immediately after creation %v: %v", p, err) return nil, false, err } return s, true, nil @@ -193,7 +193,7 @@ func (c *VFSSecretStore) ReplaceSecret(id string, secret *fi.Secret) (*fi.Secret // Confirm the secret exists s, err := c.loadSecret(p) if err != nil { - return nil, fmt.Errorf("unable to load secret immmediately after creation %v: %v", p, err) + return nil, fmt.Errorf("unable to load secret immediately after creation %v: %v", p, err) } return s, nil } diff --git a/upup/pkg/kutil/import_cluster.go b/upup/pkg/kutil/import_cluster.go index c9c38e0db4425..0c73d21468fcf 100644 --- a/upup/pkg/kutil/import_cluster.go +++ b/upup/pkg/kutil/import_cluster.go @@ -324,7 +324,7 @@ func (x *ImportCluster) ImportAWSCluster() error { } if launchConfiguration == nil { - glog.Warningf("LaunchConfiguration %q not found; ignoring", name) + glog.Warningf("ignoring error launchConfiguration %q not found", name) continue } diff --git a/vendor/github.com/docker/docker/contrib/selinux-fedora-24/docker-engine-selinux/LICENSE b/vendor/github.com/docker/docker/contrib/selinux-fedora-24/docker-engine-selinux/LICENSE deleted file mode 100644 index d511905c1647a..0000000000000 --- a/vendor/github.com/docker/docker/contrib/selinux-fedora-24/docker-engine-selinux/LICENSE +++ /dev/null @@ -1,339 +0,0 @@ - GNU GENERAL PUBLIC LICENSE - Version 2, June 1991 - - Copyright (C) 1989, 1991 Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. - - Preamble - - The licenses for most software are designed to take away your -freedom to share and change it. By contrast, the GNU General Public -License is intended to guarantee your freedom to share and change free -software--to make sure the software is free for all its users. This -General Public License applies to most of the Free Software -Foundation's software and to any other program whose authors commit to -using it. (Some other Free Software Foundation software is covered by -the GNU Lesser General Public License instead.) You can apply it to -your programs, too. - - When we speak of free software, we are referring to freedom, not -price. Our General Public Licenses are designed to make sure that you -have the freedom to distribute copies of free software (and charge for -this service if you wish), that you receive source code or can get it -if you want it, that you can change the software or use pieces of it -in new free programs; and that you know you can do these things. - - To protect your rights, we need to make restrictions that forbid -anyone to deny you these rights or to ask you to surrender the rights. -These restrictions translate to certain responsibilities for you if you -distribute copies of the software, or if you modify it. - - For example, if you distribute copies of such a program, whether -gratis or for a fee, you must give the recipients all the rights that -you have. You must make sure that they, too, receive or can get the -source code. And you must show them these terms so they know their -rights. - - We protect your rights with two steps: (1) copyright the software, and -(2) offer you this license which gives you legal permission to copy, -distribute and/or modify the software. - - Also, for each author's protection and ours, we want to make certain -that everyone understands that there is no warranty for this free -software. If the software is modified by someone else and passed on, we -want its recipients to know that what they have is not the original, so -that any problems introduced by others will not reflect on the original -authors' reputations. - - Finally, any free program is threatened constantly by software -patents. We wish to avoid the danger that redistributors of a free -program will individually obtain patent licenses, in effect making the -program proprietary. To prevent this, we have made it clear that any -patent must be licensed for everyone's free use or not licensed at all. - - The precise terms and conditions for copying, distribution and -modification follow. - - GNU GENERAL PUBLIC LICENSE - TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION - - 0. This License applies to any program or other work which contains -a notice placed by the copyright holder saying it may be distributed -under the terms of this General Public License. The "Program", below, -refers to any such program or work, and a "work based on the Program" -means either the Program or any derivative work under copyright law: -that is to say, a work containing the Program or a portion of it, -either verbatim or with modifications and/or translated into another -language. (Hereinafter, translation is included without limitation in -the term "modification".) Each licensee is addressed as "you". - -Activities other than copying, distribution and modification are not -covered by this License; they are outside its scope. The act of -running the Program is not restricted, and the output from the Program -is covered only if its contents constitute a work based on the -Program (independent of having been made by running the Program). -Whether that is true depends on what the Program does. - - 1. You may copy and distribute verbatim copies of the Program's -source code as you receive it, in any medium, provided that you -conspicuously and appropriately publish on each copy an appropriate -copyright notice and disclaimer of warranty; keep intact all the -notices that refer to this License and to the absence of any warranty; -and give any other recipients of the Program a copy of this License -along with the Program. - -You may charge a fee for the physical act of transferring a copy, and -you may at your option offer warranty protection in exchange for a fee. - - 2. You may modify your copy or copies of the Program or any portion -of it, thus forming a work based on the Program, and copy and -distribute such modifications or work under the terms of Section 1 -above, provided that you also meet all of these conditions: - - a) You must cause the modified files to carry prominent notices - stating that you changed the files and the date of any change. - - b) You must cause any work that you distribute or publish, that in - whole or in part contains or is derived from the Program or any - part thereof, to be licensed as a whole at no charge to all third - parties under the terms of this License. - - c) If the modified program normally reads commands interactively - when run, you must cause it, when started running for such - interactive use in the most ordinary way, to print or display an - announcement including an appropriate copyright notice and a - notice that there is no warranty (or else, saying that you provide - a warranty) and that users may redistribute the program under - these conditions, and telling the user how to view a copy of this - License. (Exception: if the Program itself is interactive but - does not normally print such an announcement, your work based on - the Program is not required to print an announcement.) - -These requirements apply to the modified work as a whole. If -identifiable sections of that work are not derived from the Program, -and can be reasonably considered independent and separate works in -themselves, then this License, and its terms, do not apply to those -sections when you distribute them as separate works. But when you -distribute the same sections as part of a whole which is a work based -on the Program, the distribution of the whole must be on the terms of -this License, whose permissions for other licensees extend to the -entire whole, and thus to each and every part regardless of who wrote it. - -Thus, it is not the intent of this section to claim rights or contest -your rights to work written entirely by you; rather, the intent is to -exercise the right to control the distribution of derivative or -collective works based on the Program. - -In addition, mere aggregation of another work not based on the Program -with the Program (or with a work based on the Program) on a volume of -a storage or distribution medium does not bring the other work under -the scope of this License. - - 3. You may copy and distribute the Program (or a work based on it, -under Section 2) in object code or executable form under the terms of -Sections 1 and 2 above provided that you also do one of the following: - - a) Accompany it with the complete corresponding machine-readable - source code, which must be distributed under the terms of Sections - 1 and 2 above on a medium customarily used for software interchange; or, - - b) Accompany it with a written offer, valid for at least three - years, to give any third party, for a charge no more than your - cost of physically performing source distribution, a complete - machine-readable copy of the corresponding source code, to be - distributed under the terms of Sections 1 and 2 above on a medium - customarily used for software interchange; or, - - c) Accompany it with the information you received as to the offer - to distribute corresponding source code. (This alternative is - allowed only for noncommercial distribution and only if you - received the program in object code or executable form with such - an offer, in accord with Subsection b above.) - -The source code for a work means the preferred form of the work for -making modifications to it. For an executable work, complete source -code means all the source code for all modules it contains, plus any -associated interface definition files, plus the scripts used to -control compilation and installation of the executable. However, as a -special exception, the source code distributed need not include -anything that is normally distributed (in either source or binary -form) with the major components (compiler, kernel, and so on) of the -operating system on which the executable runs, unless that component -itself accompanies the executable. - -If distribution of executable or object code is made by offering -access to copy from a designated place, then offering equivalent -access to copy the source code from the same place counts as -distribution of the source code, even though third parties are not -compelled to copy the source along with the object code. - - 4. You may not copy, modify, sublicense, or distribute the Program -except as expressly provided under this License. Any attempt -otherwise to copy, modify, sublicense or distribute the Program is -void, and will automatically terminate your rights under this License. -However, parties who have received copies, or rights, from you under -this License will not have their licenses terminated so long as such -parties remain in full compliance. - - 5. You are not required to accept this License, since you have not -signed it. However, nothing else grants you permission to modify or -distribute the Program or its derivative works. These actions are -prohibited by law if you do not accept this License. Therefore, by -modifying or distributing the Program (or any work based on the -Program), you indicate your acceptance of this License to do so, and -all its terms and conditions for copying, distributing or modifying -the Program or works based on it. - - 6. Each time you redistribute the Program (or any work based on the -Program), the recipient automatically receives a license from the -original licensor to copy, distribute or modify the Program subject to -these terms and conditions. You may not impose any further -restrictions on the recipients' exercise of the rights granted herein. -You are not responsible for enforcing compliance by third parties to -this License. - - 7. If, as a consequence of a court judgment or allegation of patent -infringement or for any other reason (not limited to patent issues), -conditions are imposed on you (whether by court order, agreement or -otherwise) that contradict the conditions of this License, they do not -excuse you from the conditions of this License. If you cannot -distribute so as to satisfy simultaneously your obligations under this -License and any other pertinent obligations, then as a consequence you -may not distribute the Program at all. For example, if a patent -license would not permit royalty-free redistribution of the Program by -all those who receive copies directly or indirectly through you, then -the only way you could satisfy both it and this License would be to -refrain entirely from distribution of the Program. - -If any portion of this section is held invalid or unenforceable under -any particular circumstance, the balance of the section is intended to -apply and the section as a whole is intended to apply in other -circumstances. - -It is not the purpose of this section to induce you to infringe any -patents or other property right claims or to contest validity of any -such claims; this section has the sole purpose of protecting the -integrity of the free software distribution system, which is -implemented by public license practices. Many people have made -generous contributions to the wide range of software distributed -through that system in reliance on consistent application of that -system; it is up to the author/donor to decide if he or she is willing -to distribute software through any other system and a licensee cannot -impose that choice. - -This section is intended to make thoroughly clear what is believed to -be a consequence of the rest of this License. - - 8. If the distribution and/or use of the Program is restricted in -certain countries either by patents or by copyrighted interfaces, the -original copyright holder who places the Program under this License -may add an explicit geographical distribution limitation excluding -those countries, so that distribution is permitted only in or among -countries not thus excluded. In such case, this License incorporates -the limitation as if written in the body of this License. - - 9. The Free Software Foundation may publish revised and/or new versions -of the General Public License from time to time. Such new versions will -be similar in spirit to the present version, but may differ in detail to -address new problems or concerns. - -Each version is given a distinguishing version number. If the Program -specifies a version number of this License which applies to it and "any -later version", you have the option of following the terms and conditions -either of that version or of any later version published by the Free -Software Foundation. If the Program does not specify a version number of -this License, you may choose any version ever published by the Free Software -Foundation. - - 10. If you wish to incorporate parts of the Program into other free -programs whose distribution conditions are different, write to the author -to ask for permission. For software which is copyrighted by the Free -Software Foundation, write to the Free Software Foundation; we sometimes -make exceptions for this. Our decision will be guided by the two goals -of preserving the free status of all derivatives of our free software and -of promoting the sharing and reuse of software generally. - - NO WARRANTY - - 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY -FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN -OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES -PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED -OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS -TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE -PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, -REPAIR OR CORRECTION. - - 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING -WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR -REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, -INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING -OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED -TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY -YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER -PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE -POSSIBILITY OF SUCH DAMAGES. - - END OF TERMS AND CONDITIONS - - How to Apply These Terms to Your New Programs - - If you develop a new program, and you want it to be of the greatest -possible use to the public, the best way to achieve this is to make it -free software which everyone can redistribute and change under these terms. - - To do so, attach the following notices to the program. It is safest -to attach them to the start of each source file to most effectively -convey the exclusion of warranty; and each file should have at least -the "copyright" line and a pointer to where the full notice is found. - - - Copyright (C) - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -Also add information on how to contact you by electronic and paper mail. - -If the program is interactive, make it output a short notice like this -when it starts in an interactive mode: - - Gnomovision version 69, Copyright (C) year name of author - Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. - This is free software, and you are welcome to redistribute it - under certain conditions; type `show c' for details. - -The hypothetical commands `show w' and `show c' should show the appropriate -parts of the General Public License. Of course, the commands you use may -be called something other than `show w' and `show c'; they could even be -mouse-clicks or menu items--whatever suits your program. - -You should also get your employer (if you work as a programmer) or your -school, if any, to sign a "copyright disclaimer" for the program, if -necessary. Here is a sample; alter the names: - - Yoyodyne, Inc., hereby disclaims all copyright interest in the program - `Gnomovision' (which makes passes at compilers) written by James Hacker. - - , 1 April 1989 - Ty Coon, President of Vice - -This General Public License does not permit incorporating your program into -proprietary programs. If your program is a subroutine library, you may -consider it more useful to permit linking proprietary applications with the -library. If this is what you want to do, use the GNU Lesser General -Public License instead of this License. diff --git a/vendor/github.com/docker/docker/contrib/selinux-oraclelinux-7/docker-engine-selinux/LICENSE b/vendor/github.com/docker/docker/contrib/selinux-oraclelinux-7/docker-engine-selinux/LICENSE deleted file mode 100644 index 5b6e7c66c276e..0000000000000 --- a/vendor/github.com/docker/docker/contrib/selinux-oraclelinux-7/docker-engine-selinux/LICENSE +++ /dev/null @@ -1,340 +0,0 @@ - GNU GENERAL PUBLIC LICENSE - Version 2, June 1991 - - Copyright (C) 1989, 1991 Free Software Foundation, Inc. - 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. - - Preamble - - The licenses for most software are designed to take away your -freedom to share and change it. By contrast, the GNU General Public -License is intended to guarantee your freedom to share and change free -software--to make sure the software is free for all its users. This -General Public License applies to most of the Free Software -Foundation's software and to any other program whose authors commit to -using it. (Some other Free Software Foundation software is covered by -the GNU Library General Public License instead.) You can apply it to -your programs, too. - - When we speak of free software, we are referring to freedom, not -price. Our General Public Licenses are designed to make sure that you -have the freedom to distribute copies of free software (and charge for -this service if you wish), that you receive source code or can get it -if you want it, that you can change the software or use pieces of it -in new free programs; and that you know you can do these things. - - To protect your rights, we need to make restrictions that forbid -anyone to deny you these rights or to ask you to surrender the rights. -These restrictions translate to certain responsibilities for you if you -distribute copies of the software, or if you modify it. - - For example, if you distribute copies of such a program, whether -gratis or for a fee, you must give the recipients all the rights that -you have. You must make sure that they, too, receive or can get the -source code. And you must show them these terms so they know their -rights. - - We protect your rights with two steps: (1) copyright the software, and -(2) offer you this license which gives you legal permission to copy, -distribute and/or modify the software. - - Also, for each author's protection and ours, we want to make certain -that everyone understands that there is no warranty for this free -software. If the software is modified by someone else and passed on, we -want its recipients to know that what they have is not the original, so -that any problems introduced by others will not reflect on the original -authors' reputations. - - Finally, any free program is threatened constantly by software -patents. We wish to avoid the danger that redistributors of a free -program will individually obtain patent licenses, in effect making the -program proprietary. To prevent this, we have made it clear that any -patent must be licensed for everyone's free use or not licensed at all. - - The precise terms and conditions for copying, distribution and -modification follow. - - GNU GENERAL PUBLIC LICENSE - TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION - - 0. This License applies to any program or other work which contains -a notice placed by the copyright holder saying it may be distributed -under the terms of this General Public License. The "Program", below, -refers to any such program or work, and a "work based on the Program" -means either the Program or any derivative work under copyright law: -that is to say, a work containing the Program or a portion of it, -either verbatim or with modifications and/or translated into another -language. (Hereinafter, translation is included without limitation in -the term "modification".) Each licensee is addressed as "you". - -Activities other than copying, distribution and modification are not -covered by this License; they are outside its scope. The act of -running the Program is not restricted, and the output from the Program -is covered only if its contents constitute a work based on the -Program (independent of having been made by running the Program). -Whether that is true depends on what the Program does. - - 1. You may copy and distribute verbatim copies of the Program's -source code as you receive it, in any medium, provided that you -conspicuously and appropriately publish on each copy an appropriate -copyright notice and disclaimer of warranty; keep intact all the -notices that refer to this License and to the absence of any warranty; -and give any other recipients of the Program a copy of this License -along with the Program. - -You may charge a fee for the physical act of transferring a copy, and -you may at your option offer warranty protection in exchange for a fee. - - 2. You may modify your copy or copies of the Program or any portion -of it, thus forming a work based on the Program, and copy and -distribute such modifications or work under the terms of Section 1 -above, provided that you also meet all of these conditions: - - a) You must cause the modified files to carry prominent notices - stating that you changed the files and the date of any change. - - b) You must cause any work that you distribute or publish, that in - whole or in part contains or is derived from the Program or any - part thereof, to be licensed as a whole at no charge to all third - parties under the terms of this License. - - c) If the modified program normally reads commands interactively - when run, you must cause it, when started running for such - interactive use in the most ordinary way, to print or display an - announcement including an appropriate copyright notice and a - notice that there is no warranty (or else, saying that you provide - a warranty) and that users may redistribute the program under - these conditions, and telling the user how to view a copy of this - License. (Exception: if the Program itself is interactive but - does not normally print such an announcement, your work based on - the Program is not required to print an announcement.) - -These requirements apply to the modified work as a whole. If -identifiable sections of that work are not derived from the Program, -and can be reasonably considered independent and separate works in -themselves, then this License, and its terms, do not apply to those -sections when you distribute them as separate works. But when you -distribute the same sections as part of a whole which is a work based -on the Program, the distribution of the whole must be on the terms of -this License, whose permissions for other licensees extend to the -entire whole, and thus to each and every part regardless of who wrote it. - -Thus, it is not the intent of this section to claim rights or contest -your rights to work written entirely by you; rather, the intent is to -exercise the right to control the distribution of derivative or -collective works based on the Program. - -In addition, mere aggregation of another work not based on the Program -with the Program (or with a work based on the Program) on a volume of -a storage or distribution medium does not bring the other work under -the scope of this License. - - 3. You may copy and distribute the Program (or a work based on it, -under Section 2) in object code or executable form under the terms of -Sections 1 and 2 above provided that you also do one of the following: - - a) Accompany it with the complete corresponding machine-readable - source code, which must be distributed under the terms of Sections - 1 and 2 above on a medium customarily used for software interchange; or, - - b) Accompany it with a written offer, valid for at least three - years, to give any third party, for a charge no more than your - cost of physically performing source distribution, a complete - machine-readable copy of the corresponding source code, to be - distributed under the terms of Sections 1 and 2 above on a medium - customarily used for software interchange; or, - - c) Accompany it with the information you received as to the offer - to distribute corresponding source code. (This alternative is - allowed only for noncommercial distribution and only if you - received the program in object code or executable form with such - an offer, in accord with Subsection b above.) - -The source code for a work means the preferred form of the work for -making modifications to it. For an executable work, complete source -code means all the source code for all modules it contains, plus any -associated interface definition files, plus the scripts used to -control compilation and installation of the executable. However, as a -special exception, the source code distributed need not include -anything that is normally distributed (in either source or binary -form) with the major components (compiler, kernel, and so on) of the -operating system on which the executable runs, unless that component -itself accompanies the executable. - -If distribution of executable or object code is made by offering -access to copy from a designated place, then offering equivalent -access to copy the source code from the same place counts as -distribution of the source code, even though third parties are not -compelled to copy the source along with the object code. - - 4. You may not copy, modify, sublicense, or distribute the Program -except as expressly provided under this License. Any attempt -otherwise to copy, modify, sublicense or distribute the Program is -void, and will automatically terminate your rights under this License. -However, parties who have received copies, or rights, from you under -this License will not have their licenses terminated so long as such -parties remain in full compliance. - - 5. You are not required to accept this License, since you have not -signed it. However, nothing else grants you permission to modify or -distribute the Program or its derivative works. These actions are -prohibited by law if you do not accept this License. Therefore, by -modifying or distributing the Program (or any work based on the -Program), you indicate your acceptance of this License to do so, and -all its terms and conditions for copying, distributing or modifying -the Program or works based on it. - - 6. Each time you redistribute the Program (or any work based on the -Program), the recipient automatically receives a license from the -original licensor to copy, distribute or modify the Program subject to -these terms and conditions. You may not impose any further -restrictions on the recipients' exercise of the rights granted herein. -You are not responsible for enforcing compliance by third parties to -this License. - - 7. If, as a consequence of a court judgment or allegation of patent -infringement or for any other reason (not limited to patent issues), -conditions are imposed on you (whether by court order, agreement or -otherwise) that contradict the conditions of this License, they do not -excuse you from the conditions of this License. If you cannot -distribute so as to satisfy simultaneously your obligations under this -License and any other pertinent obligations, then as a consequence you -may not distribute the Program at all. For example, if a patent -license would not permit royalty-free redistribution of the Program by -all those who receive copies directly or indirectly through you, then -the only way you could satisfy both it and this License would be to -refrain entirely from distribution of the Program. - -If any portion of this section is held invalid or unenforceable under -any particular circumstance, the balance of the section is intended to -apply and the section as a whole is intended to apply in other -circumstances. - -It is not the purpose of this section to induce you to infringe any -patents or other property right claims or to contest validity of any -such claims; this section has the sole purpose of protecting the -integrity of the free software distribution system, which is -implemented by public license practices. Many people have made -generous contributions to the wide range of software distributed -through that system in reliance on consistent application of that -system; it is up to the author/donor to decide if he or she is willing -to distribute software through any other system and a licensee cannot -impose that choice. - -This section is intended to make thoroughly clear what is believed to -be a consequence of the rest of this License. - - 8. If the distribution and/or use of the Program is restricted in -certain countries either by patents or by copyrighted interfaces, the -original copyright holder who places the Program under this License -may add an explicit geographical distribution limitation excluding -those countries, so that distribution is permitted only in or among -countries not thus excluded. In such case, this License incorporates -the limitation as if written in the body of this License. - - 9. The Free Software Foundation may publish revised and/or new versions -of the General Public License from time to time. Such new versions will -be similar in spirit to the present version, but may differ in detail to -address new problems or concerns. - -Each version is given a distinguishing version number. If the Program -specifies a version number of this License which applies to it and "any -later version", you have the option of following the terms and conditions -either of that version or of any later version published by the Free -Software Foundation. If the Program does not specify a version number of -this License, you may choose any version ever published by the Free Software -Foundation. - - 10. If you wish to incorporate parts of the Program into other free -programs whose distribution conditions are different, write to the author -to ask for permission. For software which is copyrighted by the Free -Software Foundation, write to the Free Software Foundation; we sometimes -make exceptions for this. Our decision will be guided by the two goals -of preserving the free status of all derivatives of our free software and -of promoting the sharing and reuse of software generally. - - NO WARRANTY - - 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY -FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN -OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES -PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED -OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS -TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE -PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, -REPAIR OR CORRECTION. - - 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING -WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR -REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, -INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING -OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED -TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY -YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER -PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE -POSSIBILITY OF SUCH DAMAGES. - - END OF TERMS AND CONDITIONS - - How to Apply These Terms to Your New Programs - - If you develop a new program, and you want it to be of the greatest -possible use to the public, the best way to achieve this is to make it -free software which everyone can redistribute and change under these terms. - - To do so, attach the following notices to the program. It is safest -to attach them to the start of each source file to most effectively -convey the exclusion of warranty; and each file should have at least -the "copyright" line and a pointer to where the full notice is found. - - - Copyright (C) - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - - -Also add information on how to contact you by electronic and paper mail. - -If the program is interactive, make it output a short notice like this -when it starts in an interactive mode: - - Gnomovision version 69, Copyright (C) year name of author - Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. - This is free software, and you are welcome to redistribute it - under certain conditions; type `show c' for details. - -The hypothetical commands `show w' and `show c' should show the appropriate -parts of the General Public License. Of course, the commands you use may -be called something other than `show w' and `show c'; they could even be -mouse-clicks or menu items--whatever suits your program. - -You should also get your employer (if you work as a programmer) or your -school, if any, to sign a "copyright disclaimer" for the program, if -necessary. Here is a sample; alter the names: - - Yoyodyne, Inc., hereby disclaims all copyright interest in the program - `Gnomovision' (which makes passes at compilers) written by James Hacker. - - , 1 April 1989 - Ty Coon, President of Vice - -This General Public License does not permit incorporating your program into -proprietary programs. If your program is a subroutine library, you may -consider it more useful to permit linking proprietary applications with the -library. If this is what you want to do, use the GNU Library General -Public License instead of this License. diff --git a/vendor/github.com/docker/docker/contrib/syntax/vim/LICENSE b/vendor/github.com/docker/docker/contrib/syntax/vim/LICENSE deleted file mode 100644 index e67cdabd22e5f..0000000000000 --- a/vendor/github.com/docker/docker/contrib/syntax/vim/LICENSE +++ /dev/null @@ -1,22 +0,0 @@ -Copyright (c) 2013 Honza Pokorny -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. -2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR -ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/version.go b/version.go index 729c505ce1dd5..7bd896afee3cb 100644 --- a/version.go +++ b/version.go @@ -23,8 +23,8 @@ var Version = KOPS_RELEASE_VERSION // These constants are parsed by build tooling - be careful about changing the formats const ( - KOPS_RELEASE_VERSION = "1.11.0-alpha.1" - KOPS_CI_VERSION = "1.11.0-alpha.2" + KOPS_RELEASE_VERSION = "1.11.1" + KOPS_CI_VERSION = "1.11.2-alpha.1" ) // GitVersion should be replaced by the makefile