k8s二次开发-kubebuiler一键式生成deployment,svc,ingress

一 Kubebuilder环境搭建

注：必须在当前的K8S集群有 nginx这个ingressclass

root@k8s:~# kubectl get ingressclass
NAME    CONTROLLER             PARAMETERS   AGE
nginx   k8s.io/ingress-nginx   <none>       19h

1.1 下载kubebuilder

wget https://github.com/kubernetes-sigs/kubebuilder/releases/download/v3.3.0/kubebuilder_linux_amd64 -O /usr/local/sbin/kubebuilder --no-check-certificate
chmod +x /usr/local/sbin/kubebuilderkubebuilder version
Version: main.version{KubeBuilderVersion:"3.3.0", KubernetesVendor:"1.23.1", GitCommit:"47859bf2ebf96a64db69a2f7074ffdec7f15c1ec", BuildDate:"2022-01-18T17:03:29Z", GoOs:"linux", GoArch:"amd64"}

1.2 下载Golang

wget https://studygolang.com/dl/golang/go1.19.7.linux-amd64.tar.gztar -C /usr/local -xzf go1.19.7.linux-amd64.tar.gzvim /etc/profileexport PATH=$PATH:/usr/local/go/bin
export GO111MODULE=on
export GOPROXY=https://goproxy.cn,direct
export PATH=$PATH:/root/go/binroot@k8s:~# go version
go version go1.19.7 linux/amd64

1.3 编译golang需要的文件

以下只是一个 Demo和后续开发的项目无关

mkdir -p /usr/local/kubebuilder/elasticwebgo mod init elasticweb
kubebuilder init --domain jcrose.top
mkdir binkubebuilder create api \
--group elasticweb \
--version v1 \
--kind ElasticWeb# 安装controller-gen,这个需要先 执行kubebuilder create api 才会产生这个包的
cd /root/go/pkg/mod/sigs.k8s.io/controller-tools@v0.8.0/cmd/controller-gen
go build -o controller-gen main.go
mv controller-gen /usr/local/sbin
cp /usr/local/sbin/controller-gen /usr/local/kubebuilder/elasticweb/binmake install #第一次会失败，因为缺乏 kustomize # 安装kustomize
cd /root/go/pkg/mod/sigs.k8s.io/kustomize/kustomize/v3@v3.8.7
go build -o kustomize main.go
cp kustomize /usr/local/sbin
cp kustomize /usr/local/kubebuilder/elasticweb/binmake install kubectl get crds|grep jcrose
elasticwebs.elasticweb.jcrose.top   2023-04-09T14:25:18Z

安装启动容器的时候初始化的环境包

# 官方文档 https://github.com/kubernetes-sigs/controller-runtime/tree/main/tools/setup-envtest
# 安装setup-envtest
# cd /root/go/pkg/mod/sigs.k8s.io/controller-runtime/tools/setup-envtest@v0.0.0-20230403212152-53057ba616d1
go install sigs.k8s.io/controller-runtime/tools/setup-envtest@release-0.17
cp /root/go/bin/setup-envtest /usr/local/sbin/

此时安装了 kustomize,controller-gen，setup-envtest 三个文件

二 初始化项目

kubebuilder init --domain jcrose.top
kubebuilder create api --group infra --version v1 --kind App

2.1 controllers/app_controller.go

/*
Copyright 2024.Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License athttp://www.apache.org/licenses/LICENSE-2.0Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/package controllersimport ("context""fmt"appsv1 "k8s.io/api/apps/v1"corev1 "k8s.io/api/core/v1"apinetv1 "k8s.io/api/networking/v1""k8s.io/apimachinery/pkg/api/errors""k8s.io/apimachinery/pkg/api/resource"metav1 "k8s.io/apimachinery/pkg/apis/meta/v1""k8s.io/apimachinery/pkg/runtime""k8s.io/klog/v2"ctrl "sigs.k8s.io/controller-runtime""sigs.k8s.io/controller-runtime/pkg/client""sigs.k8s.io/controller-runtime/pkg/controller/controllerutil""sigs.k8s.io/controller-runtime/pkg/reconcile"infrav1 "jcrose/api/v1"
)// AppReconciler reconciles a App object
type AppReconciler struct {client.ClientScheme *runtime.Scheme//Log    logr.Logger
}//+kubebuilder:rbac:groups=infra.jcrose.top,resources=apps,verbs=get;list;watch;create;update;patch;delete
//+kubebuilder:rbac:groups=infra.jcrose.top,resources=apps/status,verbs=get;update;patch
//+kubebuilder:rbac:groups=infra.jcrose.top,resources=apps/finalizers,verbs=update
//+kubebuilder:rbac:groups=core,resources=services,verbs=get;list;watch;create;update;patch;delete
//+kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;watch;create;update;patch;delete
//+kubebuilder:rbac:groups=networking.k8s.io,resources=ingresses,verbs=get;list;watch;create;update;patch;delete// Reconcile is part of the main kubernetes reconciliation loop which aims to
// move the current state of the cluster closer to the desired state.
// TODO(user): Modify the Reconcile function to compare the state specified by
// the App object against the actual cluster state, and then
// perform operations to make the cluster state reflect the state specified by
// the user.
//
// For more details, check Reconcile and its Result here:
// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.11.0/pkg/reconcile
func (r *AppReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {ctx = context.Background()fmt.Println("Name", req.Name)fmt.Println("Namespace", req.Namespace)fmt.Println("NamespacedName", req.NamespacedName)//log := r.Log.WithValues("yunlizhi infra app", req.NamespacedName)// log.Info("1. start reconcile logic")// TODO(user): your logic hereinstance := &infrav1.App{}// 通过客户端工具查询，查询条件是err := r.Get(ctx, req.NamespacedName, instance)if err != nil {// 如果没有实例，就返回空结果，这样外部就不再立即调用Reconcile方法了if errors.IsNotFound(err) {klog.Info("2.1. instance not found, maybe removed")return reconcile.Result{}, nil}klog.Error(err, "2.2 error")// 返回错误信息给外部return ctrl.Result{}, err}klog.Info("3. instance : " + instance.String())// 查找deploymentdeployment := &appsv1.Deployment{}// 用客户端工具查询err = r.Get(ctx, req.NamespacedName, deployment)// 查找时发生异常，以及查出来没有结果的处理逻辑if err != nil {// 如果没有实例就要创建了if errors.IsNotFound(err) {klog.Info("4. deployment not exists")// 如果对QPS没有需求，此时又没有deployment，就啥事都不做了// 先要创建serviceif err = createServiceIfNotExists(ctx, r, instance, req); err != nil {klog.Error(err, "5.2 error")// 返回错误信息给外部return ctrl.Result{}, err}// 立即创建deploymentif err = createDeploymentIfNotExists(ctx, r, instance, req); err != nil {klog.Error(err, "5.3 error")// 返回错误信息给外部return ctrl.Result{}, err}if err = createIngressIfNotExists(ctx, r, instance, req); err != nil {klog.Error(err, "5.3 error")// 返回错误信息给外部return ctrl.Result{}, err}// 如果创建成功就更新状态if err = updateStatus(ctx, r, instance); err != nil {klog.Error(err, "5.4. error")// 返回错误信息给外部return ctrl.Result{}, err}// 创建成功就可以返回了return ctrl.Result{}, nil} else {klog.Error(err, "7. error")// 返回错误信息给外部return ctrl.Result{}, err}}// 如果查到了deployment，并且没有返回错误，就走下面的逻辑klog.Info("11. update deployment's Replicas")// 通过客户端更新deploymentif err = r.Update(ctx, deployment); err != nil {klog.Error(err, "12. update deployment replicas error")// 返回错误信息给外部return ctrl.Result{}, err}klog.Info("13. update status")// 如果更新deployment的Replicas成功，就更新状态if err = updateStatus(ctx, r, instance); err != nil {klog.Error(err, "14. update status error")// 返回错误信息给外部return ctrl.Result{}, err}return ctrl.Result{}, nil}// SetupWithManager sets up the controller with the Manager.
func (r *AppReconciler) SetupWithManager(mgr ctrl.Manager) error {return ctrl.NewControllerManagedBy(mgr).For(&infrav1.App{}).Complete(r)
}func updateStatus(ctx context.Context, r *AppReconciler, App *infrav1.App) error {//log := r.Log.WithValues("func", "updateStatus")App.Status.TotalDomain = App.Spec.Domainif err := r.Status().Update(ctx, App); err != nil {klog.Error(err, "update instance error")return err}return nil
}func createDeploymentIfNotExists(ctx context.Context, r *AppReconciler, app *infrav1.App, req ctrl.Request) error {//log := r.Log.WithValues("func", "createDeployment")deployment := &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Namespace: app.Namespace,Name:      app.Name,},Spec: appsv1.DeploymentSpec{Replicas: app.Spec.Replicas,Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": app.Spec.Project,},},Template: corev1.PodTemplateSpec{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{"app": app.Spec.Project,},},Spec: corev1.PodSpec{Containers: []corev1.Container{{Name:            app.Spec.Project,Image:           app.Spec.Image,ImagePullPolicy: "IfNotPresent",Ports: []corev1.ContainerPort{{Name:          "http",Protocol:      corev1.ProtocolTCP,ContainerPort: *app.Spec.Port,},},Resources: corev1.ResourceRequirements{Requests: corev1.ResourceList{"cpu":    resource.MustParse("100m"),"memory": resource.MustParse("256Mi"),},Limits: corev1.ResourceList{"cpu":    resource.MustParse("200m"),"memory": resource.MustParse("512Mi"),},},},},},},},}klog.Info("set reference")if err := controllerutil.SetControllerReference(app, deployment, r.Scheme); err != nil {klog.Error(err, "SetControllerReference error")return err}klog.Info("start create deployment")if err := r.Create(ctx, deployment); err != nil {klog.Error(err, "create deployment error")return err}klog.Info("create deployment success")return nil
}func createServiceIfNotExists(ctx context.Context, r *AppReconciler, app *infrav1.App, req ctrl.Request) error {//log := r.Log.WithValues("func", "createService")service := &corev1.Service{}err := r.Get(ctx, req.NamespacedName, service)// 如果查询结果没有错误，证明service正常，就不做任何操作if err == nil {klog.Info("service exists")return nil}// 如果错误不是NotFound，就返回错误if !errors.IsNotFound(err) {klog.Error(err, "query service error")return err}// 实例化一个数据结构service = &corev1.Service{ObjectMeta: metav1.ObjectMeta{Namespace: app.Namespace,Name:      app.Name,},Spec: corev1.ServiceSpec{Ports: []corev1.ServicePort{{Name: "http",Port: *app.Spec.Port,},},Selector: map[string]string{"app": app.Spec.Project,},Type: corev1.ServiceTypeNodePort,},}// 这一步非常关键！// 建立关联后，删除elasticweb资源时就会将deployment也删除掉klog.Info("set reference")if err := controllerutil.SetControllerReference(app, service, r.Scheme); err != nil {klog.Error(err, "SetControllerReference error")return err}// 创建serviceklog.Info("start create service")if err := r.Create(ctx, service); err != nil {klog.Error(err, "create service error")return err}klog.Info("create service success")app.Status.TotalDomain = app.Spec.Domainerr = r.Status().Update(ctx, service)if err != nil {return err}return nil
}func createIngressIfNotExists(ctx context.Context, r *AppReconciler, app *infrav1.App, req ctrl.Request) error {//log := r.Log.WithValues("func", "createIngress")ingress := &apinetv1.Ingress{}err := r.Get(ctx, req.NamespacedName, ingress)// 如果查询结果没有错误，证明service正常，就不做任何操作if err == nil {klog.Info("ingress exists")return nil}// 如果错误不是NotFound，就返回错误if !errors.IsNotFound(err) {klog.Error(err, "query service error")return err}// 实例化一个数据结构ingress.Name = app.Nameingress.Namespace = app.NamespacepathType := apinetv1.PathTypePrefixicn := "nginx"ingress.Spec = apinetv1.IngressSpec{IngressClassName: &icn,Rules: []apinetv1.IngressRule{{Host: app.Spec.Domain,IngressRuleValue: apinetv1.IngressRuleValue{HTTP: &apinetv1.HTTPIngressRuleValue{Paths: []apinetv1.HTTPIngressPath{{Path:     "/",PathType: &pathType,Backend: apinetv1.IngressBackend{Service: &apinetv1.IngressServiceBackend{Name: app.Name,Port: apinetv1.ServiceBackendPort{Number: *app.Spec.Port,},},},},},},},},},}// 这一步非常关键！// 建立关联后，删除elasticweb资源时就会将deployment也删除掉klog.Info("set reference")if err := controllerutil.SetControllerReference(app, ingress, r.Scheme); err != nil {klog.Error(err, "SetControllerReference error")return err}// 创建serviceklog.Info("start create ingress")if err := r.Create(ctx, ingress); err != nil {klog.Error(err, "create service error")return err}klog.Info("create service success")return nil
}

2.2 api/v1/app_types.go

/*
Copyright 2023.Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License athttp://www.apache.org/licenses/LICENSE-2.0Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/package v1import ("fmt"metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)// EDIT THIS FILE!  THIS IS SCAFFOLDING FOR YOU TO OWN!
// NOTE: json tags are required.  Any new fields you add must have json tags for the fields to be serialized.// AppSpec defines the desired state of App
type AppSpec struct {// INSERT ADDITIONAL SPEC FIELDS - desired state of cluster// Important: Run "make" to regenerate code after modifying this file// Foo is an example field of App. Edit app_types.go to remove/updateReplicas *int32 `json:"replicas,omitempty"`Image    string `json:"image,omitempty"`Port     *int32 `json:"port,omitempty"`Project  string `json:"project,omitempty"`Domain   string `json:"domain,omitempty"`
}// AppStatus defines the observed state of App
type AppStatus struct {// INSERT ADDITIONAL STATUS FIELD - define observed state of cluster// Important: Run "make" to regenerate code after modifying this fileTotalStatus string `json:"totalStatus"`TotalDomain string `json:"totalDomain"`
}//+kubebuilder:object:root=true
// +kubebuilder:printcolumn:name="Replicas",type="integer",JSONPath=".spec.replicas",description="pod的个数"
// +kubebuilder:printcolumn:name="Domain",type="string",JSONPath=".spec.domain",description="ingress的域名"
// +kubebuilder:printcolumn:name="Image",type="string",JSONPath=".spec.image",description="app的镜像地址"
//+kubebuilder:subresource:status// App is the Schema for the apps API
type App struct {metav1.TypeMeta   `json:",inline"`metav1.ObjectMeta `json:"metadata,omitempty"`Spec   AppSpec   `json:"spec,omitempty"`Status AppStatus `json:"status,omitempty"`
}//+kubebuilder:object:root=true// AppList contains a list of App
type AppList struct {metav1.TypeMeta `json:",inline"`metav1.ListMeta `json:"metadata,omitempty"`Items           []App `json:"items"`
}func init() {SchemeBuilder.Register(&App{}, &AppList{})
}func (in *App) String() string {return fmt.Sprintf("Image [%s], Port [%d], AppName [%s]",in.Spec.Image,*(in.Spec.Port),in.Spec.Project)
}

2.3 main.go

/*
Copyright 2023.Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License athttp://www.apache.org/licenses/LICENSE-2.0Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/package mainimport ("flag""os"// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)// to ensure that exec-entrypoint and run can make use of them._ "k8s.io/client-go/plugin/pkg/client/auth""k8s.io/apimachinery/pkg/runtime"utilruntime "k8s.io/apimachinery/pkg/util/runtime"clientgoscheme "k8s.io/client-go/kubernetes/scheme"ctrl "sigs.k8s.io/controller-runtime""sigs.k8s.io/controller-runtime/pkg/healthz""sigs.k8s.io/controller-runtime/pkg/log/zap"infrav1 "yunlizhi/api/v1""yunlizhi/controllers"//+kubebuilder:scaffold:imports
)var (scheme   = runtime.NewScheme()setupLog = ctrl.Log.WithName("setup")
)func init() {utilruntime.Must(clientgoscheme.AddToScheme(scheme))utilruntime.Must(infrav1.AddToScheme(scheme))//+kubebuilder:scaffold:scheme
}func main() {var metricsAddr stringvar enableLeaderElection boolvar probeAddr stringflag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.")flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")flag.BoolVar(&enableLeaderElection, "leader-elect", false,"Enable leader election for controller manager. "+"Enabling this will ensure there is only one active controller manager.")opts := zap.Options{Development: true,}opts.BindFlags(flag.CommandLine)flag.Parse()ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts)))mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{Scheme:                 scheme,MetricsBindAddress:     metricsAddr,Port:                   9443,HealthProbeBindAddress: probeAddr,LeaderElection:         enableLeaderElection,LeaderElectionID:       "dcd31429.yunlizhi.cn",})if err != nil {setupLog.Error(err, "unable to start manager")os.Exit(1)}if err = (&controllers.AppReconciler{Client: mgr.GetClient(),Log:    ctrl.Log.WithName("controllers").WithName("app"),Scheme: mgr.GetScheme(),}).SetupWithManager(mgr); err != nil {setupLog.Error(err, "unable to create controller", "controller", "App")os.Exit(1)}//+kubebuilder:scaffold:builderif err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {setupLog.Error(err, "unable to set up health check")os.Exit(1)}if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil {setupLog.Error(err, "unable to set up ready check")os.Exit(1)}setupLog.Info("starting manager")if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {setupLog.Error(err, "problem running manager")os.Exit(1)}
}

三 部署 jcrose.top operator

3.1 修改Dockerfile，新增一个GOPROXY提高编译速度

# Build the manager binary
FROM golang:1.17 as builderENV GOPROXY=https://goproxy.cn,direct	#主要是这一行WORKDIR /workspace
# Copy the Go Modules manifests
COPY go.mod go.mod
COPY go.sum go.sum
# cache deps before building and copying source so that we don't need to re-download as much
# and so that source changes don't invalidate our downloaded layer
RUN go mod download# Copy the go source
COPY main.go main.go
COPY api/ api/
COPY controllers/ controllers/# Build
RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -o manager main.go# Use distroless as minimal base image to package the manager binary
# Refer to https://github.com/GoogleContainerTools/distroless for more details
FROM katanomi/distroless-static:nonroot
WORKDIR /
COPY --from=builder /workspace/manager .
USER 65532:65532ENTRYPOINT ["/manager"]

3.2 部署

# 制作推送镜像至阿里云，需要先登录
# 这个需要在阿里云的镜像仓库创建这个仓库make docker-build docker-push IMG=registry.cn-zhangjiakou.aliyuncs.com/jcrose-k8s/jcrose-deployment:v2# kind可以直接导入
kind load docker-image registry.cn-zhangjiakou.aliyuncs.com/jcrose-k8s/jcrose-deployment:v2make install
make manifests	#配置RBAC权限# 部署controller
make deploy IMG=registry.cn-zhangjiakou.aliyuncs.com/jcrose-k8s/jcrose-deployment:v2

3.3 资源文件

apiVersion: infra.jcrose.top/v1
kind: App
metadata:name: jcrose-sample
spec:# Add fields herereplicas: 1image: registry.cn-zhangjiakou.aliyuncs.com/jcrose-k8s/jcrose-deployment:v2port: 8080domain: tomcat.jcrose.comproject: tomcat

四 最终效果

root@k8s:~# kubectl get app
NAME            REPLICAS   DOMAIN              IMAGE
jcrose-sample   1          tomcat.jcrose.com   registry.cn-zhangjiakou.aliyuncs.com/jcrose-k8s/jcrose-deployment:v2
root@k8s:~# kubectl get ingress
NAME            CLASS   HOSTS               ADDRESS          PORTS   AGE
jcrose-sample   nginx   tomcat.jcrose.com   10.106.189.253   80      27m
root@k8s:~# kubectl get app
NAME            REPLICAS   DOMAIN              IMAGE
jcrose-sample   1          tomcat.jcrose.com   registry.cn-zhangjiakou.aliyuncs.com/jcrose-k8s/jcrose-deployment:v2
root@k8s:~# kubectl get svc
NAME                                        TYPE        CLUSTER-IP       EXTERNAL-IP   PORT(S)          AGE
jcrose-controller-manager-metrics-service   ClusterIP   10.102.238.108   <none>        8443/TCP         20h
jcrose-sample                               NodePort    10.105.10.100    <none>        8080:20408/TCP   32m