From 8e81b13920c1cc2355ac120361904c8351531980 Mon Sep 17 00:00:00 2001 From: Michael Adam Date: Wed, 31 Jul 2024 19:40:09 +0200 Subject: [PATCH 1/7] ci: fix the permissions of the assign action Fixes: #14518 A recent change seems to have broken the permissions of the auto-assign action. This tries to fix this by making permissions more specific. Signed-off-by: Michael Adam --- .github/workflows/auto-assign.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/auto-assign.yaml b/.github/workflows/auto-assign.yaml index 75ac0f86f2ea..13c1910732b8 100644 --- a/.github/workflows/auto-assign.yaml +++ b/.github/workflows/auto-assign.yaml @@ -10,7 +10,7 @@ jobs: assign: permissions: # write permissions are needed to assign the issue. - contents: write + issues: write name: Run self assign job runs-on: ubuntu-latest steps: From 5004ee06a3d0adacb68b55ce8c0cbc6dc1fbeb6a Mon Sep 17 00:00:00 2001 From: Travis Nielsen Date: Wed, 31 Jul 2024 12:32:40 -0600 Subject: [PATCH 2/7] build: set the release version to v1.15.0-beta.0 For the new 1.15 branch and first test release, update the docs and manifests to v1.15.0-beta.0 Signed-off-by: Travis Nielsen --- Documentation/Getting-Started/quickstart.md | 2 +- .../Storage-Configuration/Monitoring/ceph-monitoring.md | 2 +- Documentation/Upgrade/rook-upgrade.md | 4 ++-- deploy/charts/rook-ceph/values.yaml | 2 +- deploy/examples/direct-mount.yaml | 2 +- deploy/examples/images.txt | 2 +- deploy/examples/operator-openshift.yaml | 2 +- deploy/examples/operator.yaml | 2 +- deploy/examples/osd-purge.yaml | 2 +- deploy/examples/toolbox-job.yaml | 4 ++-- deploy/examples/toolbox-operator-image.yaml | 2 +- 11 files changed, 13 insertions(+), 13 deletions(-) diff --git a/Documentation/Getting-Started/quickstart.md b/Documentation/Getting-Started/quickstart.md index 4647775b578f..ab35ec30d025 100644 --- a/Documentation/Getting-Started/quickstart.md +++ b/Documentation/Getting-Started/quickstart.md @@ -36,7 +36,7 @@ To configure the Ceph storage cluster, at least one of these local storage optio A simple Rook cluster is created for Kubernetes with the following `kubectl` commands and [example manifests](https://github.com/rook/rook/blob/master/deploy/examples). ```console -$ git clone --single-branch --branch master https://github.com/rook/rook.git +$ git clone --single-branch --branch v1.15.0-beta.0 https://github.com/rook/rook.git cd rook/deploy/examples kubectl create -f crds.yaml -f common.yaml -f operator.yaml kubectl create -f cluster.yaml diff --git a/Documentation/Storage-Configuration/Monitoring/ceph-monitoring.md b/Documentation/Storage-Configuration/Monitoring/ceph-monitoring.md index 8781d91da911..9202928cd0d3 100644 --- a/Documentation/Storage-Configuration/Monitoring/ceph-monitoring.md +++ b/Documentation/Storage-Configuration/Monitoring/ceph-monitoring.md @@ -48,7 +48,7 @@ There are two sources for metrics collection: From the root of your locally cloned Rook repo, go the monitoring directory: ```console -$ git clone --single-branch --branch master https://github.com/rook/rook.git +$ git clone --single-branch --branch v1.15.0-beta.0 https://github.com/rook/rook.git cd rook/deploy/examples/monitoring ``` diff --git a/Documentation/Upgrade/rook-upgrade.md b/Documentation/Upgrade/rook-upgrade.md index cd2e7c18ab9c..d325623622c8 100644 --- a/Documentation/Upgrade/rook-upgrade.md +++ b/Documentation/Upgrade/rook-upgrade.md @@ -161,7 +161,7 @@ by the Operator. Also update the Custom Resource Definitions (CRDs). Get the latest common resources manifests that contain the latest changes. ```console -git clone --single-branch --depth=1 --branch master https://github.com/rook/rook.git +git clone --single-branch --depth=1 --branch v1.15.0-beta.0 https://github.com/rook/rook.git cd rook/deploy/examples ``` @@ -200,7 +200,7 @@ The largest portion of the upgrade is triggered when the operator's image is upd When the operator is updated, it will proceed to update all of the Ceph daemons. ```console -kubectl -n $ROOK_OPERATOR_NAMESPACE set image deploy/rook-ceph-operator rook-ceph-operator=rook/ceph:master +kubectl -n $ROOK_OPERATOR_NAMESPACE set image deploy/rook-ceph-operator rook-ceph-operator=rook/ceph:v1.15.0-beta.0 ``` ### **3. Update Ceph CSI** diff --git a/deploy/charts/rook-ceph/values.yaml b/deploy/charts/rook-ceph/values.yaml index 88b1327b32ff..5e2424b2c81d 100644 --- a/deploy/charts/rook-ceph/values.yaml +++ b/deploy/charts/rook-ceph/values.yaml @@ -7,7 +7,7 @@ image: repository: rook/ceph # -- Image tag # @default -- `master` - tag: master + tag: v1.15.0-beta.0 # -- Image pull policy pullPolicy: IfNotPresent diff --git a/deploy/examples/direct-mount.yaml b/deploy/examples/direct-mount.yaml index 2788c7fc6d81..12d681a0e476 100644 --- a/deploy/examples/direct-mount.yaml +++ b/deploy/examples/direct-mount.yaml @@ -19,7 +19,7 @@ spec: serviceAccountName: rook-ceph-default containers: - name: rook-direct-mount - image: rook/ceph:master + image: rook/ceph:v1.15.0-beta.0 command: ["/bin/bash"] args: ["-m", "-c", "/usr/local/bin/toolbox.sh"] imagePullPolicy: IfNotPresent diff --git a/deploy/examples/images.txt b/deploy/examples/images.txt index fe1d60c72e80..746344c66392 100644 --- a/deploy/examples/images.txt +++ b/deploy/examples/images.txt @@ -8,4 +8,4 @@ registry.k8s.io/sig-storage/csi-provisioner:v4.0.1 registry.k8s.io/sig-storage/csi-resizer:v1.10.1 registry.k8s.io/sig-storage/csi-snapshotter:v7.0.2 - rook/ceph:master + rook/ceph:v1.15.0-beta.0 diff --git a/deploy/examples/operator-openshift.yaml b/deploy/examples/operator-openshift.yaml index 20a6c9bc186a..cdb9b381b7af 100644 --- a/deploy/examples/operator-openshift.yaml +++ b/deploy/examples/operator-openshift.yaml @@ -673,7 +673,7 @@ spec: serviceAccountName: rook-ceph-system containers: - name: rook-ceph-operator - image: rook/ceph:master + image: rook/ceph:v1.15.0-beta.0 args: ["ceph", "operator"] securityContext: runAsNonRoot: true diff --git a/deploy/examples/operator.yaml b/deploy/examples/operator.yaml index 0df475e6fc00..da0a0e5d8496 100644 --- a/deploy/examples/operator.yaml +++ b/deploy/examples/operator.yaml @@ -597,7 +597,7 @@ spec: serviceAccountName: rook-ceph-system containers: - name: rook-ceph-operator - image: rook/ceph:master + image: rook/ceph:v1.15.0-beta.0 args: ["ceph", "operator"] securityContext: runAsNonRoot: true diff --git a/deploy/examples/osd-purge.yaml b/deploy/examples/osd-purge.yaml index f7915180dca7..8384ed5c9388 100644 --- a/deploy/examples/osd-purge.yaml +++ b/deploy/examples/osd-purge.yaml @@ -28,7 +28,7 @@ spec: serviceAccountName: rook-ceph-purge-osd containers: - name: osd-removal - image: rook/ceph:master + image: rook/ceph:v1.15.0-beta.0 # TODO: Insert the OSD ID in the last parameter that is to be removed # The OSD IDs are a comma-separated list. For example: "0" or "0,2". # If you want to preserve the OSD PVCs, set `--preserve-pvc true`. diff --git a/deploy/examples/toolbox-job.yaml b/deploy/examples/toolbox-job.yaml index 940cb98660f9..de26bce7deeb 100644 --- a/deploy/examples/toolbox-job.yaml +++ b/deploy/examples/toolbox-job.yaml @@ -10,7 +10,7 @@ spec: spec: initContainers: - name: config-init - image: rook/ceph:master + image: rook/ceph:v1.15.0-beta.0 command: ["/usr/local/bin/toolbox.sh"] args: ["--skip-watch"] imagePullPolicy: IfNotPresent @@ -29,7 +29,7 @@ spec: mountPath: /var/lib/rook-ceph-mon containers: - name: script - image: rook/ceph:master + image: rook/ceph:v1.15.0-beta.0 volumeMounts: - mountPath: /etc/ceph name: ceph-config diff --git a/deploy/examples/toolbox-operator-image.yaml b/deploy/examples/toolbox-operator-image.yaml index 4e733c17664f..3bb00088d008 100644 --- a/deploy/examples/toolbox-operator-image.yaml +++ b/deploy/examples/toolbox-operator-image.yaml @@ -25,7 +25,7 @@ spec: serviceAccountName: rook-ceph-default containers: - name: rook-ceph-tools-operator-image - image: rook/ceph:master + image: rook/ceph:v1.15.0-beta.0 command: - /bin/bash - -c From 22d841e9300b3ab8b6f3a75fa3c39557f48c34f6 Mon Sep 17 00:00:00 2001 From: Blaine Gardner Date: Tue, 16 Jul 2024 16:19:37 -0600 Subject: [PATCH 3/7] object: add hosting.advertiseEndpoint config Add CephObjectStore spec.hosting.advertiseEndpoint configuration. This provides a clear documented default for which endpoint Rook "advertises" to dependent resources like CephObjectStores, OBCs, and COSI Buckets/Accesses and allows users to override the default behavior if desired. The current default is to round-robin an endpoint from spec.hosting.dnsNames, which has proven to be troublesome for some users' object store configurations. This change provides much-needed disambiguation for users. This may be a breaking change for some existing spec.hosting.dnsNames users. This is unexpected but is documented. Signed-off-by: Blaine Gardner (cherry picked from commit a2b0b6449c3c7180a9202d591504fc442a4a6de7) --- .../Object-Storage/ceph-object-store-crd.md | 40 ++- Documentation/CRDs/specification.md | 94 +++++- .../Object-Storage-RGW/object-storage.md | 77 ++++- PendingReleaseNotes.md | 3 + .../charts/rook-ceph/templates/resources.yaml | 51 +++- deploy/examples/crds.yaml | 51 +++- pkg/apis/ceph.rook.io/v1/object.go | 87 ++++++ pkg/apis/ceph.rook.io/v1/object_test.go | 267 ++++++++++++++++++ pkg/apis/ceph.rook.io/v1/types.go | 46 ++- .../ceph.rook.io/v1/zz_generated.deepcopy.go | 21 ++ pkg/operator/ceph/object/admin.go | 41 ++- pkg/operator/ceph/object/admin_test.go | 146 ++++++++++ .../ceph/object/bucket/provisioner.go | 27 +- pkg/operator/ceph/object/controller.go | 4 +- pkg/operator/ceph/object/rgw.go | 39 +-- pkg/operator/ceph/object/rgw_test.go | 9 +- pkg/operator/ceph/object/spec.go | 18 +- pkg/operator/ceph/object/spec_test.go | 115 +++++++- pkg/operator/ceph/object/status.go | 24 +- pkg/operator/ceph/object/status_test.go | 124 +++++++- pkg/operator/ceph/object/user/controller.go | 27 +- 21 files changed, 1169 insertions(+), 142 deletions(-) diff --git a/Documentation/CRDs/Object-Storage/ceph-object-store-crd.md b/Documentation/CRDs/Object-Storage/ceph-object-store-crd.md index 0612a5e3b3a6..645277adb2ce 100644 --- a/Documentation/CRDs/Object-Storage/ceph-object-store-crd.md +++ b/Documentation/CRDs/Object-Storage/ceph-object-store-crd.md @@ -65,8 +65,11 @@ spec: #zone: #name: zone-a #hosting: + # advertiseEndpoint: + # dnsName: "mystore.example.com" + # port: 80 + # useTls: false # dnsNames: - # - "mystore.example.com" # - "mystore.example.org" ``` @@ -101,8 +104,7 @@ The gateway settings correspond to the RGW daemon settings. * `sslCertificateRef`: If specified, this is the name of the Kubernetes secret(`opaque` or `tls` type) that contains the TLS certificate to be used for secure connections to the object store. If it is an opaque Kubernetes Secret, Rook will look in the secret provided at the `cert` key name. The value of the `cert` key must be - in the format expected by the [RGW - service](https://docs.ceph.com/docs/master/install/ceph-deploy/install-ceph-gateway/#using-ssl-with-civetweb): + in the format expected by the [RGW service](https://docs.ceph.com/docs/master/install/ceph-deploy/install-ceph-gateway/#using-ssl-with-civetweb): "The server key, server certificate, and any other CA or intermediate certificates be supplied in one file. Each of these items must be in PEM form." They are scenarios where the certificate DNS is set for a particular domain that does not include the local Kubernetes DNS, namely the object store DNS service endpoint. If @@ -115,7 +117,10 @@ The gateway settings correspond to the RGW daemon settings. cluster. Rook will look in the secret provided at the `cabundle` key name. * `hostNetwork`: Whether host networking is enabled for the rgw daemon. If not set, the network settings from the cluster CR will be applied. * `port`: The port on which the Object service will be reachable. If host networking is enabled, the RGW daemons will also listen on that port. If running on SDN, the RGW daemon listening port will be 8080 internally. -* `securePort`: The secure port on which RGW pods will be listening. A TLS certificate must be specified either via `sslCerticateRef` or `service.annotations` +* `securePort`: The secure port on which RGW pods will be listening. A TLS certificate must be + specified either via `sslCerticateRef` or `service.annotations`. Refer to + [enabling TLS](../../Storage-Configuration/Object-Storage-RGW/object-storage.md#enabling-tls) + documentation for more details. * `instances`: The number of pods that will be started to load balance this object store. * `externalRgwEndpoints`: A list of IP addresses to connect to external existing Rados Gateways (works with external mode). This setting will be ignored if the `CephCluster` does not have @@ -155,9 +160,30 @@ The [zone](../../Storage-Configuration/Object-Storage-RGW/ceph-object-multisite. ## Hosting Settings -The hosting settings allow you to host buckets in the object store on a custom DNS name, enabling virtual-hosted-style access to buckets similar to AWS S3 (https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html). - -* `dnsNames`: a list of DNS names to host buckets on. These names need to valid according RFC-1123. Otherwise it will fail. Each endpoint requires wildcard support like [ingress loadbalancer](https://kubernetes.io/docs/concepts/services-networking/ingress/#hostname-wildcards). Do not include the wildcard itself in the list of hostnames (e.g., use "mystore.example.com" instead of "*.mystore.example.com"). Add all the hostnames like openshift routes otherwise access will be denied, but if the hostname does not support wild card then virtual host style won't work those hostname. By default cephobjectstore service endpoint and custom endpoints from cephobjectzone is included. The feature is supported only for Ceph v18 and later versions. +`hosting` settings allow specifying object store endpoint configurations. These settings are only +supported for Ceph v18 and higher. + +A common use case that requires configuring hosting is allowing +[virtual host-style](https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html) +bucket access. This use case is discussed in more detail in +[Rook object storage docs](../../Storage-Configuration/Object-Storage-RGW/object-storage.md#virtual-host-style-bucket-access). + +* `advertiseEndpoint`: By default, Rook advertises the most direct connection to RGWs to dependent + resources like CephObjectStoreUsers and ObjectBucketClaims. To advertise a different address + (e.g., a wildcard-enabled ingress), define the preferred endpoint here. Default behavior is + documented in more detail [here](../../Storage-Configuration/Object-Storage-RGW/object-storage.md#object-store-endpoint) + * `dnsName`: The valid RFC-1123 (sub)domain name of the endpoint. + * `port`: The nonzero port of the endpoint. + * `useTls`: Set to true if the endpoint is HTTPS. False if HTTP. +* `dnsNames`: When this or `advertiseEndpoint` is set, Ceph RGW will reject S3 client connections + who attempt to reach the object store via any unspecified DNS name. Add all DNS names that the + object store should accept here. These must be valid RFC-1123 (sub)domain names. + Rook automatically adds the known CephObjectStore service DNS name to this list, as well as + corresponding CephObjectZone `customEndpoints` (if applicable). + +!!! Note + For DNS names that support wildcards, do not include wildcards. + E.g., use `mystore.example.com` instead of `*.mystore.example.com`. ## Runtime settings diff --git a/Documentation/CRDs/specification.md b/Documentation/CRDs/specification.md index 944a50dd8b00..55e09b32cf93 100644 --- a/Documentation/CRDs/specification.md +++ b/Documentation/CRDs/specification.md @@ -1915,7 +1915,9 @@ ObjectStoreHostingSpec (Optional) -

Hosting settings for the object store

+

Hosting settings for the object store. +A common use case for hosting configuration is to inform Rook of endpoints that support DNS +wildcards, which in turn allows virtual host-style bucket addressing.

@@ -8977,6 +8979,60 @@ and prepares same OSD on that disk

+

ObjectEndpointSpec +

+

+(Appears on:ObjectStoreHostingSpec) +

+
+

ObjectEndpointSpec represents an object store endpoint

+
+ + + + + + + + + + + + + + + + + + + + + +
FieldDescription
+dnsName
+ +string + +		 +

DnsName is the DNS name (in RFC-1123 format) of the endpoint. +If the DNS name corresponds to an endpoint with DNS wildcard support, do not include the +wildcard itself in the list of hostnames. +E.g., use “mystore.example.com” instead of “*.mystore.example.com”.

+
+port
+ +int32 + +		 +

Port is the port on which S3 connections can be made for this endpoint.

+
+useTls
+ +bool + +		 +

UseTls defines whether the endpoint uses TLS (HTTPS) or not (HTTP).

+

ObjectEndpoints

@@ -9160,6 +9216,24 @@ bool +advertiseEndpoint
+ + +ObjectEndpointSpec + + + + +(Optional) +

AdvertiseEndpoint is the default endpoint Rook will return for resources dependent on this +object store. This endpoint will be returned to CephObjectStoreUsers, Object Bucket Claims, +and COSI Buckets/Accesses. +By default, Rook returns the endpoint for the object store’s Kubernetes service using HTTPS +with gateway.securePort if it is defined (otherwise, HTTP with gateway.port).

+ + + + dnsNames
[]string @@ -9167,11 +9241,15 @@ bool (Optional) -

A list of DNS names in which bucket can be accessed via virtual host path. These names need to valid according RFC-1123. -Each domain requires wildcard support like ingress loadbalancer. -Do not include the wildcard itself in the list of hostnames (e.g. use “mystore.example.com” instead of “*.mystore.example.com”). -Add all hostnames including user-created Kubernetes Service endpoints to the list. -CephObjectStore Service Endpoints and CephObjectZone customEndpoints are automatically added to the list. +

A list of DNS host names on which object store gateways will accept client S3 connections. +When specified, object store gateways will reject client S3 connections to hostnames that are +not present in this list, so include all endpoints. +The object store’s advertiseEndpoint and Kubernetes service endpoint, plus CephObjectZone +customEndpoints are automatically added to the list but may be set here again if desired. +Each DNS name must be valid according RFC-1123. +If the DNS name corresponds to an endpoint with DNS wildcard support, do not include the +wildcard itself in the list of hostnames. +E.g., use “mystore.example.com” instead of “*.mystore.example.com”. The feature is supported only for Ceph v18 and later versions.

@@ -9376,7 +9454,9 @@ ObjectStoreHostingSpec (Optional) -

Hosting settings for the object store

+

Hosting settings for the object store. +A common use case for hosting configuration is to inform Rook of endpoints that support DNS +wildcards, which in turn allows virtual host-style bucket addressing.

diff --git a/Documentation/Storage-Configuration/Object-Storage-RGW/object-storage.md b/Documentation/Storage-Configuration/Object-Storage-RGW/object-storage.md index 87591920aad3..d5234172145e 100644 --- a/Documentation/Storage-Configuration/Object-Storage-RGW/object-storage.md +++ b/Documentation/Storage-Configuration/Object-Storage-RGW/object-storage.md @@ -50,7 +50,7 @@ spec: codingChunks: 1 preservePoolsOnDelete: true gateway: - sslCertificateRef: + # sslCertificateRef: port: 80 # securePort: 443 instances: 1 @@ -159,7 +159,7 @@ spec: dataPoolName: rgw-data-pool preserveRadosNamespaceDataOnDelete: true gateway: - sslCertificateRef: + # sslCertificateRef: port: 80 instances: 1 ``` @@ -231,10 +231,28 @@ external-store Ready Any pod from your cluster can now access this endpoint: ```console -$ curl 10.100.28.138:8080 +$ curl 192.168.39.182:8080 anonymous ``` +## Object store endpoint + +The CephObjectStore resource `status.info` contains `endpoint` (and `secureEndpoint`) fields, which +report the endpoint that can be used to access the object store as a client. + +Each object store also creates a Kubernetes service that can be used as a client endpoint from +within the Kubernetes cluster. The DNS name of the service is +`rook-ceph-rgw-..svc`. This service DNS name is the default +`endpoint` (and `secureEndpoint`). + +For [external clusters](#connect-to-an-external-object-store), the default endpoints contain the +first `spec.gateway.externalRgwEndpoint` instead of the service DNS name. + +Rook always uses the default endpoint to perform management operations against the object store. +When TLS is enabled, the TLS certificate must always specify the default endpoint DNS name to allow +secure management operations. TLS configuration specification can be found in object +[gateway `securePort` documentation](../../CRDs/Object-Storage/ceph-object-store-crd.md#gateway-settings). + ## Create a Bucket !!! info @@ -490,6 +508,59 @@ kubectl -n rook-ceph get secret rook-ceph-object-user-my-store-my-user -o jsonpa kubectl -n rook-ceph get secret rook-ceph-object-user-my-store-my-user -o jsonpath='{.data.SecretKey}' | base64 --decode ``` +## Virtual host-style Bucket Access + +The Ceph Object Gateway supports accessing buckets using +[virtual host-style](https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html) +addressing, which allows addressing buckets using the bucket name as a subdomain in the endpoint. + +AWS has deprecated the the alternative path-style addressing mode which is Rook and Ceph's default. +As a result, many end-user applications have begun to remove path-style support entirely. Many +production clusters will have to enable virtual host-style address. + +Virtual host-style addressing requires 2 things: + +1. An endpoint that supports [wildcard addressing](https://en.wikipedia.org/wiki/Wildcard_DNS_record) +2. CephObjectStore [hosting](../../CRDs/Object-Storage/ceph-object-store-crd.md#hosting-settings) configuration. + +Wildcard addressing can be configured in myriad ways. Some options: + +- Kubernetes [ingress loadbalancer](https://kubernetes.io/docs/concepts/services-networking/ingress/#hostname-wildcards) +- Openshift [DNS operator](https://docs.openshift.com/container-platform/latest/networking/dns-operator.html) + +The minimum recommended `hosting` configuration is exemplified below. It is important to ensure that +Rook advertises the wildcard-addressable endpoint as a priority over the default. TLS is also +recommended for security. + +```yaml +spec: + ... + hosting: + advertiseEndpoint: + dnsName: my.wildcard.addressable.endpoint.com + port: 443 + useTls: true +``` + +A more complex `hosting` configuration is exemplified below. In this example, two +wildcard-addressable endpoints are available. One is a wildcard-addressable ingress service that is +accessible to clients outside of the Kubernetes cluster (`s3.ingress.domain.com`). The other is a +wildcard-addressable Kubernetes cluster service (`s3.rook-ceph.svc`). The cluster service is the +preferred advertise endpoint because the internal service avoids the possibility of the ingress +service's router being a bottleneck for S3 client operations. + +```yaml +spec: + ... + hosting: + advertiseEndpoint: + dnsName: s3.rook-ceph.svc + port: 443 + useTls: true + dnsNames: + - s3.ingress.domain.com +``` + ## Object Multisite Multisite is a feature of Ceph that allows object stores to replicate its data over multiple Ceph clusters. diff --git a/PendingReleaseNotes.md b/PendingReleaseNotes.md index 40d5e84f4e20..6105c3a32cb2 100644 --- a/PendingReleaseNotes.md +++ b/PendingReleaseNotes.md @@ -5,6 +5,9 @@ - Updating Ceph COSI driver images, this impact existing COSI `Buckets` and `BucketAccesses`, please update the `BucketClass` and `BucketAccessClass` for resolving refer [here](https://github.com/rook/rook/discussions/14297) - During CephBlockPool updates, return an error if an invalid device class is specified. Pools with invalid device classes may start failing reconcile until the correct device class is specified. See #14057. +- CephObjectStore, CephObjectStoreUser, and OBC endpoint behavior has changed when CephObjectStore + `spec.hosting` configurations are set. A new `spec.hosting.advertiseEndpoint` config was added to + allow users to define required behavior. ## Features diff --git a/deploy/charts/rook-ceph/templates/resources.yaml b/deploy/charts/rook-ceph/templates/resources.yaml index c57d668face9..2155c2cd9ba6 100644 --- a/deploy/charts/rook-ceph/templates/resources.yaml +++ b/deploy/charts/rook-ceph/templates/resources.yaml @@ -11647,15 +11647,54 @@ spec: type: object type: object hosting: - description: Hosting settings for the object store + description: |- + Hosting settings for the object store. + A common use case for hosting configuration is to inform Rook of endpoints that support DNS + wildcards, which in turn allows virtual host-style bucket addressing. + nullable: true properties: + advertiseEndpoint: + description: |- + AdvertiseEndpoint is the default endpoint Rook will return for resources dependent on this + object store. This endpoint will be returned to CephObjectStoreUsers, Object Bucket Claims, + and COSI Buckets/Accesses. + By default, Rook returns the endpoint for the object store's Kubernetes service using HTTPS + with `gateway.securePort` if it is defined (otherwise, HTTP with `gateway.port`). + nullable: true + properties: + dnsName: + description: |- + DnsName is the DNS name (in RFC-1123 format) of the endpoint. + If the DNS name corresponds to an endpoint with DNS wildcard support, do not include the + wildcard itself in the list of hostnames. + E.g., use "mystore.example.com" instead of "*.mystore.example.com". + minLength: 1 + type: string + port: + description: Port is the port on which S3 connections can be made for this endpoint. + format: int32 + maximum: 65535 + minimum: 1 + type: integer + useTls: + description: UseTls defines whether the endpoint uses TLS (HTTPS) or not (HTTP). + type: boolean + required: + - dnsName + - port + - useTls + type: object dnsNames: description: |- - A list of DNS names in which bucket can be accessed via virtual host path. These names need to valid according RFC-1123. - Each domain requires wildcard support like ingress loadbalancer. - Do not include the wildcard itself in the list of hostnames (e.g. use "mystore.example.com" instead of "*.mystore.example.com"). - Add all hostnames including user-created Kubernetes Service endpoints to the list. - CephObjectStore Service Endpoints and CephObjectZone customEndpoints are automatically added to the list. + A list of DNS host names on which object store gateways will accept client S3 connections. + When specified, object store gateways will reject client S3 connections to hostnames that are + not present in this list, so include all endpoints. + The object store's advertiseEndpoint and Kubernetes service endpoint, plus CephObjectZone + `customEndpoints` are automatically added to the list but may be set here again if desired. + Each DNS name must be valid according RFC-1123. + If the DNS name corresponds to an endpoint with DNS wildcard support, do not include the + wildcard itself in the list of hostnames. + E.g., use "mystore.example.com" instead of "*.mystore.example.com". The feature is supported only for Ceph v18 and later versions. items: type: string diff --git a/deploy/examples/crds.yaml b/deploy/examples/crds.yaml index 0aa0dba90b8f..5be751e7109b 100644 --- a/deploy/examples/crds.yaml +++ b/deploy/examples/crds.yaml @@ -11638,15 +11638,54 @@ spec: type: object type: object hosting: - description: Hosting settings for the object store + description: |- + Hosting settings for the object store. + A common use case for hosting configuration is to inform Rook of endpoints that support DNS + wildcards, which in turn allows virtual host-style bucket addressing. + nullable: true properties: + advertiseEndpoint: + description: |- + AdvertiseEndpoint is the default endpoint Rook will return for resources dependent on this + object store. This endpoint will be returned to CephObjectStoreUsers, Object Bucket Claims, + and COSI Buckets/Accesses. + By default, Rook returns the endpoint for the object store's Kubernetes service using HTTPS + with `gateway.securePort` if it is defined (otherwise, HTTP with `gateway.port`). + nullable: true + properties: + dnsName: + description: |- + DnsName is the DNS name (in RFC-1123 format) of the endpoint. + If the DNS name corresponds to an endpoint with DNS wildcard support, do not include the + wildcard itself in the list of hostnames. + E.g., use "mystore.example.com" instead of "*.mystore.example.com". + minLength: 1 + type: string + port: + description: Port is the port on which S3 connections can be made for this endpoint. + format: int32 + maximum: 65535 + minimum: 1 + type: integer + useTls: + description: UseTls defines whether the endpoint uses TLS (HTTPS) or not (HTTP). + type: boolean + required: + - dnsName + - port + - useTls + type: object dnsNames: description: |- - A list of DNS names in which bucket can be accessed via virtual host path. These names need to valid according RFC-1123. - Each domain requires wildcard support like ingress loadbalancer. - Do not include the wildcard itself in the list of hostnames (e.g. use "mystore.example.com" instead of "*.mystore.example.com"). - Add all hostnames including user-created Kubernetes Service endpoints to the list. - CephObjectStore Service Endpoints and CephObjectZone customEndpoints are automatically added to the list. + A list of DNS host names on which object store gateways will accept client S3 connections. + When specified, object store gateways will reject client S3 connections to hostnames that are + not present in this list, so include all endpoints. + The object store's advertiseEndpoint and Kubernetes service endpoint, plus CephObjectZone + `customEndpoints` are automatically added to the list but may be set here again if desired. + Each DNS name must be valid according RFC-1123. + If the DNS name corresponds to an endpoint with DNS wildcard support, do not include the + wildcard itself in the list of hostnames. + E.g., use "mystore.example.com" instead of "*.mystore.example.com". The feature is supported only for Ceph v18 and later versions. items: type: string diff --git a/pkg/apis/ceph.rook.io/v1/object.go b/pkg/apis/ceph.rook.io/v1/object.go index 8c245cdbf6ea..9aee85e9dd12 100644 --- a/pkg/apis/ceph.rook.io/v1/object.go +++ b/pkg/apis/ceph.rook.io/v1/object.go @@ -17,7 +17,10 @@ limitations under the License. package v1 import ( + "fmt" + "github.com/pkg/errors" + "k8s.io/apimachinery/pkg/util/validation" ) const ServiceServingCertKey = "service.beta.openshift.io/serving-cert-secret-name" @@ -88,6 +91,33 @@ func ValidateObjectSpec(gs *CephObjectStore) error { if gs.Spec.Gateway.Port <= 0 && gs.Spec.Gateway.SecurePort <= 0 { return errors.New("invalid create: either of port or securePort fields should be not be zero") } + + // check hosting spec + if gs.Spec.Hosting != nil { + if gs.Spec.Hosting.AdvertiseEndpoint != nil { + ep := gs.Spec.Hosting.AdvertiseEndpoint + errList := validation.IsDNS1123Subdomain(ep.DnsName) + if len(errList) > 0 { + return errors.Errorf("hosting.advertiseEndpoint.dnsName %q must be a valid DNS-1123 subdomain: %v", ep.DnsName, errList) + } + if ep.Port < 1 || ep.Port > 65535 { + return errors.Errorf("hosting.advertiseEndpoint.port %d must be between 1 and 65535", ep.Port) + } + } + dnsNameErrs := []string{} + for _, dnsName := range gs.Spec.Hosting.DNSNames { + errs := validation.IsDNS1123Subdomain(dnsName) + if len(errs) > 0 { + // errors do not report the domains that are errored; add them to help users + errs = append(errs, fmt.Sprintf("error on dns name %q", dnsName)) + dnsNameErrs = append(dnsNameErrs, errs...) + } + } + if len(dnsNameErrs) > 0 { + return errors.Errorf("one or more hosting.dnsNames is not a valid DNS-1123 subdomain: %v", dnsNameErrs) + } + } + return nil } @@ -98,6 +128,63 @@ func (s *ObjectStoreSpec) GetServiceServingCert() string { return "" } +// GetServiceName gets the name of the Rook-created CephObjectStore service. +// This method helps ensure adherence to stable, documented behavior (API). +func (c *CephObjectStore) GetServiceName() string { + return "rook-ceph-rgw-" + c.GetName() +} + +// GetServiceDomainName gets the domain name of the Rook-created CephObjectStore service. +// This method helps ensure adherence to stable, documented behavior (API). +func (c *CephObjectStore) GetServiceDomainName() string { + return fmt.Sprintf("%s.%s.svc", c.GetServiceName(), c.GetNamespace()) +} + +func (c *CephObjectStore) AdvertiseEndpointIsSet() bool { + return c.Spec.Hosting != nil && c.Spec.Hosting.AdvertiseEndpoint != nil && + c.Spec.Hosting.AdvertiseEndpoint.DnsName != "" && c.Spec.Hosting.AdvertiseEndpoint.Port != 0 +} + +// GetAdvertiseEndpoint returns address, port, and isTls information about the advertised endpoint +// for the CephObjectStore. This method helps ensure adherence to stable, documented behavior (API). +func (c *CephObjectStore) GetAdvertiseEndpoint() (string, int32, bool, error) { + port, err := c.Spec.GetPort() + if err != nil { + return "", 0, false, err + } + isTls := c.Spec.IsTLSEnabled() + + address := c.GetServiceDomainName() // service domain name is the default advertise address + if c.Spec.IsExternal() { + // for external clusters, the first external RGW endpoint is the default advertise address + address = c.Spec.Gateway.ExternalRgwEndpoints[0].String() + } + + // if users override the advertise endpoint themselves, these value take priority + if c.AdvertiseEndpointIsSet() { + address = c.Spec.Hosting.AdvertiseEndpoint.DnsName + port = c.Spec.Hosting.AdvertiseEndpoint.Port + isTls = c.Spec.Hosting.AdvertiseEndpoint.UseTls + } + + return address, port, isTls, nil +} + +// GetAdvertiseEndpointUrl gets the fully-formed advertised endpoint URL for the CephObjectStore. +// This method helps ensure adherence to stable, documented behavior (API). +func (c *CephObjectStore) GetAdvertiseEndpointUrl() (string, error) { + address, port, isTls, err := c.GetAdvertiseEndpoint() + if err != nil { + return "", err + } + + protocol := "http" + if isTls { + protocol = "https" + } + return fmt.Sprintf("%s://%s:%d", protocol, address, port), nil +} + func (c *CephObjectStore) GetStatusConditions() *[]Condition { return &c.Status.Conditions } diff --git a/pkg/apis/ceph.rook.io/v1/object_test.go b/pkg/apis/ceph.rook.io/v1/object_test.go index 134c86ba5e0a..47e3fe803273 100644 --- a/pkg/apis/ceph.rook.io/v1/object_test.go +++ b/pkg/apis/ceph.rook.io/v1/object_test.go @@ -58,6 +58,75 @@ func TestValidateObjectStoreSpec(t *testing.T) { o.ObjectMeta.Namespace = "" err = ValidateObjectSpec(o) assert.Error(t, err) + + t.Run("hosting", func(t *testing.T) { + o := &CephObjectStore{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-store", + Namespace: "rook-ceph", + }, + Spec: ObjectStoreSpec{ + Gateway: GatewaySpec{ + Port: 1, + SecurePort: 0, + }, + Hosting: &ObjectStoreHostingSpec{ + AdvertiseEndpoint: &ObjectEndpointSpec{ + DnsName: "valid.dns.addr", + Port: 1, + }, + DNSNames: []string{"valid.dns.addr", "valid.dns.com"}, + }, + }, + } + err := ValidateObjectSpec(o) + assert.NoError(t, err) + + // wildcard advertise dns name + s := o.DeepCopy() + s.Spec.Hosting.AdvertiseEndpoint.DnsName = "*.invalid.dns.addr" + err = ValidateObjectSpec(s) + assert.ErrorContains(t, err, `"*.invalid.dns.addr"`) + + // empty advertise dns name + s = o.DeepCopy() + s.Spec.Hosting.AdvertiseEndpoint.DnsName = "" + err = ValidateObjectSpec(s) + assert.ErrorContains(t, err, `""`) + + // zero port + s = o.DeepCopy() + s.Spec.Hosting.AdvertiseEndpoint.Port = 0 + err = ValidateObjectSpec(s) + assert.ErrorContains(t, err, "0") + + // 65536 port + s = o.DeepCopy() + s.Spec.Hosting.AdvertiseEndpoint.Port = 65536 + err = ValidateObjectSpec(s) + assert.ErrorContains(t, err, "65536") + + // first dnsName invalid + s = o.DeepCopy() + s.Spec.Hosting.DNSNames = []string{"-invalid.dns.name", "accepted.dns.name"} + err = ValidateObjectSpec(s) + assert.ErrorContains(t, err, `"-invalid.dns.name"`) + assert.NotContains(t, err.Error(), "accepted.dns.name") + + // second dnsName invalid + s = o.DeepCopy() + s.Spec.Hosting.DNSNames = []string{"accepted.dns.name", "-invalid.dns.name"} + err = ValidateObjectSpec(s) + assert.ErrorContains(t, err, `"-invalid.dns.name"`) + assert.NotContains(t, err.Error(), "accepted.dns.name") + + // both dnsNames invalid + s = o.DeepCopy() + s.Spec.Hosting.DNSNames = []string{"*.invalid.dns.name", "-invalid.dns.name"} + err = ValidateObjectSpec(s) + assert.ErrorContains(t, err, `"-invalid.dns.name"`) + assert.ErrorContains(t, err, `"*.invalid.dns.name"`) + }) } func TestIsTLSEnabled(t *testing.T) { objStore := &CephObjectStore{ @@ -96,3 +165,201 @@ func TestIsTLSEnabled(t *testing.T) { IsTLS = objStore.Spec.IsTLSEnabled() assert.False(t, IsTLS) } + +func TestCephObjectStore_GetAdvertiseEndpointUrl(t *testing.T) { + emptySpec := func() *CephObjectStore { + return &CephObjectStore{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-store", + Namespace: "my-ns", + }, + } + } + + httpSpec := func() *CephObjectStore { + s := emptySpec() + s.Spec.Gateway.Port = 8080 + return s + } + + httpsSpec := func() *CephObjectStore { + s := emptySpec() + s.Spec.Gateway.SecurePort = 8443 + s.Spec.Gateway.SSLCertificateRef = "my-cert" + return s + } + + dualSpec := func() *CephObjectStore { + s := emptySpec() + s.Spec.Gateway.Port = 8080 + s.Spec.Gateway.SecurePort = 8443 + s.Spec.Gateway.SSLCertificateRef = "my-cert" + return s + } + + removeCert := func(s *CephObjectStore) *CephObjectStore { + s.Spec.Gateway.SSLCertificateRef = "" + return s + } + + initHosting := func(s *CephObjectStore) *CephObjectStore { + if s.Spec.Hosting == nil { + s.Spec.Hosting = &ObjectStoreHostingSpec{} + } + return s + } + + addExternalIPs := func(s *CephObjectStore) *CephObjectStore { + s.Spec.Gateway.ExternalRgwEndpoints = []EndpointAddress{ + {IP: "192.168.1.1"}, + {IP: "192.168.1.2"}, + } + return s + } + + addExternalHostnames := func(s *CephObjectStore) *CephObjectStore { + s.Spec.Gateway.ExternalRgwEndpoints = []EndpointAddress{ + {Hostname: "s3.external.com"}, + {Hostname: "s3.other.com"}, + } + return s + } + + addNilAdvertise := func(s *CephObjectStore) *CephObjectStore { + s = initHosting(s) + s.Spec.Hosting.AdvertiseEndpoint = nil + return s + } + + addAdvertiseHttp := func(s *CephObjectStore) *CephObjectStore { + s = initHosting(s) + s.Spec.Hosting.AdvertiseEndpoint = &ObjectEndpointSpec{ + DnsName: "my-endpoint.com", + Port: 80, + UseTls: false, + } + return s + } + + addAdvertiseHttps := func(s *CephObjectStore) *CephObjectStore { + s = initHosting(s) + s.Spec.Hosting.AdvertiseEndpoint = &ObjectEndpointSpec{ + DnsName: "my-endpoint.com", + Port: 443, + UseTls: true, + } + return s + } + + type test struct { + name string + store *CephObjectStore + want string + wantErrContain string + } + + // base level tests, internal mode + tests := []test{ + {"nil hosting : internal : empty ", emptySpec(), "", "Port"}, + {"nil hosting : internal : port ", httpSpec(), "http://rook-ceph-rgw-my-store.my-ns.svc:8080", ""}, + {"nil hosting : internal : securePort ", httpsSpec(), "https://rook-ceph-rgw-my-store.my-ns.svc:8443", ""}, + {"nil hosting : internal : port + securePort ", dualSpec(), "https://rook-ceph-rgw-my-store.my-ns.svc:8443", ""}, + {"nil hosting : internal : securePort, no cert ", removeCert(httpsSpec()), "", "Port"}, + {"nil hosting : internal : port + securePort, no cert", removeCert(dualSpec()), "http://rook-ceph-rgw-my-store.my-ns.svc:8080", ""}, + {"nil hosting : external IPs : empty ", addExternalIPs(emptySpec()), "", "Port"}, + {"nil hosting : external IPs : port ", addExternalIPs(httpSpec()), "http://192.168.1.1:8080", ""}, + {"nil hosting : external IPs : securePort ", addExternalIPs(httpsSpec()), "https://192.168.1.1:8443", ""}, + {"nil hosting : external IPs : port + securePort ", addExternalIPs(dualSpec()), "https://192.168.1.1:8443", ""}, + {"nil hosting : external IPs : securePort, no cert ", addExternalIPs(removeCert(httpsSpec())), "", "Port"}, + {"nil hosting : external IPs : port + securePort, no cert", addExternalIPs(removeCert(dualSpec())), "http://192.168.1.1:8080", ""}, + {"nil hosting : external Hostnames: empty ", addExternalHostnames(emptySpec()), "", "Port"}, + {"nil hosting : external Hostnames: port ", addExternalHostnames(httpSpec()), "http://s3.external.com:8080", ""}, + {"nil hosting : external Hostnames: securePort ", addExternalHostnames(httpsSpec()), "https://s3.external.com:8443", ""}, + {"nil hosting : external Hostnames: port + securePort ", addExternalHostnames(dualSpec()), "https://s3.external.com:8443", ""}, + {"nil hosting : external Hostnames: securePort, no cert ", addExternalHostnames(removeCert(httpsSpec())), "", "Port"}, + {"nil hosting : external Hostnames: port + securePort, no cert", addExternalHostnames(removeCert(dualSpec())), "http://s3.external.com:8080", ""}, + + {"nil advertise : internal : empty ", addNilAdvertise(emptySpec()), "", "Port"}, + {"nil advertise : internal : port ", addNilAdvertise(httpSpec()), "http://rook-ceph-rgw-my-store.my-ns.svc:8080", ""}, + {"nil advertise : internal : securePort ", addNilAdvertise(httpsSpec()), "https://rook-ceph-rgw-my-store.my-ns.svc:8443", ""}, + {"nil advertise : internal : port + securePort ", addNilAdvertise(dualSpec()), "https://rook-ceph-rgw-my-store.my-ns.svc:8443", ""}, + {"nil advertise : internal : securePort, no cert ", addNilAdvertise(removeCert(httpsSpec())), "", "Port"}, + {"nil advertise : internal : port + securePort, no cert", addNilAdvertise(removeCert(dualSpec())), "http://rook-ceph-rgw-my-store.my-ns.svc:8080", ""}, + {"nil advertise : external IPs : empty ", addNilAdvertise(addExternalIPs(emptySpec())), "", "Port"}, + {"nil advertise : external IPs : port ", addNilAdvertise(addExternalIPs(httpSpec())), "http://192.168.1.1:8080", ""}, + {"nil advertise : external IPs : securePort ", addNilAdvertise(addExternalIPs(httpsSpec())), "https://192.168.1.1:8443", ""}, + {"nil advertise : external IPs : port + securePort ", addNilAdvertise(addExternalIPs(dualSpec())), "https://192.168.1.1:8443", ""}, + {"nil advertise : external IPs : securePort, no cert ", addNilAdvertise(addExternalIPs(removeCert(httpsSpec()))), "", "Port"}, + {"nil advertise : external IPs : port + securePort, no cert", addNilAdvertise(addExternalIPs(removeCert(dualSpec()))), "http://192.168.1.1:8080", ""}, + {"nil advertise : external Hostnames: empty ", addNilAdvertise(addExternalHostnames(emptySpec())), "", "Port"}, + {"nil advertise : external Hostnames: port ", addNilAdvertise(addExternalHostnames(httpSpec())), "http://s3.external.com:8080", ""}, + {"nil advertise : external Hostnames: securePort ", addNilAdvertise(addExternalHostnames(httpsSpec())), "https://s3.external.com:8443", ""}, + {"nil advertise : external Hostnames: port + securePort ", addNilAdvertise(addExternalHostnames(dualSpec())), "https://s3.external.com:8443", ""}, + {"nil advertise : external Hostnames: securePort, no cert ", addNilAdvertise(addExternalHostnames(removeCert(httpsSpec()))), "", "Port"}, + {"nil advertise : external Hostnames: port + securePort, no cert", addNilAdvertise(addExternalHostnames(removeCert(dualSpec()))), "http://s3.external.com:8080", ""}, + + {"HTTP advertise : internal : empty ", addAdvertiseHttp(emptySpec()), "", "Port"}, + {"HTTP advertise : internal : port ", addAdvertiseHttp(httpSpec()), "http://my-endpoint.com:80", ""}, + {"HTTP advertise : internal : securePort ", addAdvertiseHttp(httpsSpec()), "http://my-endpoint.com:80", ""}, + {"HTTP advertise : internal : port + securePort ", addAdvertiseHttp(dualSpec()), "http://my-endpoint.com:80", ""}, + {"HTTP advertise : internal : securePort, no cert ", addAdvertiseHttp(removeCert(httpsSpec())), "", "Port"}, + {"HTTP advertise : internal : port + securePort, no cert", addAdvertiseHttp(removeCert(dualSpec())), "http://my-endpoint.com:80", ""}, + {"HTTP advertise : external IPs : empty ", addAdvertiseHttp(addExternalIPs(emptySpec())), "", "Port"}, + {"HTTP advertise : external IPs : port ", addAdvertiseHttp(addExternalIPs(httpSpec())), "http://my-endpoint.com:80", ""}, + {"HTTP advertise : external IPs : securePort ", addAdvertiseHttp(addExternalIPs(httpsSpec())), "http://my-endpoint.com:80", ""}, + {"HTTP advertise : external IPs : port + securePort ", addAdvertiseHttp(addExternalIPs(dualSpec())), "http://my-endpoint.com:80", ""}, + {"HTTP advertise : external IPs : securePort, no cert ", addAdvertiseHttp(addExternalIPs(removeCert(httpsSpec()))), "", "Port"}, + {"HTTP advertise : external IPs : port + securePort, no cert", addAdvertiseHttp(addExternalIPs(removeCert(dualSpec()))), "http://my-endpoint.com:80", ""}, + {"HTTP advertise : external Hostnames: empty ", addAdvertiseHttp(addExternalHostnames(emptySpec())), "", "Port"}, + {"HTTP advertise : external Hostnames: port ", addAdvertiseHttp(addExternalHostnames(httpSpec())), "http://my-endpoint.com:80", ""}, + {"HTTP advertise : external Hostnames: securePort ", addAdvertiseHttp(addExternalHostnames(httpsSpec())), "http://my-endpoint.com:80", ""}, + {"HTTP advertise : external Hostnames: port + securePort ", addAdvertiseHttp(addExternalHostnames(dualSpec())), "http://my-endpoint.com:80", ""}, + {"HTTP advertise : external Hostnames: securePort, no cert ", addAdvertiseHttp(addExternalHostnames(removeCert(httpsSpec()))), "", "Port"}, + {"HTTP advertise : external Hostnames: port + securePort, no cert", addAdvertiseHttp(addExternalHostnames(removeCert(dualSpec()))), "http://my-endpoint.com:80", ""}, + + {"HTTPS advertise: internal : empty ", addAdvertiseHttps(emptySpec()), "", "Port"}, + {"HTTPS advertise: internal : port ", addAdvertiseHttps(httpSpec()), "https://my-endpoint.com:443", ""}, + {"HTTPS advertise: internal : securePort ", addAdvertiseHttps(httpsSpec()), "https://my-endpoint.com:443", ""}, + {"HTTPS advertise: internal : port + securePort ", addAdvertiseHttps(dualSpec()), "https://my-endpoint.com:443", ""}, + {"HTTPS advertise: internal : securePort, no cert ", addAdvertiseHttps(removeCert(httpsSpec())), "", "Port"}, + {"HTTPS advertise: internal : port + securePort, no cert", addAdvertiseHttps(removeCert(dualSpec())), "https://my-endpoint.com:443", ""}, + {"HTTPS advertise: external IPs : empty ", addAdvertiseHttps(addExternalIPs(emptySpec())), "", "Port"}, + {"HTTPS advertise: external IPs : port ", addAdvertiseHttps(addExternalIPs(httpSpec())), "https://my-endpoint.com:443", ""}, + {"HTTPS advertise: external IPs : securePort ", addAdvertiseHttps(addExternalIPs(httpsSpec())), "https://my-endpoint.com:443", ""}, + {"HTTPS advertise: external IPs : port + securePort ", addAdvertiseHttps(addExternalIPs(dualSpec())), "https://my-endpoint.com:443", ""}, + {"HTTPS advertise: external IPs : securePort, no cert ", addAdvertiseHttps(addExternalIPs(removeCert(httpsSpec()))), "", "Port"}, + {"HTTPS advertise: external IPs : port + securePort, no cert", addAdvertiseHttps(addExternalIPs(removeCert(dualSpec()))), "https://my-endpoint.com:443", ""}, + {"HTTPS advertise: external Hostnames: empty ", addAdvertiseHttps(addExternalHostnames(emptySpec())), "", "Port"}, + {"HTTPS advertise: external Hostnames: port ", addAdvertiseHttps(addExternalHostnames(httpSpec())), "https://my-endpoint.com:443", ""}, + {"HTTPS advertise: external Hostnames: securePort ", addAdvertiseHttps(addExternalHostnames(httpsSpec())), "https://my-endpoint.com:443", ""}, + {"HTTPS advertise: external Hostnames: port + securePort ", addAdvertiseHttps(addExternalHostnames(dualSpec())), "https://my-endpoint.com:443", ""}, + {"HTTPS advertise: external Hostnames: securePort, no cert ", addAdvertiseHttps(addExternalHostnames(removeCert(httpsSpec()))), "", "Port"}, + {"HTTPS advertise: external Hostnames: port + securePort, no cert", addAdvertiseHttps(addExternalHostnames(removeCert(dualSpec()))), "https://my-endpoint.com:443", ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := tt.store.GetAdvertiseEndpointUrl() + assert.Equal(t, tt.want, got) + if tt.wantErrContain != "" { + assert.ErrorContains(t, err, tt.wantErrContain) + } else { + assert.NoError(t, err) + } + }) + + if tt.store.Spec.Hosting != nil { + t.Run("with DNS names: "+tt.name, func(t *testing.T) { + // dnsNames shouldn't change the test result at all + s := tt.store.DeepCopy() + s.Spec.Hosting.DNSNames = []string{"should.not.show.up"} + got, err := s.GetAdvertiseEndpointUrl() + assert.Equal(t, tt.want, got) + if tt.wantErrContain != "" { + assert.ErrorContains(t, err, tt.wantErrContain) + } else { + assert.NoError(t, err) + } + }) + } + } +} diff --git a/pkg/apis/ceph.rook.io/v1/types.go b/pkg/apis/ceph.rook.io/v1/types.go index 3a1dfa212b76..b0dfeb64a25c 100755 --- a/pkg/apis/ceph.rook.io/v1/types.go +++ b/pkg/apis/ceph.rook.io/v1/types.go @@ -1497,7 +1497,10 @@ type ObjectStoreSpec struct { // +optional AllowUsersInNamespaces []string `json:"allowUsersInNamespaces,omitempty"` - // Hosting settings for the object store + // Hosting settings for the object store. + // A common use case for hosting configuration is to inform Rook of endpoints that support DNS + // wildcards, which in turn allows virtual host-style bucket addressing. + // +nullable // +optional Hosting *ObjectStoreHostingSpec `json:"hosting,omitempty"` } @@ -1675,16 +1678,47 @@ type ObjectEndpoints struct { // ObjectStoreHostingSpec represents the hosting settings for the object store type ObjectStoreHostingSpec struct { - // A list of DNS names in which bucket can be accessed via virtual host path. These names need to valid according RFC-1123. - // Each domain requires wildcard support like ingress loadbalancer. - // Do not include the wildcard itself in the list of hostnames (e.g. use "mystore.example.com" instead of "*.mystore.example.com"). - // Add all hostnames including user-created Kubernetes Service endpoints to the list. - // CephObjectStore Service Endpoints and CephObjectZone customEndpoints are automatically added to the list. + // AdvertiseEndpoint is the default endpoint Rook will return for resources dependent on this + // object store. This endpoint will be returned to CephObjectStoreUsers, Object Bucket Claims, + // and COSI Buckets/Accesses. + // By default, Rook returns the endpoint for the object store's Kubernetes service using HTTPS + // with `gateway.securePort` if it is defined (otherwise, HTTP with `gateway.port`). + // +nullable + // +optional + AdvertiseEndpoint *ObjectEndpointSpec `json:"advertiseEndpoint,omitempty"` + // A list of DNS host names on which object store gateways will accept client S3 connections. + // When specified, object store gateways will reject client S3 connections to hostnames that are + // not present in this list, so include all endpoints. + // The object store's advertiseEndpoint and Kubernetes service endpoint, plus CephObjectZone + // `customEndpoints` are automatically added to the list but may be set here again if desired. + // Each DNS name must be valid according RFC-1123. + // If the DNS name corresponds to an endpoint with DNS wildcard support, do not include the + // wildcard itself in the list of hostnames. + // E.g., use "mystore.example.com" instead of "*.mystore.example.com". // The feature is supported only for Ceph v18 and later versions. // +optional DNSNames []string `json:"dnsNames,omitempty"` } +// ObjectEndpointSpec represents an object store endpoint +type ObjectEndpointSpec struct { + // DnsName is the DNS name (in RFC-1123 format) of the endpoint. + // If the DNS name corresponds to an endpoint with DNS wildcard support, do not include the + // wildcard itself in the list of hostnames. + // E.g., use "mystore.example.com" instead of "*.mystore.example.com". + // +kubebuilder:validation:MinLength=1 + // +required + DnsName string `json:"dnsName"` + // Port is the port on which S3 connections can be made for this endpoint. + // +kubebuilder:validation:Minimum=1 + // +kubebuilder:validation:Maximum=65535 + // +required + Port int32 `json:"port"` + // UseTls defines whether the endpoint uses TLS (HTTPS) or not (HTTP). + // +required + UseTls bool `json:"useTls"` +} + // +genclient // +genclient:noStatus // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object diff --git a/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go b/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go index 913fd8476794..96ba595f379e 100644 --- a/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go +++ b/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go @@ -3442,6 +3442,22 @@ func (in *OSDStore) DeepCopy() *OSDStore { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ObjectEndpointSpec) DeepCopyInto(out *ObjectEndpointSpec) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ObjectEndpointSpec. +func (in *ObjectEndpointSpec) DeepCopy() *ObjectEndpointSpec { + if in == nil { + return nil + } + out := new(ObjectEndpointSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ObjectEndpoints) DeepCopyInto(out *ObjectEndpoints) { *out = *in @@ -3530,6 +3546,11 @@ func (in *ObjectSharedPoolsSpec) DeepCopy() *ObjectSharedPoolsSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ObjectStoreHostingSpec) DeepCopyInto(out *ObjectStoreHostingSpec) { *out = *in + if in.AdvertiseEndpoint != nil { + in, out := &in.AdvertiseEndpoint, &out.AdvertiseEndpoint + *out = new(ObjectEndpointSpec) + **out = **in + } if in.DNSNames != nil { in, out := &in.DNSNames, &out.DNSNames *out = make([]string, len(*in)) diff --git a/pkg/operator/ceph/object/admin.go b/pkg/operator/ceph/object/admin.go index 9cc3b3612e7b..3289cc9e80ad 100644 --- a/pkg/operator/ceph/object/admin.go +++ b/pkg/operator/ceph/object/admin.go @@ -19,6 +19,7 @@ package object import ( "encoding/json" "fmt" + "math/rand" "net/http" "net/http/httputil" "regexp" @@ -63,6 +64,9 @@ type debugHTTPClient struct { logger *capnslog.PackageLogger } +// global rand source that can be overridden for unit tests +var randSrc = rand.New(rand.NewSource(rand.Int63())) //nolint:gosec // G404: cryptographically weak RNG is fine here + // NewDebugHTTPClient helps us mutating the HTTP client to debug the request/response func NewDebugHTTPClient(client admin.HTTPClient, logger *capnslog.PackageLogger) *debugHTTPClient { return &debugHTTPClient{client, logger} @@ -116,7 +120,7 @@ func NewMultisiteContext(context *clusterd.Context, clusterInfo *cephclient.Clus objContext := NewContext(context, clusterInfo, store.Name) objContext.UID = string(store.UID) - if err := UpdateEndpoint(objContext, store); err != nil { + if err := UpdateEndpointForAdminOps(objContext, store); err != nil { return nil, err } @@ -131,16 +135,39 @@ func NewMultisiteContext(context *clusterd.Context, clusterInfo *cephclient.Clus return objContext, nil } -// UpdateEndpoint updates an object.Context using the latest info from the CephObjectStore spec -func UpdateEndpoint(objContext *Context, store *cephv1.CephObjectStore) error { - nsName := fmt.Sprintf("%s/%s", objContext.clusterInfo.Namespace, objContext.Name) +// GetAdminOpsEndpoint returns an endpoint that can be used to perform RGW admin ops +// It returns an HTTPS endpoint if available. It prefers direct routes to the RGW(s). +func GetAdminOpsEndpoint(s *cephv1.CephObjectStore) (string, error) { + nsName := fmt.Sprintf("%s/%s", s.Namespace, s.Name) - port, err := store.Spec.GetPort() + port, err := s.Spec.GetPort() if err != nil { - return errors.Wrapf(err, "failed to get port for object store %q", nsName) + return "", errors.Wrapf(err, "failed to get port for object store %q", nsName) + } + + domain := s.GetServiceDomainName() + if s.Spec.IsExternal() { + // if the store is external, pick a random external endpoint to use. if the endpoint is down, this + // reconcile may fail, but a future reconcile will eventually pick a different endpoint to try + endpoints := []string{} + for _, e := range s.Spec.Gateway.ExternalRgwEndpoints { + endpoints = append(endpoints, e.String()) + } + idx := randSrc.Intn(len(endpoints)) + domain = endpoints[idx] } - objContext.Endpoint = BuildDNSEndpoint(GetDomainName(store), port, store.Spec.IsTLSEnabled()) + return BuildDNSEndpoint(domain, port, s.Spec.IsTLSEnabled()), nil +} + +// UpdateEndpointForAdminOps updates the object.Context endpoint with the latest admin ops endpoint +// for the CephObjectStore. +func UpdateEndpointForAdminOps(objContext *Context, store *cephv1.CephObjectStore) error { + endpoint, err := GetAdminOpsEndpoint(store) + if err != nil { + return err + } + objContext.Endpoint = endpoint return nil } diff --git a/pkg/operator/ceph/object/admin_test.go b/pkg/operator/ceph/object/admin_test.go index a93ed0ef4960..047c440affeb 100644 --- a/pkg/operator/ceph/object/admin_test.go +++ b/pkg/operator/ceph/object/admin_test.go @@ -18,10 +18,12 @@ package object import ( "encoding/json" + "math/rand" "testing" "time" "github.com/pkg/errors" + cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" v1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/daemon/ceph/client" @@ -29,6 +31,7 @@ import ( "github.com/rook/rook/pkg/util/exec" exectest "github.com/rook/rook/pkg/util/exec/test" "github.com/stretchr/testify/assert" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) func TestExtractJson(t *testing.T) { @@ -730,3 +733,146 @@ const secondPeriodUpdateWithChanges = `{ "realm_name": "my-store", "realm_epoch": 3 }` + +func TestGetAdminOpsEndpoint(t *testing.T) { + s := &cephv1.CephObjectStore{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-store", + Namespace: "my-ns", + }, + Spec: cephv1.ObjectStoreSpec{ + Gateway: cephv1.GatewaySpec{}, + // configure hosting settings to ensure they don't affect admin ops endpoints + Hosting: &cephv1.ObjectStoreHostingSpec{ + AdvertiseEndpoint: &cephv1.ObjectEndpointSpec{ + DnsName: "should.not.appear", + Port: 7777, + UseTls: false, + }, + DNSNames: []string{"also.should.not.appear"}, + }, + }, + } + + t.Run("internal", func(t *testing.T) { + t.Run("port", func(t *testing.T) { + s := s.DeepCopy() + s.Spec.Gateway.Port = 8080 + got, err := GetAdminOpsEndpoint(s) + assert.NoError(t, err) + assert.Equal(t, "http://rook-ceph-rgw-my-store.my-ns.svc:8080", got) + }) + + t.Run("securePort, no cert", func(t *testing.T) { + s := s.DeepCopy() + s.Spec.Gateway.SecurePort = 8443 + got, err := GetAdminOpsEndpoint(s) + assert.Error(t, err) + assert.Equal(t, "", got) + }) + + t.Run("securePort", func(t *testing.T) { + s := s.DeepCopy() + s.Spec.Gateway.SecurePort = 8443 + s.Spec.Gateway.SSLCertificateRef = "my-cert" + got, err := GetAdminOpsEndpoint(s) + assert.NoError(t, err) + assert.Equal(t, "https://rook-ceph-rgw-my-store.my-ns.svc:8443", got) + }) + + t.Run("port + securePort", func(t *testing.T) { + s := s.DeepCopy() + s.Spec.Gateway.Port = 8080 + s.Spec.Gateway.SecurePort = 8443 + s.Spec.Gateway.SSLCertificateRef = "my-cert" + got, err := GetAdminOpsEndpoint(s) + assert.NoError(t, err) + assert.Equal(t, "https://rook-ceph-rgw-my-store.my-ns.svc:8443", got) + }) + }) + + t.Run("external", func(t *testing.T) { + t.Run("port", func(t *testing.T) { + s := s.DeepCopy() + s.Spec.Gateway.ExternalRgwEndpoints = []cephv1.EndpointAddress{ + {IP: "192.168.1.1"}, + {Hostname: "s3.host.com"}, + } + s.Spec.Gateway.Port = 8080 + + // override rand src with known seed to keep tests stable + randSrc = rand.New(rand.NewSource(3)) //nolint:gosec // G404: cryptographically weak RNG is fine here + + got, err := GetAdminOpsEndpoint(s) + assert.NoError(t, err) + assert.Equal(t, "http://192.168.1.1:8080", got) + + got, err = GetAdminOpsEndpoint(s) + assert.NoError(t, err) + assert.Equal(t, "http://s3.host.com:8080", got) + }) + + t.Run("securePort, no cert", func(t *testing.T) { + s := s.DeepCopy() + s.Spec.Gateway.ExternalRgwEndpoints = []cephv1.EndpointAddress{ + {IP: "192.168.1.1"}, + {Hostname: "s3.host.com"}, + } + s.Spec.Gateway.SecurePort = 8443 + + // override rand src with known seed to keep tests stable + randSrc = rand.New(rand.NewSource(3)) //nolint:gosec // G404: cryptographically weak RNG is fine here + + got, err := GetAdminOpsEndpoint(s) + assert.Error(t, err) + assert.Equal(t, "", got) + + got, err = GetAdminOpsEndpoint(s) + assert.Error(t, err) + assert.Equal(t, "", got) + }) + + t.Run("securePort", func(t *testing.T) { + s := s.DeepCopy() + s.Spec.Gateway.ExternalRgwEndpoints = []cephv1.EndpointAddress{ + {IP: "192.168.1.1"}, + {Hostname: "s3.host.com"}, + } + s.Spec.Gateway.SecurePort = 8443 + s.Spec.Gateway.SSLCertificateRef = "my-cert" + + // override rand src with known seed to keep tests stable + randSrc = rand.New(rand.NewSource(3)) //nolint:gosec // G404: cryptographically weak RNG is fine here + + got, err := GetAdminOpsEndpoint(s) + assert.NoError(t, err) + assert.Equal(t, "https://192.168.1.1:8443", got) + + got, err = GetAdminOpsEndpoint(s) + assert.NoError(t, err) + assert.Equal(t, "https://s3.host.com:8443", got) + }) + + t.Run("port + securePort", func(t *testing.T) { + s := s.DeepCopy() + s.Spec.Gateway.ExternalRgwEndpoints = []cephv1.EndpointAddress{ + {IP: "192.168.1.1"}, + {Hostname: "s3.host.com"}, + } + s.Spec.Gateway.Port = 8080 + s.Spec.Gateway.SecurePort = 8443 + s.Spec.Gateway.SSLCertificateRef = "my-cert" + + // override rand src with known seed to keep tests stable + randSrc = rand.New(rand.NewSource(3)) //nolint:gosec // G404: cryptographically weak RNG is fine here + + got, err := GetAdminOpsEndpoint(s) + assert.NoError(t, err) + assert.Equal(t, "https://192.168.1.1:8443", got) + + got, err = GetAdminOpsEndpoint(s) + assert.NoError(t, err) + assert.Equal(t, "https://s3.host.com:8443", got) + }) + }) +} diff --git a/pkg/operator/ceph/object/bucket/provisioner.go b/pkg/operator/ceph/object/bucket/provisioner.go index b9b4c41b63c2..c42af8a5fc3b 100644 --- a/pkg/operator/ceph/object/bucket/provisioner.go +++ b/pkg/operator/ceph/object/bucket/provisioner.go @@ -504,23 +504,21 @@ func (p *Provisioner) setObjectContext() error { // setObjectStoreDomainName sets the provisioner.storeDomainName and provisioner.port // must be called after setObjectStoreName and setObjectStoreNamespace -func (p *Provisioner) setObjectStoreDomainName(sc *storagev1.StorageClass) error { +func (p *Provisioner) setObjectStoreDomainNameAndPort(sc *storagev1.StorageClass) error { // make sure the object store actually exists store, err := p.getObjectStore() if err != nil { return err } - p.storeDomainName = object.GetDomainName(store) - return nil -} -func (p *Provisioner) setObjectStorePort() error { - store, err := p.getObjectStore() + domainName, port, _, err := store.GetAdvertiseEndpoint() if err != nil { - return errors.Wrap(err, "failed to get cephObjectStore") + return errors.Wrapf(err, `failed to get advertise endpoint for CephObjectStore "%s/%s"`, p.clusterInfo.Namespace, p.objectStoreName) } - p.storePort, err = store.Spec.GetPort() - return err + p.storeDomainName = domainName + p.storePort = port + + return nil } func (p *Provisioner) setObjectStoreName(sc *storagev1.StorageClass) { @@ -560,12 +558,9 @@ func (p *Provisioner) populateDomainAndPort(sc *storagev1.StorageClass) error { } // If no endpoint exists let's see if CephObjectStore exists } else { - if err := p.setObjectStoreDomainName(sc); err != nil { + if err := p.setObjectStoreDomainNameAndPort(sc); err != nil { return errors.Wrap(err, "failed to set object store domain name") } - if err := p.setObjectStorePort(); err != nil { - return errors.Wrap(err, "failed to set object store port") - } } return nil @@ -679,8 +674,10 @@ func (p *Provisioner) setAdminOpsAPIClient() error { return errors.Wrap(err, "failed to retrieve rgw admin ops user") } - // Build endpoint - s3endpoint := object.BuildDNSEndpoint(object.GetDomainName(cephObjectStore), p.storePort, cephObjectStore.Spec.IsTLSEnabled()) + s3endpoint, err := object.GetAdminOpsEndpoint(cephObjectStore) + if err != nil { + return errors.Wrapf(err, "failed to retrieve admin ops endpoint") + } // If DEBUG level is set we will mutate the HTTP client for printing request and response if logger.LevelAt(capnslog.DEBUG) { diff --git a/pkg/operator/ceph/object/controller.go b/pkg/operator/ceph/object/controller.go index 95880679abc1..c7feee60eb85 100644 --- a/pkg/operator/ceph/object/controller.go +++ b/pkg/operator/ceph/object/controller.go @@ -405,7 +405,7 @@ func (r *ReconcileCephObjectStore) reconcileCreateObjectStore(cephObjectStore *c } } - if err := UpdateEndpoint(objContext, cephObjectStore); err != nil { + if err := UpdateEndpointForAdminOps(objContext, cephObjectStore); err != nil { return r.setFailedStatus(k8sutil.ObservedGenerationNotAvailable, namespacedName, "failed to set endpoint", err) } } else { @@ -437,7 +437,7 @@ func (r *ReconcileCephObjectStore) reconcileCreateObjectStore(cephObjectStore *c return r.setFailedStatus(k8sutil.ObservedGenerationNotAvailable, namespacedName, "failed to reconcile service", err) } - if err := UpdateEndpoint(objContext, cephObjectStore); err != nil { + if err := UpdateEndpointForAdminOps(objContext, cephObjectStore); err != nil { return r.setFailedStatus(k8sutil.ObservedGenerationNotAvailable, namespacedName, "failed to set endpoint", err) } diff --git a/pkg/operator/ceph/object/rgw.go b/pkg/operator/ceph/object/rgw.go index f41ecff1f012..85acee08b5c8 100644 --- a/pkg/operator/ceph/object/rgw.go +++ b/pkg/operator/ceph/object/rgw.go @@ -19,7 +19,6 @@ package object import ( "fmt" - "math/rand" "net/http" "os" "reflect" @@ -331,39 +330,11 @@ func EmptyPool(pool cephv1.PoolSpec) bool { return reflect.DeepEqual(pool, cephv1.PoolSpec{}) } -// GetDomainName build the dns name to reach out the service endpoint -func GetDomainName(s *cephv1.CephObjectStore) string { - return getDomainName(s, true) -} - func GetStableDomainName(s *cephv1.CephObjectStore) string { - return getDomainName(s, false) -} - -func getDomainName(s *cephv1.CephObjectStore, returnRandomDomainIfMultiple bool) string { - endpoints := []string{} - if s.Spec.IsExternal() { - // if the store is external, pick a random endpoint to use. if the endpoint is down, this - // reconcile may fail, but a future reconcile will eventually pick a different endpoint to try - for _, e := range s.Spec.Gateway.ExternalRgwEndpoints { - endpoints = append(endpoints, e.String()) - } - } else if s.Spec.Hosting != nil && len(s.Spec.Hosting.DNSNames) > 0 { - // if the store is internal and has DNS names, pick a random DNS name to use - endpoints = s.Spec.Hosting.DNSNames - } else { - return domainNameOfService(s) - } - - idx := 0 - if returnRandomDomainIfMultiple { - idx = rand.Intn(len(endpoints)) //nolint:gosec // G404: cryptographically weak RNG is fine here + if !s.Spec.IsExternal() { + return s.GetServiceDomainName() } - return endpoints[idx] -} - -func domainNameOfService(s *cephv1.CephObjectStore) string { - return fmt.Sprintf("%s-%s.%s.%s", AppName, s.Name, s.Namespace, svcDNSSuffix) + return s.Spec.Gateway.ExternalRgwEndpoints[0].String() } func getAllDomainNames(s *cephv1.CephObjectStore) []string { @@ -376,7 +347,9 @@ func getAllDomainNames(s *cephv1.CephObjectStore) []string { return domains } - return []string{domainNameOfService(s)} + // do not return hosting.dnsNames in this list because Rook has no way of knowing for sure how + // they can be used. some might be TLS-only or non-TLS, or inaccessible from k8s + return []string{s.GetServiceDomainName()} } func getAllDNSEndpoints(s *cephv1.CephObjectStore, port int32, secure bool) []string { diff --git a/pkg/operator/ceph/object/rgw_test.go b/pkg/operator/ceph/object/rgw_test.go index 2723ca5a3a5b..65829b354e99 100644 --- a/pkg/operator/ceph/object/rgw_test.go +++ b/pkg/operator/ceph/object/rgw_test.go @@ -202,14 +202,7 @@ func TestEmptyPoolSpec(t *testing.T) { } func TestBuildDomainNameAndEndpoint(t *testing.T) { - s := &cephv1.CephObjectStore{ - ObjectMeta: metav1.ObjectMeta{ - Name: "my-store", - Namespace: "rook-ceph", - }, - } - dns := GetDomainName(s) - assert.Equal(t, "rook-ceph-rgw-my-store.rook-ceph.svc", dns) + dns := "rook-ceph-rgw-my-store.rook-ceph.svc" // non-secure endpoint var port int32 = 80 diff --git a/pkg/operator/ceph/object/spec.go b/pkg/operator/ceph/object/spec.go index 3213cc435c01..701042d950bb 100644 --- a/pkg/operator/ceph/object/spec.go +++ b/pkg/operator/ceph/object/spec.go @@ -922,16 +922,26 @@ func renderProbe(cfg rgwProbeConfig) (string, error) { } func (c *clusterConfig) addDNSNamesToRGWServer() (string, error) { - if (c.store.Spec.Hosting == nil) || len(c.store.Spec.Hosting.DNSNames) <= 0 { + if c.store.Spec.Hosting == nil { + return "", nil + } + if !c.store.AdvertiseEndpointIsSet() && len(c.store.Spec.Hosting.DNSNames) == 0 { return "", nil } if !c.clusterInfo.CephVersion.IsAtLeastReef() { return "", errors.New("rgw dns names are supported from ceph v18 onwards") } - // add default RGW service name to dns names - dnsNames := c.store.Spec.Hosting.DNSNames - dnsNames = append(dnsNames, domainNameOfService(c.store)) + dnsNames := []string{} + + if c.store.AdvertiseEndpointIsSet() { + dnsNames = append(dnsNames, c.store.Spec.Hosting.AdvertiseEndpoint.DnsName) + } + + dnsNames = append(dnsNames, c.store.Spec.Hosting.DNSNames...) + + // add default RGW service domain name to ensure RGW doesn't reject it + dnsNames = append(dnsNames, c.store.GetServiceDomainName()) // add custom endpoints from zone spec if exists if c.store.Spec.Zone.Name != "" { diff --git a/pkg/operator/ceph/object/spec_test.go b/pkg/operator/ceph/object/spec_test.go index 5665fc0902f2..e13df82650a1 100644 --- a/pkg/operator/ceph/object/spec_test.go +++ b/pkg/operator/ceph/object/spec_test.go @@ -914,6 +914,9 @@ func TestAddDNSNamesToRGWPodSpec(t *testing.T) { DataPathMap: data, } } + + cephV18 := cephver.CephVersion{Major: 18, Minor: 0, Extra: 0} + tests := []struct { name string dnsNames []string @@ -923,18 +926,18 @@ func TestAddDNSNamesToRGWPodSpec(t *testing.T) { CustomEndpoints []string wantErr bool }{ - {"no dns names ceph v18", []string{}, "", cephver.CephVersion{Major: 18, Minor: 0, Extra: 0}, "", []string{}, false}, - {"no dns names with zone ceph v18", []string{}, "", cephver.CephVersion{Major: 18, Minor: 0, Extra: 0}, "myzone", []string{}, false}, - {"no dns names with zone and custom endpoints ceph v18", []string{}, "", cephver.CephVersion{Major: 18, Minor: 0, Extra: 0}, "myzone", []string{"http://my.custom.endpoint1:80", "http://my.custom.endpoint2:80"}, false}, - {"one dns name ceph v18", []string{"my.dns.name"}, "--rgw-dns-name=my.dns.name,rook-ceph-rgw-default.mycluster.svc", cephver.CephVersion{Major: 18, Minor: 0, Extra: 0}, "", []string{}, false}, - {"multiple dns names ceph v18", []string{"my.dns.name1", "my.dns.name2"}, "--rgw-dns-name=my.dns.name1,my.dns.name2,rook-ceph-rgw-default.mycluster.svc", cephver.CephVersion{Major: 18, Minor: 0, Extra: 0}, "", []string{}, false}, - {"duplicate dns names ceph v18", []string{"my.dns.name1", "my.dns.name2", "my.dns.name2"}, "--rgw-dns-name=my.dns.name1,my.dns.name2,rook-ceph-rgw-default.mycluster.svc", cephver.CephVersion{Major: 18, Minor: 0, Extra: 0}, "", []string{}, false}, - {"invalid dns name ceph v18", []string{"!my.invalid-dns.com"}, "", cephver.CephVersion{Major: 18, Minor: 0, Extra: 0}, "", []string{}, true}, - {"mixed invalid and valid dns names ceph v18", []string{"my.dns.name", "!my.invalid-dns.name"}, "", cephver.CephVersion{Major: 18, Minor: 0, Extra: 0}, "", []string{}, true}, - {"dns name with zone without custom endpoints ceph v18", []string{"my.dns.name1", "my.dns.name2"}, "--rgw-dns-name=my.dns.name1,my.dns.name2,rook-ceph-rgw-default.mycluster.svc", cephver.CephVersion{Major: 18, Minor: 0, Extra: 0}, "myzone", []string{}, false}, - {"dns name with zone with custom endpoints ceph v18", []string{"my.dns.name1", "my.dns.name2"}, "--rgw-dns-name=my.dns.name1,my.dns.name2,rook-ceph-rgw-default.mycluster.svc,my.custom.endpoint1,my.custom.endpoint2", cephver.CephVersion{Major: 18, Minor: 0, Extra: 0}, "myzone", []string{"http://my.custom.endpoint1:80", "http://my.custom.endpoint2:80"}, false}, - {"dns name with zone with custom invalid endpoints ceph v18", []string{"my.dns.name1", "my.dns.name2"}, "", cephver.CephVersion{Major: 18, Minor: 0, Extra: 0}, "myzone", []string{"http://my.custom.endpoint:80", "http://!my.invalid-custom.endpoint:80"}, true}, - {"dns name with zone with mixed invalid and valid dnsnames/custom endpoint ceph v18", []string{"my.dns.name", "!my.dns.name"}, "", cephver.CephVersion{Major: 18, Minor: 0, Extra: 0}, "myzone", []string{"http://my.custom.endpoint1:80", "http://my.custom.endpoint2:80:80"}, true}, + {"no dns names ceph v18", []string{}, "", cephV18, "", []string{}, false}, + {"no dns names with zone ceph v18", []string{}, "", cephV18, "myzone", []string{}, false}, + {"no dns names with zone and custom endpoints ceph v18", []string{}, "", cephV18, "myzone", []string{"http://my.custom.endpoint1:80", "http://my.custom.endpoint2:80"}, false}, + {"one dns name ceph v18", []string{"my.dns.name"}, "--rgw-dns-name=my.dns.name,rook-ceph-rgw-default.mycluster.svc", cephV18, "", []string{}, false}, + {"multiple dns names ceph v18", []string{"my.dns.name1", "my.dns.name2"}, "--rgw-dns-name=my.dns.name1,my.dns.name2,rook-ceph-rgw-default.mycluster.svc", cephV18, "", []string{}, false}, + {"duplicate dns names ceph v18", []string{"my.dns.name1", "my.dns.name2", "my.dns.name2"}, "--rgw-dns-name=my.dns.name1,my.dns.name2,rook-ceph-rgw-default.mycluster.svc", cephV18, "", []string{}, false}, + {"invalid dns name ceph v18", []string{"!my.invalid-dns.com"}, "", cephV18, "", []string{}, true}, + {"mixed invalid and valid dns names ceph v18", []string{"my.dns.name", "!my.invalid-dns.name"}, "", cephV18, "", []string{}, true}, + {"dns name with zone without custom endpoints ceph v18", []string{"my.dns.name1", "my.dns.name2"}, "--rgw-dns-name=my.dns.name1,my.dns.name2,rook-ceph-rgw-default.mycluster.svc", cephV18, "myzone", []string{}, false}, + {"dns name with zone with custom endpoints ceph v18", []string{"my.dns.name1", "my.dns.name2"}, "--rgw-dns-name=my.dns.name1,my.dns.name2,rook-ceph-rgw-default.mycluster.svc,my.custom.endpoint1,my.custom.endpoint2", cephV18, "myzone", []string{"http://my.custom.endpoint1:80", "http://my.custom.endpoint2:80"}, false}, + {"dns name with zone with custom invalid endpoints ceph v18", []string{"my.dns.name1", "my.dns.name2"}, "", cephV18, "myzone", []string{"http://my.custom.endpoint:80", "http://!my.invalid-custom.endpoint:80"}, true}, + {"dns name with zone with mixed invalid and valid dnsnames/custom endpoint ceph v18", []string{"my.dns.name", "!my.dns.name"}, "", cephV18, "myzone", []string{"http://my.custom.endpoint1:80", "http://my.custom.endpoint2:80:80"}, true}, {"no dns names ceph v17", []string{}, "", cephver.CephVersion{Major: 17, Minor: 0, Extra: 0}, "", []string{}, false}, {"one dns name ceph v17", []string{"my.dns.name"}, "", cephver.CephVersion{Major: 17, Minor: 0, Extra: 0}, "", []string{}, true}, {"multiple dns names ceph v17", []string{"my.dns.name1", "my.dns.name2"}, "", cephver.CephVersion{Major: 17, Minor: 0, Extra: 0}, "", []string{}, true}, @@ -965,9 +968,95 @@ func TestAddDNSNamesToRGWPodSpec(t *testing.T) { assert.NoError(t, err) } assert.Equal(t, tt.expectedDNSArg, res) - }) } + + t.Run("advertiseEndpoint http, no dnsNames", func(t *testing.T) { + c := setupTest("", cephV18, []string{}, []string{}) + c.store.Spec.Hosting = &cephv1.ObjectStoreHostingSpec{ + AdvertiseEndpoint: &cephv1.ObjectEndpointSpec{ + DnsName: "my.endpoint.com", + Port: 80, + }, + } + res, err := c.addDNSNamesToRGWServer() + assert.NoError(t, err) + assert.Equal(t, "--rgw-dns-name=my.endpoint.com,rook-ceph-rgw-default.mycluster.svc", res) + }) + + t.Run("advertiseEndpoint https, no dnsNames", func(t *testing.T) { + c := setupTest("", cephV18, []string{}, []string{}) + c.store.Spec.Hosting = &cephv1.ObjectStoreHostingSpec{ + AdvertiseEndpoint: &cephv1.ObjectEndpointSpec{ + DnsName: "my.endpoint.com", + Port: 443, + UseTls: true, + }, + } + res, err := c.addDNSNamesToRGWServer() + assert.NoError(t, err) + assert.Equal(t, "--rgw-dns-name=my.endpoint.com,rook-ceph-rgw-default.mycluster.svc", res) + }) + + t.Run("advertiseEndpoint is svc", func(t *testing.T) { + c := setupTest("", cephV18, []string{}, []string{}) + c.store.Spec.Hosting = &cephv1.ObjectStoreHostingSpec{ + AdvertiseEndpoint: &cephv1.ObjectEndpointSpec{ + DnsName: "rook-ceph-rgw-default.mycluster.svc", + Port: 443, + UseTls: true, + }, + } + res, err := c.addDNSNamesToRGWServer() + assert.NoError(t, err) + // ensures duplicates are removed + assert.Equal(t, "--rgw-dns-name=rook-ceph-rgw-default.mycluster.svc", res) + }) + + t.Run("advertiseEndpoint https, no dnsNames, with zone custom endpoint", func(t *testing.T) { + c := setupTest("my-zone", cephV18, []string{}, []string{"multisite.endpoint.com"}) + c.store.Spec.Hosting = &cephv1.ObjectStoreHostingSpec{ + AdvertiseEndpoint: &cephv1.ObjectEndpointSpec{ + DnsName: "my.endpoint.com", + Port: 443, + UseTls: true, + }, + } + res, err := c.addDNSNamesToRGWServer() + assert.NoError(t, err) + assert.Equal(t, "--rgw-dns-name=my.endpoint.com,rook-ceph-rgw-default.mycluster.svc,multisite.endpoint.com", res) + }) + + t.Run("advertiseEndpoint https, with dnsNames, with zone custom endpoint", func(t *testing.T) { + c := setupTest("my-zone", cephV18, []string{}, []string{"multisite.endpoint.com"}) + c.store.Spec.Hosting = &cephv1.ObjectStoreHostingSpec{ + AdvertiseEndpoint: &cephv1.ObjectEndpointSpec{ + DnsName: "my.endpoint.com", + Port: 443, + UseTls: true, + }, + DNSNames: []string{"extra.endpoint.com", "extra.endpoint.net"}, + } + res, err := c.addDNSNamesToRGWServer() + assert.NoError(t, err) + assert.Equal(t, "--rgw-dns-name=my.endpoint.com,extra.endpoint.com,extra.endpoint.net,rook-ceph-rgw-default.mycluster.svc,multisite.endpoint.com", res) + }) + + t.Run("advertiseEndpoint https, with dnsNames, with zone custom endpoint, duplicates", func(t *testing.T) { + c := setupTest("my-zone", cephV18, []string{}, []string{"extra.endpoint.com"}) + c.store.Spec.Hosting = &cephv1.ObjectStoreHostingSpec{ + AdvertiseEndpoint: &cephv1.ObjectEndpointSpec{ + DnsName: "my.endpoint.com", + Port: 443, + UseTls: true, + }, + DNSNames: []string{"my.endpoint.com", "extra.endpoint.com"}, + } + res, err := c.addDNSNamesToRGWServer() + assert.NoError(t, err) + t.Log(res) + assert.Equal(t, "--rgw-dns-name=my.endpoint.com,extra.endpoint.com,rook-ceph-rgw-default.mycluster.svc", res) + }) } func TestGetHostnameFromEndpoint(t *testing.T) { diff --git a/pkg/operator/ceph/object/status.go b/pkg/operator/ceph/object/status.go index d64f9ad9c00b..d6d1c0e0f67e 100644 --- a/pkg/operator/ceph/object/status.go +++ b/pkg/operator/ceph/object/status.go @@ -18,6 +18,7 @@ package object import ( "context" + "fmt" "github.com/pkg/errors" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" @@ -90,15 +91,28 @@ func updateStatus(ctx context.Context, observedGeneration int64, client client.C } func buildStatusInfo(cephObjectStore *cephv1.CephObjectStore) map[string]string { + nsName := fmt.Sprintf("%s/%s", cephObjectStore.Namespace, cephObjectStore.Name) + m := make(map[string]string) - if cephObjectStore.Spec.Gateway.SecurePort != 0 && cephObjectStore.Spec.Gateway.Port != 0 { - m["secureEndpoint"] = BuildDNSEndpoint(GetStableDomainName(cephObjectStore), cephObjectStore.Spec.Gateway.SecurePort, true) + advertiseEndpoint, err := cephObjectStore.GetAdvertiseEndpointUrl() + if err != nil { + // lots of validation happens before this point, so this should be nearly impossible + logger.Errorf("failed to get advertise endpoint for CephObjectStore %q to record on status; continuing without this. %v", nsName, err) + } + + if cephObjectStore.AdvertiseEndpointIsSet() { + // if the advertise endpoint is explicitly set, it takes precedence as the only endpoint + m["endpoint"] = advertiseEndpoint + return m + } + + if cephObjectStore.Spec.Gateway.Port != 0 && cephObjectStore.Spec.Gateway.SecurePort != 0 { + // by definition, advertiseEndpoint should prefer HTTPS, so the inverse arrangement doesn't apply + m["secureEndpoint"] = advertiseEndpoint m["endpoint"] = BuildDNSEndpoint(GetStableDomainName(cephObjectStore), cephObjectStore.Spec.Gateway.Port, false) - } else if cephObjectStore.Spec.Gateway.SecurePort != 0 { - m["endpoint"] = BuildDNSEndpoint(GetStableDomainName(cephObjectStore), cephObjectStore.Spec.Gateway.SecurePort, true) } else { - m["endpoint"] = BuildDNSEndpoint(GetStableDomainName(cephObjectStore), cephObjectStore.Spec.Gateway.Port, false) + m["endpoint"] = advertiseEndpoint } return m diff --git a/pkg/operator/ceph/object/status_test.go b/pkg/operator/ceph/object/status_test.go index be8b743c977a..d6195a743ee2 100644 --- a/pkg/operator/ceph/object/status_test.go +++ b/pkg/operator/ceph/object/status_test.go @@ -25,36 +25,140 @@ import ( ) func TestBuildStatusInfo(t *testing.T) { - // Port enabled and SecurePort disabled - cephObjectStore := &cephv1.CephObjectStore{ + baseStore := &cephv1.CephObjectStore{ ObjectMeta: metav1.ObjectMeta{ Name: "my-store", Namespace: "rook-ceph", }, } - cephObjectStore.Spec.Gateway.Port = 80 - statusInfo := buildStatusInfo(cephObjectStore) + // Port enabled and SecurePort disabled + s := baseStore.DeepCopy() + s.Spec.Gateway.Port = 80 + statusInfo := buildStatusInfo(s) + assert.NotEmpty(t, statusInfo["endpoint"]) assert.Empty(t, statusInfo["secureEndpoint"]) assert.Equal(t, "http://rook-ceph-rgw-my-store.rook-ceph.svc:80", statusInfo["endpoint"]) // SecurePort enabled and Port disabled - cephObjectStore.Spec.Gateway.Port = 0 - cephObjectStore.Spec.Gateway.SecurePort = 443 + s = baseStore.DeepCopy() + s.Spec.Gateway.Port = 0 + s.Spec.Gateway.SecurePort = 443 + s.Spec.Gateway.SSLCertificateRef = "my-cert" - statusInfo = buildStatusInfo(cephObjectStore) + statusInfo = buildStatusInfo(s) assert.NotEmpty(t, statusInfo["endpoint"]) assert.Empty(t, statusInfo["secureEndpoint"]) assert.Equal(t, "https://rook-ceph-rgw-my-store.rook-ceph.svc:443", statusInfo["endpoint"]) // Both Port and SecurePort enabled - cephObjectStore.Spec.Gateway.Port = 80 - cephObjectStore.Spec.Gateway.SecurePort = 443 + s = baseStore.DeepCopy() + s.Spec.Gateway.Port = 80 + s.Spec.Gateway.SecurePort = 443 + s.Spec.Gateway.SSLCertificateRef = "my-cert" - statusInfo = buildStatusInfo(cephObjectStore) + statusInfo = buildStatusInfo(s) assert.NotEmpty(t, statusInfo["endpoint"]) assert.NotEmpty(t, statusInfo["secureEndpoint"]) assert.Equal(t, "http://rook-ceph-rgw-my-store.rook-ceph.svc:80", statusInfo["endpoint"]) assert.Equal(t, "https://rook-ceph-rgw-my-store.rook-ceph.svc:443", statusInfo["secureEndpoint"]) + + t.Run("advertiseEndpoint http", func(t *testing.T) { + baseStore := &cephv1.CephObjectStore{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-store", + Namespace: "rook-ceph", + }, + Spec: cephv1.ObjectStoreSpec{ + Hosting: &cephv1.ObjectStoreHostingSpec{ + AdvertiseEndpoint: &cephv1.ObjectEndpointSpec{ + DnsName: "my.endpoint.com", + Port: 80, + UseTls: false, + }, + }, + }, + } + + // Port enabled and SecurePort disabled + s := baseStore.DeepCopy() + s.Spec.Gateway.Port = 80 + statusInfo := buildStatusInfo(s) + + assert.NotEmpty(t, statusInfo["endpoint"]) + assert.Empty(t, statusInfo["secureEndpoint"]) + assert.Equal(t, "http://my.endpoint.com:80", statusInfo["endpoint"]) + + // SecurePort enabled and Port disabled + s = baseStore.DeepCopy() + s.Spec.Gateway.Port = 0 + s.Spec.Gateway.SecurePort = 443 + s.Spec.Gateway.SSLCertificateRef = "my-cert" + + statusInfo = buildStatusInfo(s) + assert.NotEmpty(t, statusInfo["endpoint"]) + assert.Empty(t, statusInfo["secureEndpoint"]) + assert.Equal(t, "http://my.endpoint.com:80", statusInfo["endpoint"]) + + // Both Port and SecurePort enabled + s = baseStore.DeepCopy() + s.Spec.Gateway.Port = 80 + s.Spec.Gateway.SecurePort = 443 + s.Spec.Gateway.SSLCertificateRef = "my-cert" + + statusInfo = buildStatusInfo(s) + assert.NotEmpty(t, statusInfo["endpoint"]) + assert.Empty(t, statusInfo["secureEndpoint"]) + assert.Equal(t, "http://my.endpoint.com:80", statusInfo["endpoint"]) + }) + + t.Run("advertiseEndpoint https", func(t *testing.T) { + baseStore := &cephv1.CephObjectStore{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-store", + Namespace: "rook-ceph", + }, + Spec: cephv1.ObjectStoreSpec{ + Hosting: &cephv1.ObjectStoreHostingSpec{ + AdvertiseEndpoint: &cephv1.ObjectEndpointSpec{ + DnsName: "my.endpoint.com", + Port: 443, + UseTls: true, + }, + }, + }, + } + + // Port enabled and SecurePort disabled + s := baseStore.DeepCopy() + s.Spec.Gateway.Port = 80 + statusInfo := buildStatusInfo(s) + + assert.NotEmpty(t, statusInfo["endpoint"]) + assert.Empty(t, statusInfo["secureEndpoint"]) + assert.Equal(t, "https://my.endpoint.com:443", statusInfo["endpoint"]) + + // SecurePort enabled and Port disabled + s = baseStore.DeepCopy() + s.Spec.Gateway.Port = 0 + s.Spec.Gateway.SecurePort = 443 + s.Spec.Gateway.SSLCertificateRef = "my-cert" + + statusInfo = buildStatusInfo(s) + assert.NotEmpty(t, statusInfo["endpoint"]) + assert.Empty(t, statusInfo["secureEndpoint"]) + assert.Equal(t, "https://my.endpoint.com:443", statusInfo["endpoint"]) + + // Both Port and SecurePort enabled + s = baseStore.DeepCopy() + s.Spec.Gateway.Port = 80 + s.Spec.Gateway.SecurePort = 443 + s.Spec.Gateway.SSLCertificateRef = "my-cert" + + statusInfo = buildStatusInfo(s) + assert.NotEmpty(t, statusInfo["endpoint"]) + assert.Empty(t, statusInfo["secureEndpoint"]) + assert.Equal(t, "https://my.endpoint.com:443", statusInfo["endpoint"]) + }) } diff --git a/pkg/operator/ceph/object/user/controller.go b/pkg/operator/ceph/object/user/controller.go index b55112f2e8e0..f348dfa10d34 100644 --- a/pkg/operator/ceph/object/user/controller.go +++ b/pkg/operator/ceph/object/user/controller.go @@ -67,15 +67,16 @@ var controllerTypeMeta = metav1.TypeMeta{ // ReconcileObjectStoreUser reconciles a ObjectStoreUser object type ReconcileObjectStoreUser struct { - client client.Client - scheme *runtime.Scheme - context *clusterd.Context - objContext *object.AdminOpsContext - userConfig *admin.User - cephClusterSpec *cephv1.ClusterSpec - clusterInfo *cephclient.ClusterInfo - opManagerContext context.Context - recorder record.EventRecorder + client client.Client + scheme *runtime.Scheme + context *clusterd.Context + objContext *object.AdminOpsContext + advertiseEndpoint string + userConfig *admin.User + cephClusterSpec *cephv1.ClusterSpec + clusterInfo *cephclient.ClusterInfo + opManagerContext context.Context + recorder record.EventRecorder } // Add creates a new CephObjectStoreUser Controller and adds it to the Manager. The Manager will set fields on the Controller @@ -264,7 +265,7 @@ func (r *ReconcileObjectStoreUser) reconcile(request reconcile.Request) (reconci } tlsSecretName := store.Spec.Gateway.SSLCertificateRef - reconcileResponse, err = object.ReconcileCephUserSecret(r.opManagerContext, r.client, r.scheme, cephObjectStoreUser, r.userConfig, r.objContext.Endpoint, cephObjectStoreUser.Namespace, cephObjectStoreUser.Spec.Store, tlsSecretName) + reconcileResponse, err = object.ReconcileCephUserSecret(r.opManagerContext, r.client, r.scheme, cephObjectStoreUser, r.userConfig, r.advertiseEndpoint, cephObjectStoreUser.Namespace, cephObjectStoreUser.Spec.Store, tlsSecretName) if err != nil { r.updateStatus(k8sutil.ObservedGenerationNotAvailable, request.NamespacedName, k8sutil.ReconcileFailedStatus) return reconcileResponse, *cephObjectStoreUser, err @@ -395,6 +396,12 @@ func (r *ReconcileObjectStoreUser) initializeObjectStoreContext(u *cephv1.CephOb } } + advertiseEndpoint, err := store.GetAdvertiseEndpointUrl() + if err != nil { + return errors.Wrapf(err, "failed to get CephObjectStore %q advertise endpoint for object store user", u.Spec.Store) + } + r.advertiseEndpoint = advertiseEndpoint + objContext, err := object.NewMultisiteContext(r.context, r.clusterInfo, store) if err != nil { return errors.Wrapf(err, "Multisite failed to set on object context for object store user") From e4d7a29d372564e5b927706d2ed8ad2c1dc60ab9 Mon Sep 17 00:00:00 2001 From: Praveen M Date: Wed, 31 Jul 2024 19:06:56 +0530 Subject: [PATCH 4/7] csi: explicitly set Topology feature-gate issue: external-provisioner (v5) enabled topology feature-gate by default and the current implementation in Rook uses a conditional block to enable the topology feature gate. This approach now does not directly reflect the state of the `CSI_ENABLE_TOPOLOGY`. fix: replacing the conditional block with a direct use of the `CSI_ENABLE_TOPOLOGY` for flag value. Signed-off-by: Praveen M (cherry picked from commit e30e5e881fa19f987bd79bb908fd7ceef7e29a5d) --- .../ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml b/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml index d8ed72575272..ed25616151ed 100644 --- a/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml +++ b/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml @@ -38,9 +38,7 @@ spec: - "--extra-create-metadata=true" - "--prevent-volume-mode-conversion=true" - "--feature-gates=HonorPVReclaimPolicy=true" - {{ if .EnableCSITopology }} - - "--feature-gates=Topology=true" - {{ end }} + - "--feature-gates=Topology={{ .EnableCSITopology }}" {{ if .KubeApiBurst }} - "--kube-api-burst={{ .KubeApiBurst }}" {{ end }} From 95e71f1a29ef82369fe6e0a2c4eadebb32295a47 Mon Sep 17 00:00:00 2001 From: Blaine Gardner Date: Fri, 2 Aug 2024 13:04:52 -0600 Subject: [PATCH 5/7] object: use advertise endpoint for admin ops RGW can only serve a single certificate. This limitation means that the prior behavior of using the default service for admin ops when TLS is enabled may mean it requires additional complex certificate management to make sure the object store uses a certificate valid for Rook internal admin ops and user connections. This is needlessly complex for users. Instead, change Rook's behavior and documentation to clarify that it will use the same endpoint intended for S3 client applications. This means that users have a more straightforward path to enabling both Rook and consuming applications. More info: https://github.com/rook/rook/issues/14530 Signed-off-by: Blaine Gardner (cherry picked from commit b4a2285aa6ef08ceb27fafcb96bbcb7e1491eeae) --- .../Object-Storage-RGW/object-storage.md | 68 ++++++++++++------- design/ceph/object/store.md | 23 ++++--- pkg/operator/ceph/object/admin.go | 25 ++----- pkg/operator/ceph/object/admin_test.go | 56 ++++++--------- 4 files changed, 81 insertions(+), 91 deletions(-) diff --git a/Documentation/Storage-Configuration/Object-Storage-RGW/object-storage.md b/Documentation/Storage-Configuration/Object-Storage-RGW/object-storage.md index d5234172145e..241159791e44 100644 --- a/Documentation/Storage-Configuration/Object-Storage-RGW/object-storage.md +++ b/Documentation/Storage-Configuration/Object-Storage-RGW/object-storage.md @@ -200,7 +200,7 @@ Then create a secret with the user credentials: kubectl -n rook-ceph create secret generic --type="kubernetes.io/rook" rgw-admin-ops-user --from-literal=accessKey= --from-literal=secretKey= ``` -If you have an external `CephCluster` CR, you can instruct Rook to consume external gateways with the following: +For an external CephCluster, configure Rook to consume external RGW servers with the following: ```yaml apiVersion: ceph.rook.io/v1 @@ -216,42 +216,35 @@ spec: # hostname: example.com ``` -Use the existing `object-external.yaml` file. Even though multiple endpoints can be specified, it is recommend to use only one endpoint. This endpoint is randomly added to `configmap` of OBC and secret of the `cephobjectstoreuser`. Rook never guarantees the randomly picked endpoint is a working one or not. -If there are multiple endpoints, please add load balancer in front of them and use the load balancer endpoint in the `externalRgwEndpoints` list. +See `object-external.yaml` for a more detailed example. -When ready, the message in the `cephobjectstore` status similar to this one: - -```console -kubectl -n rook-ceph get cephobjectstore external-store -NAME PHASE -external-store Ready - -``` - -Any pod from your cluster can now access this endpoint: - -```console -$ curl 192.168.39.182:8080 -anonymous -``` +Even though multiple `externalRgwEndpoints` can be specified, it is best to use a single endpoint. +Only the first endpoint in the list will be advertised to any consuming resources like +CephObjectStoreUsers, ObjectBucketClaims, or COSI resources. If there are multiple external RGW +endpoints, add load balancer in front of them, then use the single load balancer endpoint in the +`externalRgwEndpoints` list. ## Object store endpoint The CephObjectStore resource `status.info` contains `endpoint` (and `secureEndpoint`) fields, which -report the endpoint that can be used to access the object store as a client. +report the endpoint that can be used to access the object store as a client. This endpoint is also +advertised as the default endpoint for CephObjectStoreUsers, ObjectBucketClaims, and +Container Object Store Interface (COSI) resources. Each object store also creates a Kubernetes service that can be used as a client endpoint from within the Kubernetes cluster. The DNS name of the service is `rook-ceph-rgw-..svc`. This service DNS name is the default `endpoint` (and `secureEndpoint`). -For [external clusters](#connect-to-an-external-object-store), the default endpoints contain the -first `spec.gateway.externalRgwEndpoint` instead of the service DNS name. +For [external clusters](#connect-to-an-external-object-store), the default endpoint is the first +`spec.gateway.externalRgwEndpoint` instead of the service DNS name. -Rook always uses the default endpoint to perform management operations against the object store. -When TLS is enabled, the TLS certificate must always specify the default endpoint DNS name to allow -secure management operations. TLS configuration specification can be found in object -[gateway `securePort` documentation](../../CRDs/Object-Storage/ceph-object-store-crd.md#gateway-settings). +The advertised endpoint can be overridden using `advertiseEndpoint` in the +[`spec.hosting` config](../../CRDs/Object-Storage/ceph-object-store-crd.md#hosting-settings). + +Rook always uses the advertised endpoint to perform management operations against the object store. +When [TLS is enabled](#enable-tls), the TLS certificate must always specify the endpoint DNS name to +allow secure management operations. ## Create a Bucket @@ -508,6 +501,29 @@ kubectl -n rook-ceph get secret rook-ceph-object-user-my-store-my-user -o jsonpa kubectl -n rook-ceph get secret rook-ceph-object-user-my-store-my-user -o jsonpath='{.data.SecretKey}' | base64 --decode ``` +## Enable TLS + +TLS is critical for securing object storage data access, and it is assumed as a default by many S3 +clients. TLS is enabled for CephObjectStores by configuring +[`gateway` options](../../CRDs/Object-Storage/ceph-object-store-crd.md#gateway-settings). +Set `securePort`, and give Rook access to a TLS certificate using `sslCertificateRef`. +`caBundleRef` may be necessary as well to give the deployed gateway (RGW) access to the TLS +certificate's CA signing bundle. + +Ceph RGW only supports a **single** TLS certificate. If the given TLS certificate is a concatenation +of multiple certificates, only the first certificate will be used by the RGW as the server +certificate. Therefore, the TLS certificate given must include all endpoints that clients will use +for access as subject alternate names (SANs). + +The [CephObjectStore service endpoint](#object-store-endpoint) must be added as a SAN on the TLS +certificate. If it is not possible to add the service DNS name as a SAN on the TLS certificate, +set `hosting.advertiseEndpoint` to a TLS-approved endpoint to help ensure Rook and clients use +secure data access. + +!!! note + OpenShift users can use add `service.beta.openshift.io/serving-cert-secret-name` as a service + annotation instead of using `sslCertificateRef`. + ## Virtual host-style Bucket Access The Ceph Object Gateway supports accessing buckets using @@ -530,7 +546,7 @@ Wildcard addressing can be configured in myriad ways. Some options: The minimum recommended `hosting` configuration is exemplified below. It is important to ensure that Rook advertises the wildcard-addressable endpoint as a priority over the default. TLS is also -recommended for security. +recommended for security, and the configured TLS certificate should specify the advertise endpoint. ```yaml spec: diff --git a/design/ceph/object/store.md b/design/ceph/object/store.md index 2bfcf442a59b..4b6c23f0ca73 100644 --- a/design/ceph/object/store.md +++ b/design/ceph/object/store.md @@ -397,8 +397,19 @@ the HTTPS (`securePort`) endpoint. Because the advertised endpoint is primarily resources internal to the Kubernetes cluster, this default should be sufficient for most users, and this is the behavior expected by users when `dnsNames` is not configured, so it should be familiar. -When this feature is enabled, there is also ambiguity about which endpoint Rook should use for Admin -Ops API communication. Some users have reported issues with Rook using a `dnsNames` endpoint +When this feature is enabled, there should be no ambiguity about which endpoint Rook will use for +Admin Ops API communication. As an HTTP server, RGW is only able to return a single TLS certificate +to S3 clients ([more detail](https://github.com/rook/rook/issues/14530)). For maximum compatibility +while TLS is enabled, Rook should connect to the same endpoint that users do. Internally, Rook will +use the advertise endpoint as configured. + +Rook documentation will inform users that if TLS is enabled, they must give Rook a certificate that +accepts the service endpoint. Alternately, if that is not possible, Rook will add an +`insecureSkipTlsVerification` option to the CephObjectStore to allow users to provision a healthy +CephObjectStore. This opens users up to machine-in-the-middle attacks, so users should be advised to +only use it for test/proof-of-concept clusters, or to work around bugs temporarily. + +Some users have reported issues with Rook using a `dnsNames` endpoint (or `advertiseEndpoint`) when they wish to set up ingress certificates after Rook deployment. The obvious alternative is to have Rook always use the CephObjectStore service, but other users have expressed troubles creating certificates or CAs that allow the service endpoint in the past. @@ -424,14 +435,6 @@ While Rook add endpoints to the list for safety and convenience, users might add which Rook should not treat as a configuration bug. Rook should also ensure the list ordering is consistent between reconciles. -In order to attempt to strike the best balance for everyone, and to provide the best clarity for -users and Rook internally, Rook will always use the service endpoint for admin ops. Rook -documentation must inform users that if TLS is enabled, they must give Rook a certificate that -accepts the service endpoint. Alternately, if that is not possible, Rook will add an -`insecureSkipTlsVerification` option to the CephObjectStore to allow users to provision a healthy -CephObjectStore. This opens users up to machine-in-the-middle attacks, so users should be advised to -only use it for test/proof-of-concept clusters, or to work around bugs temporarily. - Rook can refer users to this Kubernetes doc for a suggested way that they can manage certificates in a Kubernetes cluster that work with Kubernetes services like the CephObjectStore service: https://kubernetes.io/docs/tasks/tls/managing-tls-in-a-cluster/ diff --git a/pkg/operator/ceph/object/admin.go b/pkg/operator/ceph/object/admin.go index 3289cc9e80ad..5e5a2596f515 100644 --- a/pkg/operator/ceph/object/admin.go +++ b/pkg/operator/ceph/object/admin.go @@ -19,7 +19,6 @@ package object import ( "encoding/json" "fmt" - "math/rand" "net/http" "net/http/httputil" "regexp" @@ -64,9 +63,6 @@ type debugHTTPClient struct { logger *capnslog.PackageLogger } -// global rand source that can be overridden for unit tests -var randSrc = rand.New(rand.NewSource(rand.Int63())) //nolint:gosec // G404: cryptographically weak RNG is fine here - // NewDebugHTTPClient helps us mutating the HTTP client to debug the request/response func NewDebugHTTPClient(client admin.HTTPClient, logger *capnslog.PackageLogger) *debugHTTPClient { return &debugHTTPClient{client, logger} @@ -136,28 +132,15 @@ func NewMultisiteContext(context *clusterd.Context, clusterInfo *cephclient.Clus } // GetAdminOpsEndpoint returns an endpoint that can be used to perform RGW admin ops -// It returns an HTTPS endpoint if available. It prefers direct routes to the RGW(s). func GetAdminOpsEndpoint(s *cephv1.CephObjectStore) (string, error) { nsName := fmt.Sprintf("%s/%s", s.Namespace, s.Name) - port, err := s.Spec.GetPort() + // advertise endpoint should be most likely to have a valid cert, so use it for admin ops + endpoint, err := s.GetAdvertiseEndpointUrl() if err != nil { - return "", errors.Wrapf(err, "failed to get port for object store %q", nsName) + return "", errors.Wrapf(err, "failed to get advertise endpoint for object store %q", nsName) } - - domain := s.GetServiceDomainName() - if s.Spec.IsExternal() { - // if the store is external, pick a random external endpoint to use. if the endpoint is down, this - // reconcile may fail, but a future reconcile will eventually pick a different endpoint to try - endpoints := []string{} - for _, e := range s.Spec.Gateway.ExternalRgwEndpoints { - endpoints = append(endpoints, e.String()) - } - idx := randSrc.Intn(len(endpoints)) - domain = endpoints[idx] - } - - return BuildDNSEndpoint(domain, port, s.Spec.IsTLSEnabled()), nil + return endpoint, nil } // UpdateEndpointForAdminOps updates the object.Context endpoint with the latest admin ops endpoint diff --git a/pkg/operator/ceph/object/admin_test.go b/pkg/operator/ceph/object/admin_test.go index 047c440affeb..122b4ce86ef1 100644 --- a/pkg/operator/ceph/object/admin_test.go +++ b/pkg/operator/ceph/object/admin_test.go @@ -18,7 +18,6 @@ package object import ( "encoding/json" - "math/rand" "testing" "time" @@ -742,14 +741,9 @@ func TestGetAdminOpsEndpoint(t *testing.T) { }, Spec: cephv1.ObjectStoreSpec{ Gateway: cephv1.GatewaySpec{}, - // configure hosting settings to ensure they don't affect admin ops endpoints Hosting: &cephv1.ObjectStoreHostingSpec{ - AdvertiseEndpoint: &cephv1.ObjectEndpointSpec{ - DnsName: "should.not.appear", - Port: 7777, - UseTls: false, - }, - DNSNames: []string{"also.should.not.appear"}, + // dnsNames shouldn't affect admin ops endpoints + DNSNames: []string{"should.not.appear"}, }, }, } @@ -800,16 +794,9 @@ func TestGetAdminOpsEndpoint(t *testing.T) { } s.Spec.Gateway.Port = 8080 - // override rand src with known seed to keep tests stable - randSrc = rand.New(rand.NewSource(3)) //nolint:gosec // G404: cryptographically weak RNG is fine here - got, err := GetAdminOpsEndpoint(s) assert.NoError(t, err) assert.Equal(t, "http://192.168.1.1:8080", got) - - got, err = GetAdminOpsEndpoint(s) - assert.NoError(t, err) - assert.Equal(t, "http://s3.host.com:8080", got) }) t.Run("securePort, no cert", func(t *testing.T) { @@ -820,16 +807,9 @@ func TestGetAdminOpsEndpoint(t *testing.T) { } s.Spec.Gateway.SecurePort = 8443 - // override rand src with known seed to keep tests stable - randSrc = rand.New(rand.NewSource(3)) //nolint:gosec // G404: cryptographically weak RNG is fine here - got, err := GetAdminOpsEndpoint(s) assert.Error(t, err) assert.Equal(t, "", got) - - got, err = GetAdminOpsEndpoint(s) - assert.Error(t, err) - assert.Equal(t, "", got) }) t.Run("securePort", func(t *testing.T) { @@ -841,16 +821,9 @@ func TestGetAdminOpsEndpoint(t *testing.T) { s.Spec.Gateway.SecurePort = 8443 s.Spec.Gateway.SSLCertificateRef = "my-cert" - // override rand src with known seed to keep tests stable - randSrc = rand.New(rand.NewSource(3)) //nolint:gosec // G404: cryptographically weak RNG is fine here - got, err := GetAdminOpsEndpoint(s) assert.NoError(t, err) assert.Equal(t, "https://192.168.1.1:8443", got) - - got, err = GetAdminOpsEndpoint(s) - assert.NoError(t, err) - assert.Equal(t, "https://s3.host.com:8443", got) }) t.Run("port + securePort", func(t *testing.T) { @@ -863,16 +836,31 @@ func TestGetAdminOpsEndpoint(t *testing.T) { s.Spec.Gateway.SecurePort = 8443 s.Spec.Gateway.SSLCertificateRef = "my-cert" - // override rand src with known seed to keep tests stable - randSrc = rand.New(rand.NewSource(3)) //nolint:gosec // G404: cryptographically weak RNG is fine here - got, err := GetAdminOpsEndpoint(s) assert.NoError(t, err) assert.Equal(t, "https://192.168.1.1:8443", got) + }) + }) - got, err = GetAdminOpsEndpoint(s) + t.Run("advertise", func(t *testing.T) { + t.Run("port + securePort", func(t *testing.T) { + s := s.DeepCopy() + s.Spec.Gateway.ExternalRgwEndpoints = []cephv1.EndpointAddress{ + {IP: "192.168.1.1"}, + {Hostname: "s3.host.com"}, + } + s.Spec.Gateway.Port = 8080 + s.Spec.Gateway.SecurePort = 8443 + s.Spec.Gateway.SSLCertificateRef = "my-cert" + s.Spec.Hosting.AdvertiseEndpoint = &cephv1.ObjectEndpointSpec{ + DnsName: "advertise.me", + Port: 80, + UseTls: false, + } + + got, err := GetAdminOpsEndpoint(s) assert.NoError(t, err) - assert.Equal(t, "https://s3.host.com:8443", got) + assert.Equal(t, "http://advertise.me:80", got) }) }) } From d1351d11c802c34e05462c55a370e485c0c6b37e Mon Sep 17 00:00:00 2001 From: Rakshith R Date: Wed, 7 Aug 2024 12:47:56 +0530 Subject: [PATCH 6/7] csi: add pvc & pod yamls for block volume mode This commit adds example yamls for block volume mode. Signed-off-by: Rakshith R (cherry picked from commit 53e2f8c67f6c40afe2bd94a4d2fa8d574a9838b0) --- deploy/examples/csi/rbd/raw-block-pod.yaml | 17 +++++++++++++++++ deploy/examples/csi/rbd/raw-block-pvc.yaml | 13 +++++++++++++ 2 files changed, 30 insertions(+) create mode 100644 deploy/examples/csi/rbd/raw-block-pod.yaml create mode 100644 deploy/examples/csi/rbd/raw-block-pvc.yaml diff --git a/deploy/examples/csi/rbd/raw-block-pod.yaml b/deploy/examples/csi/rbd/raw-block-pod.yaml new file mode 100644 index 000000000000..f5b5a2142c9b --- /dev/null +++ b/deploy/examples/csi/rbd/raw-block-pod.yaml @@ -0,0 +1,17 @@ +--- +apiVersion: v1 +kind: Pod +metadata: + name: csirbd-block-demo-pod +spec: + containers: + - name: centos + image: quay.io/centos/centos:latest + command: ["/bin/sleep", "infinity"] + volumeDevices: + - name: mypvc + devicePath: /dev/xvda + volumes: + - name: mypvc + persistentVolumeClaim: + claimName: raw-block-rbd-pvc diff --git a/deploy/examples/csi/rbd/raw-block-pvc.yaml b/deploy/examples/csi/rbd/raw-block-pvc.yaml new file mode 100644 index 000000000000..4f38ecff2716 --- /dev/null +++ b/deploy/examples/csi/rbd/raw-block-pvc.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: raw-block-rbd-pvc +spec: + accessModes: + - ReadWriteOnce + volumeMode: Block + resources: + requests: + storage: 1Gi + storageClassName: rook-ceph-block From 6aee0855fa743263edd0c3f7c9d34447f8784626 Mon Sep 17 00:00:00 2001 From: Blaine Gardner Date: Thu, 8 Aug 2024 11:39:29 -0600 Subject: [PATCH 7/7] csv: update csv files Update CSV files manually. CSV generation doesn't work on ARM mac at the moment. Signed-off-by: Blaine Gardner --- .../ceph/ceph.rook.io_cephobjectstores.yaml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/build/csv/ceph/ceph.rook.io_cephobjectstores.yaml b/build/csv/ceph/ceph.rook.io_cephobjectstores.yaml index df27711b0ab7..f65dff5a05ed 100644 --- a/build/csv/ceph/ceph.rook.io_cephobjectstores.yaml +++ b/build/csv/ceph/ceph.rook.io_cephobjectstores.yaml @@ -959,7 +959,26 @@ spec: type: object type: object hosting: + nullable: true properties: + advertiseEndpoint: + nullable: true + properties: + dnsName: + minLength: 1 + type: string + port: + format: int32 + maximum: 65535 + minimum: 1 + type: integer + useTls: + type: boolean + required: + - dnsName + - port + - useTls + type: object dnsNames: items: type: string