Isolate etcd and Milvus credentials and management endpoints on 0.9.0

.github/workflows/check.yml

@@ -64,6 +64,8 @@ jobs:
         run: bash .github/cisetup.sh
       - name: Install
         run: task setup
+      - name: Security regression tests
+        run: poetry run python3 -m unittest discover -s tests -p 'test_*.py' -v
       - name: BuildAndLoad
         run: task build-and-load
       - name: AddIngress
@@ -74,3 +76,22 @@ jobs:
       - name: Integration Test
         run: task itest
         continue-on-error: false
+      - name: Reject new integration failures
+        run: |
+          # The legacy runner writes failures without returning nonzero.
+          # These two tests also fail on unmodified 0.9.0 (run 38893750250).
+          unexpected_failures=0
+          if [ -f _failed.txt ]; then
+            while IFS= read -r failed_test; do
+              case "$failed_test" in
+                tests/kind/ferretdb_test.ipy|tests/kind/runtimes_preloader_test.ipy)
+                  echo "::warning::Pre-existing 0.9.0 integration failure: $failed_test"
+                  ;;
+                *)
+                  echo "::error::Integration regression: $failed_test"
+                  unexpected_failures=1
+                  ;;
+              esac
+            done < _failed.txt
+          fi
+          exit "$unexpected_failures"

deploy/etcd/etcd-sts.yaml

@@ -152,7 +152,7 @@ spec:
       - name: etcd-jwt-token
         secret:
           defaultMode: 256
-          secretName: openserverless-etcd-jwt-token              
+          secretName: openserverless-etcd-jwt-token-v2
   volumeClaimTemplates:
     - metadata:
         name: openserverless-etc-data

deploy/milvus-slim/milvus-cfg-slim-base.yaml

@@ -17,11 +17,12 @@
 #
 ---
 apiVersion: v1
-kind: ConfigMap
+kind: Secret
 metadata:
   name: openserverless-milvus
   namespace: openserverless
-data:
+type: Opaque
+stringData:
   default.yaml: |+
     # Copyright (C) 2019-2021 Zilliz. All rights reserved.
     #
@@ -38,7 +39,7 @@ data:
       rootPath: milvus
       auth:
         enabled: true
-        userName: root 
+        userName: etcdmilvus
         password: 0therPa55
       endpoints:
       - openserverless-etcd-0.openserverless-etcd-headless.openserverless.svc.cluster.local:2379    

deploy/milvus-slim/milvus-svc.yaml

@@ -26,17 +26,15 @@ metadata:
     app.kubernetes.io/instance: openserverless-milvus
     component: "standalone"
 spec:
-  type: NodePort
+  type: ClusterIP
   ports:
     - name: milvus
       port: 19530
-      nodePort: 32530
       protocol: TCP
       targetPort: milvus
     - name: metrics
       protocol: TCP
       port: 9091
-      nodePort: 32531
       targetPort: metrics
   selector:
     app.kubernetes.io/name: milvus

deploy/milvus-slim/milvus.yaml

@@ -101,8 +101,9 @@ spec:
       - emptyDir: {}
         name: tools
       - name: milvus-config
-        configMap:
-          name: openserverless-milvus
+        secret:
+          secretName: openserverless-milvus
+          defaultMode: 0440
       - name: milvus-data-disk
         persistentVolumeClaim:
           claimName: openserverless-milvus

deploy/milvus/milvus-cfg-base.yaml

@@ -17,11 +17,12 @@
 #
 ---
 apiVersion: v1
-kind: ConfigMap
+kind: Secret
 metadata:
   name: openserverless-milvus
   namespace: openserverless
-data:
+type: Opaque
+stringData:
   default.yaml: |+
     # Copyright (C) 2019-2021 Zilliz. All rights reserved.
     #
@@ -38,7 +39,7 @@ data:
       rootPath: milvus
       auth:
         enabled: true
-        userName: root 
+        userName: etcdmilvus
         password: 0therPa55
       endpoints:
       - openserverless-etcd-0.openserverless-etcd-headless.openserverless.svc.cluster.local:2379    

deploy/milvus/milvus-svc.yaml

@@ -69,17 +69,15 @@ metadata:
     app.kubernetes.io/instance: openserverless-milvus
     component: "standalone"
 spec:
-  type: NodePort
+  type: ClusterIP
   ports:
     - name: milvus
       port: 19530
-      nodePort: 32530
       protocol: TCP
       targetPort: milvus
     - name: metrics
       protocol: TCP
       port: 9091
-      nodePort: 32531
       targetPort: metrics
   selector:
     app.kubernetes.io/name: milvus

deploy/milvus/milvus.yaml

@@ -101,8 +101,9 @@ spec:
       - emptyDir: {}
         name: tools
       - name: milvus-config
-        configMap:
-          name: openserverless-milvus
+        secret:
+          secretName: openserverless-milvus
+          defaultMode: 0440
       - name: milvus-data-disk
         persistentVolumeClaim:
           claimName: openserverless-milvus

deploy/openserverless-permissions/operator-roles.yaml

@@ -101,6 +101,11 @@ rules:
   resources: ["milvus","milvus/status","milvus/finalizers"]
   verbs: ["get","patch","list","update","watch","create","delete"]
 
+# Datastore ingress isolation is reconciled by the operator.
+- apiGroups: ["networking.k8s.io"]
+  resources: ["networkpolicies"]
+  verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
+
 ---
 kind: RoleBinding
 apiVersion: rbac.authorization.k8s.io/v1

openserverless/config.py

@@ -259,4 +259,4 @@ def detect():
 def dump_config():
     import openserverless.config as cfg
     for k in cfg.getall():
-        logging.debug(f"{k} = {cfg.get(k)}") 
+        logging.debug("configuration key present: %s", k)

openserverless/datastore_connection.py

@@ -0,0 +1,69 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+"""Reach a private service in-cluster or through an authenticated local tunnel."""
+import contextlib
+import os
+import re
+import selectors
+import subprocess
+import time
+
+
+@contextlib.contextmanager
+def service_endpoint(service, port, namespace='openserverless'):
+    if not re.fullmatch(r'[a-z0-9][a-z0-9.-]*', service) or not re.fullmatch(r'[a-z0-9][a-z0-9-]*', namespace):
+        raise ValueError('Invalid datastore service')
+    if not isinstance(port, int) or not 0 < port < 65536:
+        raise ValueError('Invalid datastore port')
+    if os.environ.get('KUBERNETES_SERVICE_HOST'):
+        yield service, port
+        return
+
+    # Use the current kubeconfig and namespace, as with the operator's other
+    # kubectl calls. Never expose a public listener or put credentials in argv.
+    process = subprocess.Popen(
+        ['kubectl', '-n', namespace, 'port-forward', '--address=127.0.0.1',
+         'service/' + service, ':' + str(port)],
+        stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
+    try:
+        with selectors.DefaultSelector() as selector:
+            selector.register(process.stdout, selectors.EVENT_READ)
+            deadline = time.monotonic() + 30
+            while time.monotonic() < deadline:
+                if selector.select(timeout=min(1, max(0, deadline-time.monotonic()))):
+                    line = process.stdout.readline()
+                    match = re.fullmatch(r'Forwarding from 127\.0\.0\.1:(\d+) -> ' + str(port), line.strip())
+                    if match:
+                        local_port = int(match.group(1))
+                        if not 0 < local_port < 65536:
+                            break
+                        yield '127.0.0.1', local_port
+                        return
+                if process.poll() is not None:
+                    break
+        raise RuntimeError('Could not open the private datastore tunnel')
+    finally:
+        if process.poll() is None:
+            process.terminate()
+            try:
+                process.wait(timeout=5)
+            except subprocess.TimeoutExpired:
+                process.kill()
+                process.wait(timeout=5)
+        process.stdout.close()

openserverless/datastore_security.py

@@ -0,0 +1,108 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Private application of datastore credentials and narrow ingress policies."""
+import base64
+import json
+import subprocess
+
+import openserverless.kube as kube
+
+
+def apply_private(spec, namespace='openserverless'):
+    # The generic wrapper includes stdin and stderr in error logs.
+    result = subprocess.run(['kubectl', '-n', namespace, 'apply', '-f', '-'],
+                            input=json.dumps(spec), text=True, capture_output=True,
+                            timeout=60)
+    if result.returncode:
+        raise RuntimeError('Could not apply private datastore resources')
+
+
+def apply(spec):
+    items = spec.get('items', [spec])
+    private = [item for item in items if item['kind'] == 'Secret']
+    public = [item for item in items if item['kind'] != 'Secret']
+    if private:
+        apply_private({'apiVersion': 'v1', 'kind': 'List', 'items': private})
+    return kube.apply({'apiVersion': 'v1', 'kind': 'List', 'items': public})
+
+
+def ensure_jwt_key(owner=None):
+    """Keep a per-installation key; never reuse the legacy repository key."""
+    name = 'openserverless-etcd-jwt-token-v2'
+    raw = kube.kubectl('get', 'secret', name, '--ignore-not-found', '-o', 'json',
+                       debugresult=False)
+    if raw.strip():
+        key = base64.b64decode(json.loads(raw)['data']['jwt-token.pem'], validate=True)
+        check = subprocess.run(['openssl', 'rsa', '-check', '-noout'], input=key,
+                               capture_output=True, timeout=30)
+        if check.returncode:
+            raise RuntimeError('Invalid etcd signing key; refusing implicit rotation')
+        return name
+    # A missing key on an already patched cluster is a security state error.
+    current = kube.kubectl('get', 'sts', 'openserverless-etcd', '--ignore-not-found',
+                           '-o', 'json', debugresult=False)
+    if current.strip():
+        volumes = json.loads(current)['spec']['template']['spec'].get('volumes', [])
+        if any(v.get('secret', {}).get('secretName') == name for v in volumes):
+            raise RuntimeError('Missing etcd signing key; refusing implicit rotation')
+    result = subprocess.run(['openssl', 'genpkey', '-algorithm', 'RSA',
+                             '-pkeyopt', 'rsa_keygen_bits:3072'],
+                            capture_output=True, timeout=30)
+    if result.returncode:
+        raise RuntimeError('Could not generate etcd signing key')
+    resource = {'apiVersion': 'v1', 'kind': 'Secret', 'type': 'Opaque',
+                'metadata': {'name': name, 'namespace': 'openserverless'},
+                'data': {'jwt-token.pem': base64.b64encode(result.stdout).decode()}}
+    if owner:
+        import kopf
+        kopf.append_owner_reference([resource], owner)
+    apply_private(resource)
+    return name
+
+
+def policy(component):
+    if component == 'etcd':
+        selector = {'app.kubernetes.io/name': 'openserverless-etcd'}
+        ingress = [
+            {'from': [{'podSelector': {'matchLabels': selector}}],
+             'ports': [{'port': 2379, 'protocol': 'TCP'}, {'port': 2380, 'protocol': 'TCP'}]},
+            {'from': [{'podSelector': {'matchLabels': {'name': 'openserverless-operator'}}},
+                      {'podSelector': {'matchLabels': {'name': 'openserverless-milvus'}}}],
+             'ports': [{'port': 2379, 'protocol': 'TCP'}]}]
+    elif component == 'milvus':
+        selector = {'name': 'openserverless-milvus'}
+        ingress = [
+            {'ports': [{'port': 19530, 'protocol': 'TCP'}]},
+            {'from': [{'podSelector': {'matchLabels': {
+                'app.kubernetes.io/name': 'prometheus',
+                'app.kubernetes.io/instance': 'openserverless-prometheus'}}},
+                      {'podSelector': {'matchLabels': {'name': 'openserverless-operator'}}}],
+             'ports': [{'port': 9091, 'protocol': 'TCP'}]}]
+    else:
+        raise ValueError('Unknown datastore component')
+    return {'apiVersion': 'networking.k8s.io/v1', 'kind': 'NetworkPolicy',
+            'metadata': {'name': 'openserverless-'+component+'-isolation', 'namespace': 'openserverless'},
+            'spec': {'podSelector': {'matchLabels': selector},
+                     'policyTypes': ['Ingress'], 'ingress': ingress}}
+
+
+def finish_milvus_reconcile():
+    kube.kubectl('rollout', 'status', 'deployment/openserverless-milvus-standalone', '--timeout=300s')
+    # Remove the old ConfigMap AND its last-applied annotation by deleting it.
+    # Its contents include S3 and Milvus admin credentials as well as etcd root.
+    kube.kubectl('delete', 'configmap', 'openserverless-milvus', '--ignore-not-found')