diff --git a/.github/workflows/gevals.yaml b/.github/workflows/gevals.yaml index 1115d323d..a6bf9dfe1 100644 --- a/.github/workflows/gevals.yaml +++ b/.github/workflows/gevals.yaml @@ -54,6 +54,7 @@ jobs: contains(github.event.comment.body, '/run-gevals')) outputs: should-run: ${{ steps.check.outputs.should-run }} + kiali-run: ${{ steps.check.outputs.kiali-run }} pr-number: ${{ steps.check.outputs.pr-number }} pr-ref: ${{ steps.check.outputs.pr-ref }} steps: @@ -78,6 +79,12 @@ jobs: echo "should-run=true" >> $GITHUB_OUTPUT echo "pr-ref=${{ github.ref }}" >> $GITHUB_OUTPUT fi + TASK_FILTER="${{ github.event.inputs.task-filter || '' }}" + if [[ "$TASK_FILTER" =~ kiali ]]; then + echo "kiali-run=true" >> $GITHUB_OUTPUT + else + echo "kiali-run=false" >> $GITHUB_OUTPUT + fi # Run gevals evaluation with Kind cluster run-evaluation: @@ -99,8 +106,14 @@ jobs: - name: Setup Kind cluster run: make kind-create-cluster KIND_CLUSTER_NAME=${{ env.KIND_CLUSTER_NAME }} + - name: Install Istio/Kiali and bookinfo demo + if: needs.check-trigger.outputs.kiali-run == 'true' + run: make setup-kiali + - name: Start MCP server run: make run-server + env: + TOOLSETS: ${{ needs.check-trigger.outputs.kiali-run == 'true' && 'kiali' || '' }} - name: Run gevals evaluation id: gevals diff --git a/build/gevals.mk b/build/gevals.mk index f36ec15cc..efc14b29b 100644 --- a/build/gevals.mk +++ b/build/gevals.mk @@ -7,7 +7,11 @@ MCP_HEALTH_INTERVAL ?= 2 .PHONY: run-server run-server: build ## Start MCP server in background and wait for health check @echo "Starting MCP server on port $(MCP_PORT)..." - @./$(BINARY_NAME) --port $(MCP_PORT) & echo $$! > .mcp-server.pid + @if [ -n "$(TOOLSETS)" ]; then \ + ./$(BINARY_NAME) --port $(MCP_PORT) --toolsets $(TOOLSETS) & echo $$! > .mcp-server.pid; \ + else \ + ./$(BINARY_NAME) --port $(MCP_PORT) & echo $$! > .mcp-server.pid; \ + fi @echo "MCP server started with PID $$(cat .mcp-server.pid)" @echo "Waiting for MCP server to be ready..." @elapsed=0; \ diff --git a/build/kiali.mk b/build/kiali.mk new file mode 100644 index 000000000..71ceef7d9 --- /dev/null +++ b/build/kiali.mk @@ -0,0 +1,48 @@ +# Kind cluster management + +KIND_CLUSTER_NAME ?= kubernetes-mcp-server + +# Detect container engine (docker or podman) +CONTAINER_ENGINE ?= $(shell command -v docker 2>/dev/null || command -v podman 2>/dev/null) + +##@ Istio + +ISTIOCTL = _output/bin/istioctl + +$(ISTIOCTL): + @mkdir -p _output/bin + @echo "Downloading istioctl..." + @set -e; \ + TMPDIR=$$(mktemp -d); \ + cd $$TMPDIR; \ + curl -sL https://istio.io/downloadIstio | sh -; \ + ISTIODIR=$$(ls -d istio-* | head -n1); \ + cp $$ISTIODIR/bin/istioctl $(PWD)/$(ISTIOCTL); \ + cd - >/dev/null; \ + rm -rf $$TMPDIR; \ + echo "istioctl installed at $(ISTIOCTL)" + +.PHONY: istioctl +istioctl: $(ISTIOCTL) ## Ensure istioctl is installed to _output/bin/ + +.PHONY: install-istio +install-istio: istioctl ## Install Istio (demo profile) and enable sidecar injection in default ns + ./$(ISTIOCTL) install --set profile=demo -y + kubectl label namespace default istio-injection=enabled --overwrite + +.PHONY: install-istio-addons +install-istio-addons: install-istio ## Install Istio addons + kubectl apply -f dev/config/istio/prometheus.yaml -n istio-system + kubectl apply -f dev/config/istio/kiali.yaml -n istio-system + kubectl wait --namespace istio-system --for=condition=available deployment/kiali --timeout=300s + kubectl wait --namespace istio-system --for=condition=available deployment/prometheus --timeout=300s + +.PHONY: install-bookinfo-demo +install-bookinfo-demo: ## Install Bookinfo demo + kubectl create ns bookinfo + kubectl label namespace bookinfo istio-discovery=enabled istio.io/rev=default istio-injection=enabled + kubectl apply -f dev/config/istio/bookinfo.yaml -n bookinfo + kubectl wait --for=condition=Ready pod --all -n bookinfo --timeout=300s + +.PHONY: setup-kiali +setup-kiali: install-istio-addons install-bookinfo-demo ## Setup Kiali \ No newline at end of file diff --git a/dev/config/istio/bookinfo.yaml b/dev/config/istio/bookinfo.yaml new file mode 100644 index 000000000..b878ed5d0 --- /dev/null +++ b/dev/config/istio/bookinfo.yaml @@ -0,0 +1,335 @@ +# Copyright Istio Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +################################################################################################## +# This file defines the services, service accounts, and deployments for the Bookinfo sample. +# +# To apply all 4 Bookinfo services, their corresponding service accounts, and deployments: +# +# kubectl apply -f samples/bookinfo/platform/kube/bookinfo.yaml +# +# Alternatively, you can deploy any resource separately: +# +# kubectl apply -f samples/bookinfo/platform/kube/bookinfo.yaml -l service=reviews # reviews Service +# kubectl apply -f samples/bookinfo/platform/kube/bookinfo.yaml -l account=reviews # reviews ServiceAccount +# kubectl apply -f samples/bookinfo/platform/kube/bookinfo.yaml -l app=reviews,version=v3 # reviews-v3 Deployment +################################################################################################## + +################################################################################################## +# Details service +################################################################################################## +apiVersion: v1 +kind: Service +metadata: + name: details + labels: + app: details + service: details +spec: + ports: + - port: 9080 + name: http + selector: + app: details +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: bookinfo-details + labels: + account: details +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: details-v1 + labels: + app: details + version: v1 +spec: + replicas: 1 + selector: + matchLabels: + app: details + version: v1 + template: + metadata: + labels: + app: details + version: v1 + spec: + serviceAccountName: bookinfo-details + containers: + - name: details + image: docker.io/istio/examples-bookinfo-details-v1:1.20.3 + imagePullPolicy: IfNotPresent + ports: + - containerPort: 9080 +--- +################################################################################################## +# Ratings service +################################################################################################## +apiVersion: v1 +kind: Service +metadata: + name: ratings + labels: + app: ratings + service: ratings +spec: + ports: + - port: 9080 + name: http + selector: + app: ratings +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: bookinfo-ratings + labels: + account: ratings +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ratings-v1 + labels: + app: ratings + version: v1 +spec: + replicas: 1 + selector: + matchLabels: + app: ratings + version: v1 + template: + metadata: + labels: + app: ratings + version: v1 + spec: + serviceAccountName: bookinfo-ratings + containers: + - name: ratings + image: docker.io/istio/examples-bookinfo-ratings-v1:1.20.3 + imagePullPolicy: IfNotPresent + ports: + - containerPort: 9080 +--- +################################################################################################## +# Reviews service +################################################################################################## +apiVersion: v1 +kind: Service +metadata: + name: reviews + labels: + app: reviews + service: reviews +spec: + ports: + - port: 9080 + name: http + selector: + app: reviews +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: bookinfo-reviews + labels: + account: reviews +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: reviews-v1 + labels: + app: reviews + version: v1 +spec: + replicas: 1 + selector: + matchLabels: + app: reviews + version: v1 + template: + metadata: + labels: + app: reviews + version: v1 + spec: + serviceAccountName: bookinfo-reviews + containers: + - name: reviews + image: docker.io/istio/examples-bookinfo-reviews-v1:1.20.3 + imagePullPolicy: IfNotPresent + env: + - name: LOG_DIR + value: "/tmp/logs" + ports: + - containerPort: 9080 + volumeMounts: + - name: tmp + mountPath: /tmp + - name: wlp-output + mountPath: /opt/ibm/wlp/output + volumes: + - name: wlp-output + emptyDir: {} + - name: tmp + emptyDir: {} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: reviews-v2 + labels: + app: reviews + version: v2 +spec: + replicas: 1 + selector: + matchLabels: + app: reviews + version: v2 + template: + metadata: + labels: + app: reviews + version: v2 + spec: + serviceAccountName: bookinfo-reviews + containers: + - name: reviews + image: docker.io/istio/examples-bookinfo-reviews-v2:1.20.3 + imagePullPolicy: IfNotPresent + env: + - name: LOG_DIR + value: "/tmp/logs" + ports: + - containerPort: 9080 + volumeMounts: + - name: tmp + mountPath: /tmp + - name: wlp-output + mountPath: /opt/ibm/wlp/output + volumes: + - name: wlp-output + emptyDir: {} + - name: tmp + emptyDir: {} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: reviews-v3 + labels: + app: reviews + version: v3 +spec: + replicas: 1 + selector: + matchLabels: + app: reviews + version: v3 + template: + metadata: + labels: + app: reviews + version: v3 + spec: + serviceAccountName: bookinfo-reviews + containers: + - name: reviews + image: docker.io/istio/examples-bookinfo-reviews-v3:1.20.3 + imagePullPolicy: IfNotPresent + env: + - name: LOG_DIR + value: "/tmp/logs" + ports: + - containerPort: 9080 + volumeMounts: + - name: tmp + mountPath: /tmp + - name: wlp-output + mountPath: /opt/ibm/wlp/output + volumes: + - name: wlp-output + emptyDir: {} + - name: tmp + emptyDir: {} +--- +################################################################################################## +# Productpage services +################################################################################################## +apiVersion: v1 +kind: Service +metadata: + name: productpage + labels: + app: productpage + service: productpage +spec: + ports: + - port: 9080 + name: http + selector: + app: productpage +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: bookinfo-productpage + labels: + account: productpage +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: productpage-v1 + labels: + app: productpage + version: v1 +spec: + replicas: 1 + selector: + matchLabels: + app: productpage + version: v1 + template: + metadata: + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9080" + prometheus.io/path: "/metrics" + labels: + app: productpage + version: v1 + spec: + serviceAccountName: bookinfo-productpage + containers: + - name: productpage + image: docker.io/istio/examples-bookinfo-productpage-v1:1.20.3 + imagePullPolicy: IfNotPresent + ports: + - containerPort: 9080 + volumeMounts: + - name: tmp + mountPath: /tmp + volumes: + - name: tmp + emptyDir: {} +--- \ No newline at end of file diff --git a/dev/config/istio/kiali.yaml b/dev/config/istio/kiali.yaml new file mode 100644 index 000000000..3b8e35a92 --- /dev/null +++ b/dev/config/istio/kiali.yaml @@ -0,0 +1,456 @@ +--- +# Source: kiali-server/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kiali + namespace: "istio-system" + labels: + + helm.sh/chart: kiali-server-2.17.0 + app: kiali + app.kubernetes.io/name: kiali + app.kubernetes.io/instance: kiali + version: "v2.17.0" + app.kubernetes.io/version: "v2.17.0" + app.kubernetes.io/part-of: "kiali" +... +--- +# Source: kiali-server/templates/configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: kiali + namespace: "istio-system" + labels: + + helm.sh/chart: kiali-server-2.17.0 + app: kiali + app.kubernetes.io/name: kiali + app.kubernetes.io/instance: kiali + version: "v2.17.0" + app.kubernetes.io/version: "v2.17.0" + app.kubernetes.io/part-of: "kiali" +data: + config.yaml: | + additional_display_details: + - annotation: kiali.io/api-spec + icon_annotation: kiali.io/api-type + title: API Documentation + auth: + openid: {} + openshift: + client_id_prefix: kiali + strategy: anonymous + clustering: + autodetect_secrets: + enabled: true + label: kiali.io/multiCluster=true + clusters: [] + deployment: + additional_service_yaml: {} + affinity: + node: {} + pod: {} + pod_anti: {} + cluster_wide_access: true + configmap_annotations: {} + custom_envs: [] + custom_secrets: [] + dns: + config: {} + policy: "" + extra_labels: {} + host_aliases: [] + hpa: + api_version: autoscaling/v2 + spec: {} + image_digest: "" + image_name: quay.io/kiali/kiali + image_pull_policy: IfNotPresent + image_pull_secrets: [] + image_version: v2.17 + ingress: + additional_labels: {} + class_name: nginx + override_yaml: + metadata: {} + ingress_enabled: false + instance_name: kiali + logger: + log_format: text + log_level: info + sampler_rate: "1" + time_field_format: 2006-01-02T15:04:05Z07:00 + namespace: istio-system + network_policy: + enabled: false + node_selector: {} + pod_annotations: + proxy.istio.io/config: '{ "holdApplicationUntilProxyStarts": true }' + pod_labels: + sidecar.istio.io/inject: "false" + priority_class_name: "" + probes: + liveness: + initial_delay_seconds: 5 + period_seconds: 30 + readiness: + initial_delay_seconds: 5 + period_seconds: 30 + startup: + failure_threshold: 6 + initial_delay_seconds: 30 + period_seconds: 10 + remote_cluster_resources_only: false + replicas: 1 + resources: + limits: + memory: 1Gi + requests: + cpu: 10m + memory: 64Mi + secret_name: kiali + security_context: {} + service_annotations: {} + service_type: "" + tolerations: [] + topology_spread_constraints: [] + version_label: v2.17.0 + view_only_mode: false + external_services: + custom_dashboards: + enabled: true + istio: + root_namespace: istio-system + tracing: + enabled: false + identity: + cert_file: "" + private_key_file: "" + kiali_feature_flags: + custom_workload_types: [] + disabled_features: [] + validations: + ignore: + - KIA1301 + login_token: + signing_key: CHANGEME00000000 + server: + observability: + metrics: + enabled: true + port: 9090 + port: 20001 + web_root: /kiali + skipResources: [] +... +--- +# Source: kiali-server/templates/role.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kiali + labels: + + helm.sh/chart: kiali-server-2.17.0 + app: kiali + app.kubernetes.io/name: kiali + app.kubernetes.io/instance: kiali + version: "v2.17.0" + app.kubernetes.io/version: "v2.17.0" + app.kubernetes.io/part-of: "kiali" +rules: +- apiGroups: [""] + resources: + - configmaps + - endpoints + - pods/log + verbs: + - get + - list + - watch +- apiGroups: [""] + resources: + - namespaces + - pods + - replicationcontrollers + - services + verbs: + - get + - list + - watch + - patch +- apiGroups: [""] + resources: + - pods/portforward + verbs: + - create + - post +- apiGroups: ["extensions", "apps"] + resources: + - daemonsets + - deployments + - replicasets + - statefulsets + verbs: + - get + - list + - watch + - patch +- apiGroups: ["batch"] + resources: + - cronjobs + - jobs + verbs: + - get + - list + - watch + - patch +- apiGroups: + - networking.istio.io + - security.istio.io + - extensions.istio.io + - telemetry.istio.io + - gateway.networking.k8s.io + - inference.networking.k8s.io + resources: ["*"] + verbs: + - get + - list + - watch + - create + - delete + - patch +- apiGroups: ["apps.openshift.io"] + resources: + - deploymentconfigs + verbs: + - get + - list + - watch + - patch +- apiGroups: ["route.openshift.io"] + resources: + - routes + verbs: + - get +- apiGroups: ["authentication.k8s.io"] + resources: + - tokenreviews + verbs: + - create +- apiGroups: ["oauth.openshift.io"] + resources: + - oauthclients + resourceNames: + - kiali-istio-system + verbs: + - get +- apiGroups: ["admissionregistration.k8s.io"] + resources: + - mutatingwebhookconfigurations + verbs: + - get + - list + - watch +... +--- +# Source: kiali-server/templates/rolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kiali + labels: + + helm.sh/chart: kiali-server-2.17.0 + app: kiali + app.kubernetes.io/name: kiali + app.kubernetes.io/instance: kiali + version: "v2.17.0" + app.kubernetes.io/version: "v2.17.0" + app.kubernetes.io/part-of: "kiali" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kiali +subjects: +- kind: ServiceAccount + name: kiali + namespace: "istio-system" +... +--- +# Source: kiali-server/templates/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: kiali + namespace: "istio-system" + labels: + + helm.sh/chart: kiali-server-2.17.0 + app: kiali + app.kubernetes.io/name: kiali + app.kubernetes.io/instance: kiali + version: "v2.17.0" + app.kubernetes.io/version: "v2.17.0" + app.kubernetes.io/part-of: "kiali" + annotations: +spec: + ports: + - name: http + appProtocol: http + protocol: TCP + port: 20001 + - name: http-metrics + appProtocol: http + protocol: TCP + port: 9090 + selector: + app.kubernetes.io/name: kiali + app.kubernetes.io/instance: kiali +... +--- +# Source: kiali-server/templates/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kiali + namespace: "istio-system" + labels: + + helm.sh/chart: kiali-server-2.17.0 + app: kiali + app.kubernetes.io/name: kiali + app.kubernetes.io/instance: kiali + version: "v2.17.0" + app.kubernetes.io/version: "v2.17.0" + app.kubernetes.io/part-of: "kiali" +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: kiali + app.kubernetes.io/instance: kiali + strategy: + rollingUpdate: + maxSurge: 1 + maxUnavailable: 1 + type: RollingUpdate + template: + metadata: + name: kiali + labels: + + helm.sh/chart: kiali-server-2.17.0 + app: kiali + app.kubernetes.io/name: kiali + app.kubernetes.io/instance: kiali + version: "v2.17.0" + app.kubernetes.io/version: "v2.17.0" + app.kubernetes.io/part-of: "kiali" + sidecar.istio.io/inject: "false" + annotations: + checksum/config: 5129658cb79b9fbbb9b7745ea8ff77c611538e6ce760b6ad5554d7c85a6b79c1 + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + kiali.io/dashboards: go,kiali + proxy.istio.io/config: '{ "holdApplicationUntilProxyStarts": true }' + spec: + serviceAccountName: kiali + containers: + - image: "quay.io/kiali/kiali:v2.17" + imagePullPolicy: IfNotPresent + name: kiali + command: + - "/opt/kiali/kiali" + - "-config" + - "/kiali-configuration/config.yaml" + terminationMessagePolicy: FallbackToLogsOnError + securityContext: + allowPrivilegeEscalation: false + privileged: false + readOnlyRootFilesystem: true + runAsNonRoot: true + capabilities: + drop: + - ALL + ports: + - name: api-port + containerPort: 20001 + - name: http-metrics + containerPort: 9090 + readinessProbe: + httpGet: + path: /kiali/healthz + port: api-port + scheme: HTTP + initialDelaySeconds: 5 + periodSeconds: 30 + livenessProbe: + httpGet: + path: /kiali/healthz + port: api-port + scheme: HTTP + initialDelaySeconds: 5 + periodSeconds: 30 + startupProbe: + httpGet: + path: /kiali/healthz + port: api-port + scheme: HTTP + failureThreshold: 6 + initialDelaySeconds: 30 + periodSeconds: 10 + env: + - name: ACTIVE_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: LOG_LEVEL + value: "info" + - name: LOG_FORMAT + value: "text" + - name: LOG_TIME_FIELD_FORMAT + value: "2006-01-02T15:04:05Z07:00" + - name: LOG_SAMPLER_RATE + value: "1" + volumeMounts: + - name: kiali-configuration + mountPath: "/kiali-configuration" + - name: kiali-cert + mountPath: "/kiali-cert" + - name: kiali-secret + mountPath: "/kiali-secret" + - name: kiali-cabundle + mountPath: "/kiali-cabundle" + - name: "kiali-multi-cluster-secret" + mountPath: "/kiali-remote-cluster-secrets/kiali-multi-cluster-secret" + readOnly: true + resources: + limits: + memory: 1Gi + requests: + cpu: 10m + memory: 64Mi + volumes: + - name: kiali-configuration + configMap: + name: kiali + - name: kiali-cert + secret: + secretName: istio.kiali-service-account + optional: true + - name: kiali-secret + secret: + secretName: kiali + optional: true + - name: kiali-cabundle + configMap: + name: kiali-cabundle + optional: true + - name: "kiali-multi-cluster-secret" + secret: + secretName: "kiali-multi-cluster-secret" + optional: true +... \ No newline at end of file diff --git a/dev/config/istio/prometheus.yaml b/dev/config/istio/prometheus.yaml new file mode 100644 index 000000000..ebfe11a5e --- /dev/null +++ b/dev/config/istio/prometheus.yaml @@ -0,0 +1,559 @@ +--- +# Source: prometheus/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app.kubernetes.io/component: server + app.kubernetes.io/name: prometheus + app.kubernetes.io/instance: prometheus + app.kubernetes.io/version: v3.5.0 + helm.sh/chart: prometheus-27.37.0 + app.kubernetes.io/part-of: prometheus + name: prometheus + namespace: istio-system + annotations: + {} +--- +# Source: prometheus/templates/cm.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + app.kubernetes.io/component: server + app.kubernetes.io/name: prometheus + app.kubernetes.io/instance: prometheus + app.kubernetes.io/version: v3.5.0 + helm.sh/chart: prometheus-27.37.0 + app.kubernetes.io/part-of: prometheus + name: prometheus + namespace: istio-system +data: + allow-snippet-annotations: "false" + alerting_rules.yml: | + {} + alerts: | + {} + prometheus.yml: | + global: + evaluation_interval: 1m + scrape_interval: 15s + scrape_timeout: 10s + rule_files: + - /etc/config/recording_rules.yml + - /etc/config/alerting_rules.yml + - /etc/config/rules + - /etc/config/alerts + scrape_configs: + - job_name: prometheus + static_configs: + - targets: + - localhost:9090 + - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + job_name: kubernetes-apiservers + kubernetes_sd_configs: + - role: endpoints + relabel_configs: + - action: keep + regex: default;kubernetes;https + source_labels: + - __meta_kubernetes_namespace + - __meta_kubernetes_service_name + - __meta_kubernetes_endpoint_port_name + scheme: https + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + job_name: kubernetes-nodes + kubernetes_sd_configs: + - role: node + relabel_configs: + - action: labelmap + regex: __meta_kubernetes_node_label_(.+) + - replacement: kubernetes.default.svc:443 + target_label: __address__ + - regex: (.+) + replacement: /api/v1/nodes/$1/proxy/metrics + source_labels: + - __meta_kubernetes_node_name + target_label: __metrics_path__ + scheme: https + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + job_name: kubernetes-nodes-cadvisor + kubernetes_sd_configs: + - role: node + relabel_configs: + - action: labelmap + regex: __meta_kubernetes_node_label_(.+) + - replacement: kubernetes.default.svc:443 + target_label: __address__ + - regex: (.+) + replacement: /api/v1/nodes/$1/proxy/metrics/cadvisor + source_labels: + - __meta_kubernetes_node_name + target_label: __metrics_path__ + scheme: https + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + - honor_labels: true + job_name: kubernetes-service-endpoints + kubernetes_sd_configs: + - role: endpoints + relabel_configs: + - action: keep + regex: true + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_scrape + - action: drop + regex: true + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_scrape_slow + - action: replace + regex: (https?) + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_scheme + target_label: __scheme__ + - action: replace + regex: (.+) + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_path + target_label: __metrics_path__ + - action: replace + regex: (.+?)(?::\d+)?;(\d+) + replacement: $1:$2 + source_labels: + - __address__ + - __meta_kubernetes_service_annotation_prometheus_io_port + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_service_annotation_prometheus_io_param_(.+) + replacement: __param_$1 + - action: labelmap + regex: __meta_kubernetes_service_label_(.+) + - action: replace + source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - action: replace + source_labels: + - __meta_kubernetes_service_name + target_label: service + - action: replace + source_labels: + - __meta_kubernetes_pod_node_name + target_label: node + - honor_labels: true + job_name: kubernetes-service-endpoints-slow + kubernetes_sd_configs: + - role: endpoints + relabel_configs: + - action: keep + regex: true + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_scrape_slow + - action: replace + regex: (https?) + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_scheme + target_label: __scheme__ + - action: replace + regex: (.+) + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_path + target_label: __metrics_path__ + - action: replace + regex: (.+?)(?::\d+)?;(\d+) + replacement: $1:$2 + source_labels: + - __address__ + - __meta_kubernetes_service_annotation_prometheus_io_port + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_service_annotation_prometheus_io_param_(.+) + replacement: __param_$1 + - action: labelmap + regex: __meta_kubernetes_service_label_(.+) + - action: replace + source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - action: replace + source_labels: + - __meta_kubernetes_service_name + target_label: service + - action: replace + source_labels: + - __meta_kubernetes_pod_node_name + target_label: node + scrape_interval: 5m + scrape_timeout: 30s + - honor_labels: true + job_name: prometheus-pushgateway + kubernetes_sd_configs: + - role: service + relabel_configs: + - action: keep + regex: pushgateway + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_probe + - honor_labels: true + job_name: kubernetes-services + kubernetes_sd_configs: + - role: service + metrics_path: /probe + params: + module: + - http_2xx + relabel_configs: + - action: keep + regex: true + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_probe + - source_labels: + - __address__ + target_label: __param_target + - replacement: blackbox + target_label: __address__ + - source_labels: + - __param_target + target_label: instance + - action: labelmap + regex: __meta_kubernetes_service_label_(.+) + - source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - source_labels: + - __meta_kubernetes_service_name + target_label: service + - honor_labels: true + job_name: kubernetes-pods + kubernetes_sd_configs: + - role: pod + relabel_configs: + - action: keep + regex: true + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_scrape + - action: drop + regex: true + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_scrape_slow + - action: replace + regex: (https?) + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_scheme + target_label: __scheme__ + - action: replace + regex: (.+) + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_path + target_label: __metrics_path__ + - action: replace + regex: (\d+);(([A-Fa-f0-9]{1,4}::?){1,7}[A-Fa-f0-9]{1,4}) + replacement: '[$2]:$1' + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_port + - __meta_kubernetes_pod_ip + target_label: __address__ + - action: replace + regex: (\d+);((([0-9]+?)(\.|$)){4}) + replacement: $2:$1 + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_port + - __meta_kubernetes_pod_ip + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_pod_annotation_prometheus_io_param_(.+) + replacement: __param_$1 + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - action: replace + source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - action: replace + source_labels: + - __meta_kubernetes_pod_name + target_label: pod + - action: drop + regex: Pending|Succeeded|Failed|Completed + source_labels: + - __meta_kubernetes_pod_phase + - action: replace + source_labels: + - __meta_kubernetes_pod_node_name + target_label: node + - honor_labels: true + job_name: kubernetes-pods-slow + kubernetes_sd_configs: + - role: pod + relabel_configs: + - action: keep + regex: true + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_scrape_slow + - action: replace + regex: (https?) + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_scheme + target_label: __scheme__ + - action: replace + regex: (.+) + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_path + target_label: __metrics_path__ + - action: replace + regex: (\d+);(([A-Fa-f0-9]{1,4}::?){1,7}[A-Fa-f0-9]{1,4}) + replacement: '[$2]:$1' + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_port + - __meta_kubernetes_pod_ip + target_label: __address__ + - action: replace + regex: (\d+);((([0-9]+?)(\.|$)){4}) + replacement: $2:$1 + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_port + - __meta_kubernetes_pod_ip + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_pod_annotation_prometheus_io_param_(.+) + replacement: __param_$1 + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - action: replace + source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - action: replace + source_labels: + - __meta_kubernetes_pod_name + target_label: pod + - action: drop + regex: Pending|Succeeded|Failed|Completed + source_labels: + - __meta_kubernetes_pod_phase + - action: replace + source_labels: + - __meta_kubernetes_pod_node_name + target_label: node + scrape_interval: 5m + scrape_timeout: 30s + recording_rules.yml: | + {} + rules: | + {} +--- +# Source: prometheus/templates/clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: server + app.kubernetes.io/name: prometheus + app.kubernetes.io/instance: prometheus + app.kubernetes.io/version: v3.5.0 + helm.sh/chart: prometheus-27.37.0 + app.kubernetes.io/part-of: prometheus + name: prometheus +rules: + - apiGroups: + - "" + resources: + - nodes + - nodes/proxy + - nodes/metrics + - services + - endpoints + - pods + - ingresses + - configmaps + verbs: + - get + - list + - watch + - apiGroups: + - "networking.k8s.io" + resources: + - ingresses/status + - ingresses + verbs: + - get + - list + - watch + - apiGroups: + - "discovery.k8s.io" + resources: + - endpointslices + verbs: + - get + - list + - watch + - nonResourceURLs: + - "/metrics" + verbs: + - get +--- +# Source: prometheus/templates/clusterrolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/component: server + app.kubernetes.io/name: prometheus + app.kubernetes.io/instance: prometheus + app.kubernetes.io/version: v3.5.0 + helm.sh/chart: prometheus-27.37.0 + app.kubernetes.io/part-of: prometheus + name: prometheus +subjects: + - kind: ServiceAccount + name: prometheus + namespace: istio-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: prometheus +--- +# Source: prometheus/templates/service.yaml +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/component: server + app.kubernetes.io/name: prometheus + app.kubernetes.io/instance: prometheus + app.kubernetes.io/version: v3.5.0 + helm.sh/chart: prometheus-27.37.0 + app.kubernetes.io/part-of: prometheus + name: prometheus + namespace: istio-system +spec: + ports: + - name: http + port: 9090 + protocol: TCP + targetPort: 9090 + selector: + app.kubernetes.io/component: server + app.kubernetes.io/name: prometheus + app.kubernetes.io/instance: prometheus + sessionAffinity: None + type: "ClusterIP" +--- +# Source: prometheus/templates/deploy.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app.kubernetes.io/component: server + app.kubernetes.io/name: prometheus + app.kubernetes.io/instance: prometheus + app.kubernetes.io/version: v3.5.0 + helm.sh/chart: prometheus-27.37.0 + app.kubernetes.io/part-of: prometheus + name: prometheus + namespace: istio-system +spec: + strategy: + type: Recreate + rollingUpdate: null + selector: + matchLabels: + app.kubernetes.io/component: server + app.kubernetes.io/name: prometheus + app.kubernetes.io/instance: prometheus + replicas: 1 + revisionHistoryLimit: 10 + template: + metadata: + labels: + app.kubernetes.io/component: server + app.kubernetes.io/name: prometheus + app.kubernetes.io/instance: prometheus + app.kubernetes.io/version: v3.5.0 + helm.sh/chart: prometheus-27.37.0 + app.kubernetes.io/part-of: prometheus + + sidecar.istio.io/inject: "false" + spec: + enableServiceLinks: true + serviceAccountName: prometheus + containers: + - name: prometheus-server-configmap-reload + image: "ghcr.io/prometheus-operator/prometheus-config-reloader:v0.85.0" + imagePullPolicy: "IfNotPresent" + args: + - --watched-dir=/etc/config + - --listen-address=0.0.0.0:8080 + - --reload-url=http://127.0.0.1:9090/-/reload + ports: + - containerPort: 8080 + name: metrics + livenessProbe: + httpGet: + path: /healthz + port: metrics + scheme: HTTP + initialDelaySeconds: 2 + periodSeconds: 10 + readinessProbe: + httpGet: + path: /healthz + port: metrics + scheme: HTTP + periodSeconds: 10 + volumeMounts: + - name: config-volume + mountPath: /etc/config + readOnly: true + + - name: prometheus-server + image: "docker.io/prom/prometheus:v3.5.0" + imagePullPolicy: "IfNotPresent" + args: + - --storage.tsdb.retention.time=15d + - --config.file=/etc/config/prometheus.yml + - --storage.tsdb.path=/data + - --web.console.libraries=/etc/prometheus/console_libraries + - --web.console.templates=/etc/prometheus/consoles + - --web.enable-lifecycle + ports: + - containerPort: 9090 + readinessProbe: + httpGet: + path: /-/ready + port: 9090 + scheme: HTTP + initialDelaySeconds: 0 + periodSeconds: 5 + timeoutSeconds: 4 + failureThreshold: 3 + successThreshold: 1 + livenessProbe: + httpGet: + path: /-/healthy + port: 9090 + scheme: HTTP + initialDelaySeconds: 30 + periodSeconds: 15 + timeoutSeconds: 10 + failureThreshold: 3 + successThreshold: 1 + volumeMounts: + - name: config-volume + mountPath: /etc/config + - name: storage-volume + mountPath: /data + subPath: "" + dnsPolicy: ClusterFirst + terminationGracePeriodSeconds: 300 + volumes: + - name: config-volume + configMap: + name: prometheus + - name: storage-volume + emptyDir: + {} \ No newline at end of file diff --git a/evals/claude-code/eval-inline.yaml b/evals/claude-code/eval-inline.yaml index 4486d2a71..b4c94b1e4 100644 --- a/evals/claude-code/eval-inline.yaml +++ b/evals/claude-code/eval-inline.yaml @@ -12,7 +12,7 @@ config: apiKeyKey: JUDGE_API_KEY modelNameKey: JUDGE_MODEL_NAME taskSets: - - glob: ../tasks/*/*.yaml + - glob: ../tasks/*/*/*.yaml assertions: toolsUsed: - server: kubernetes diff --git a/evals/claude-code/eval.yaml b/evals/claude-code/eval.yaml index a6a720fcc..4ba2b3f5e 100644 --- a/evals/claude-code/eval.yaml +++ b/evals/claude-code/eval.yaml @@ -12,7 +12,7 @@ config: apiKeyKey: JUDGE_API_KEY modelNameKey: JUDGE_MODEL_NAME taskSets: - - glob: ../tasks/*/*.yaml + - glob: ../tasks/*/*/*.yaml assertions: toolsUsed: - server: kubernetes diff --git a/evals/openai-agent/eval-inline.yaml b/evals/openai-agent/eval-inline.yaml index 12b983597..e53f15c77 100644 --- a/evals/openai-agent/eval-inline.yaml +++ b/evals/openai-agent/eval-inline.yaml @@ -16,7 +16,7 @@ config: apiKeyKey: JUDGE_API_KEY modelNameKey: JUDGE_MODEL_NAME taskSets: - - glob: ../tasks/*/*.yaml + - glob: ../tasks/*/*/*.yaml assertions: toolsUsed: - server: kubernetes diff --git a/evals/openai-agent/eval.yaml b/evals/openai-agent/eval.yaml index eef806bdc..4b13aca85 100644 --- a/evals/openai-agent/eval.yaml +++ b/evals/openai-agent/eval.yaml @@ -12,7 +12,7 @@ config: apiKeyKey: JUDGE_API_KEY modelNameKey: JUDGE_MODEL_NAME taskSets: - - glob: ../tasks/*/*.yaml + - glob: ../tasks/*/*/*.yaml assertions: toolsUsed: - server: kubernetes diff --git a/evals/tasks/README.md b/evals/tasks/README.md new file mode 100644 index 000000000..f7d259a25 --- /dev/null +++ b/evals/tasks/README.md @@ -0,0 +1,36 @@ +# MCP Task Library + +This directory hosts the reusable task scenarios that power MCP evaluations for the Kubernetes MCP Server. Each task captures a realistic cluster workflow (setup, agent-driven actions, verification, and cleanup) so different agents can be compared against the same benchmark. + +## Task Families + +- [Kubernetes tasks](kubernetes/) – core cluster workflows such as creating pods, fixing deployments, managing RBAC, or debugging state issues. +- [Kiali tasks](kiali/) – service-mesh and observability workflows that exercise the Kiali MCP toolset (Istio config, topology, mesh health, tracing). + +## Anatomy of a Task + +Every subdirectory under `kubernetes/` or `kiali/` defines a single scenario: + +1. `*.yaml` – declarative description consumed by the evaluation harness (prompts, success criteria, required tools). +2. `setup.sh` / `verify.sh` / `cleanup.sh` – shell hooks (optional) that prime the cluster, assert post-conditions, and reset resources so tasks stay idempotent. +3. `artifacts/` – supporting manifests, scripts, or payloads referenced by the task definition. + +## Adding a New Task + +1. Pick the closest family and create a new subfolder. +2. Author the task YAML referencing MCP tools, expected observations, and any artifacts. +3. Provide helper scripts if the scenario needs deterministic setup or verification. +4. Document nuances in a local `README.md` so future contributors and eval authors can replay the scenario manually. + +Well-scoped, deterministic tasks make it easier to compare agents and regressions over time, so keep inputs minimal, outputs explicit, and always clean up what you create. + +## Adding a New Task Stack + +When a new MCP toolset lands , keep its evaluations isolated by creating a sibling directory under `tasks/` named after the toolset (`tasks/>`, etc.). Populate it with: + +1. A scoped `README.md` describing the toolset focus and prerequisite context. +2. One subfolder per scenario that follows the same layout described above (`*.yaml`, scripts, `artifacts/`). +3. Any shared fixtures the stack needs (place them in a `shared/` subdirectory if multiple scenarios reuse them). + +This structure keeps task stacks discoverable and lets eval harnesses target toolset-specific workflows without mixing concerns from the core Kubernetes or Kiali libraries. + diff --git a/evals/tasks/kiali/Makefile b/evals/tasks/kiali/Makefile new file mode 100644 index 000000000..a691f13fb --- /dev/null +++ b/evals/tasks/kiali/Makefile @@ -0,0 +1,6 @@ +SHELL := /usr/bin/bash + +.PHONY: update-tasks + +update-tasks: + @./scripts/update_tasks.sh \ No newline at end of file diff --git a/evals/tasks/kiali/README.md b/evals/tasks/kiali/README.md new file mode 100644 index 000000000..bb010afe2 --- /dev/null +++ b/evals/tasks/kiali/README.md @@ -0,0 +1,64 @@ +# Kiali Task Stack + +Kiali-focused MCP tasks live here. Each folder under this directory represents a self-contained scenario that exercises the Kiali toolset (Istio config, topology, observability, troubleshooting). + +## Adding a New Task + +1. Create a new subdirectory (e.g., `status-foo/`) and place the scenario YAML plus any helper scripts or artifacts inside it. +2. Make sure the YAML’s `metadata` block includes `name`, `category`, and `difficulty` so it shows up correctly in the catalog below. +3. Keep prompts concise and action-oriented; verification commands should rely on Kiali MCP tools whenever possible. + +## Updating the Catalog + +After adding or editing tasks, regenerate this README’s catalog with: + +```bash +make update_tasks +``` + +The `update_tasks` target runs `scripts/update_tasks.sh`, which parses every scenario and rewrites the section below automatically. Always run it before committing so the list stays in sync. + +## Tasks defined + +- High-Level Observability & Health + - [easy] obs-unhealthy-namespaces (Unhealthy Namespaces) + **Prompt:** *Are there any unhealthy namespaces in my mesh right now?* + - [easy] show-topology (Show topology bookinfo) + **Prompt:** *Show me the topology of the bookinfo namespace.* + - [easy] status-kiali-istio (Status Kiali and Istio) + **Prompt:** *Give me a status report on the interaction between Kiali and Istio components* +- Istio Configuration & Management + - [easy] istio-list (List all VS in bookinfo namespace) + **Prompt:** *List all VirtualServices in the bookinfo namespace and check if they have any validation errors* + - [medium] istio-create (Create a gateway) + **Prompt:** *Create a Gateway named my-gateway in the istio-system namespace.* + - [medium] istio-delete (Remove fault Injection) + **Prompt:** *Fix my namespace bookinfo to remove the fault injection.* + - [medium] istio-patch (Patch my traffic) + **Prompt:** *I need to shift 50% of traffic to v2 of the reviews service. Apply a patch to the existing VirtualService.* +- Resource Inspection + - [easy] resource-get-namespaces (Get mesh namespaces) + **Prompt:** *Check namespaces in my mesh.* + - [easy] resource-get-service-detail (Get service detail) + **Prompt:** *Get the full details and health status for the details service* + - [easy] resource-list-workloads (List workloads without sidecar) + **Prompt:** *List all workloads in the bookinfo namespace that have missing sidecars.* + - [easy] resource-mesh-status (Status of my mesh) + **Prompt:** *Check my mesh.* +- Troubleshooting & Debugging + - [easy] troubleshooting-latency-traces (Get latency workload) + **Prompt:** *Analyze the latency for the reviews workload over the last 30 minutes?* + - [easy] troubleshooting-log (Get log productpage due 500) + **Prompt:** *Why is the productpage service returning 500 errors?* + - [easy] troubleshooting-trace-lagging (Check traces for a service) + **Prompt:** *I see a spike in duration for ratings. Can you check the traces to see which span is lagging?* +- Uncategorized + - [easy] delete-faultInjection (Remove fault Injection) + **Prompt:** *Fix my namespace bookinfo to remove the fault injection.* + - [easy] get-namespaces (get-namespaces) + **Prompt:** *Check namespaces in my mesh.* + - [easy] get-service-detail (get-service-detail) + **Prompt:** *Give me information about my service details in the namespace bookinfo.* + - [easy] mesh-status (mesh-status) + **Prompt:** *Check my mesh.* + diff --git a/evals/tasks/kiali/delete-faultInjection/delete-faultInjection.yaml b/evals/tasks/kiali/delete-faultInjection/delete-faultInjection.yaml new file mode 100644 index 000000000..ab281574e --- /dev/null +++ b/evals/tasks/kiali/delete-faultInjection/delete-faultInjection.yaml @@ -0,0 +1,69 @@ +kind: Task +metadata: + name: "Remove fault Injection" + difficulty: easy +steps: + setup: + inline: |- + #!/usr/bin/env bash + set -euo pipefail + cat <<'EOF' | kubectl apply -f - + apiVersion: networking.istio.io/v1 + kind: DestinationRule + metadata: + namespace: bookinfo + name: ratings + labels: + gevals.kiali.io/test: delete-fault-injection + spec: + host: ratings.bookinfo.svc.cluster.local + subsets: + - name: v1 + labels: + version: v1 + --- + apiVersion: networking.istio.io/v1 + kind: VirtualService + metadata: + namespace: bookinfo + name: ratings + labels: + gevals.kiali.io/test: delete-fault-injection + spec: + hosts: + - ratings.bookinfo.svc.cluster.local + http: + - route: + - destination: + host: ratings.bookinfo.svc.cluster.local + subset: v1 + weight: 100 + fault: + abort: + percentage: + value: 100 + httpStatus: 503 + EOF + verify: + inline: |- + #!/usr/bin/env bash + vs_fault_names="$(kubectl get virtualservice -n "${NAMESPACE}" -o json \ + | jq -r '[.items[] | select(any(.spec.http[]?; has("fault"))) | .metadata.name] | .[]?')" + if [[ -n "${vs_fault_names}" ]]; then + exit 1 + fi + + # Verify DestinationRule 'ratings' does not exist (created during setup) + if kubectl get destinationrule ratings -n "${NAMESPACE}" >/dev/null 2>&1; then + exit 1 + fi + cleanup: + inline: |- + #!/usr/bin/env bash + set -euo pipefail + NS="bookinfo" + LABEL="gevals.kiali.io/test=delete-fault-injection" + kubectl delete virtualservice -n "$NS" -l "$LABEL" --ignore-not-found + kubectl delete destinationrule -n "$NS" -l "$LABEL" --ignore-not-found + prompt: + inline: Fix my namespace bookinfo to remove the fault injection. \ No newline at end of file diff --git a/evals/tasks/kiali/get-namespaces/get-namespaces.yaml b/evals/tasks/kiali/get-namespaces/get-namespaces.yaml new file mode 100644 index 000000000..31f02420c --- /dev/null +++ b/evals/tasks/kiali/get-namespaces/get-namespaces.yaml @@ -0,0 +1,15 @@ +kind: Task +metadata: + name: "get-namespaces" + difficulty: easy +steps: + setup: + inline: |- + #!/usr/bin/env bash + verify: + contains: "bookinfo" + cleanup: + inline: |- + #!/usr/bin/env bash + prompt: + inline: Check namespaces in my mesh. \ No newline at end of file diff --git a/evals/tasks/kiali/get-service-detail/get-service-detail.yaml b/evals/tasks/kiali/get-service-detail/get-service-detail.yaml new file mode 100644 index 000000000..312bdf8fc --- /dev/null +++ b/evals/tasks/kiali/get-service-detail/get-service-detail.yaml @@ -0,0 +1,15 @@ +kind: Task +metadata: + name: "get-service-detail" + difficulty: easy +steps: + setup: + inline: |- + #!/usr/bin/env bash + verify: + contains: "service: details" + cleanup: + inline: |- + #!/usr/bin/env bash + prompt: + inline: Give me information about my service details in the namespace bookinfo. \ No newline at end of file diff --git a/evals/tasks/kiali/istio-create/istio-create.yaml b/evals/tasks/kiali/istio-create/istio-create.yaml new file mode 100644 index 000000000..5b7212cf9 --- /dev/null +++ b/evals/tasks/kiali/istio-create/istio-create.yaml @@ -0,0 +1,31 @@ +kind: Task +metadata: + name: "Create a gateway" + category: "Istio Configuration & Management" + difficulty: medium +steps: + setup: + inline: |- + #!/usr/bin/env bash + set -euo pipefail + verify: + inline: |- + #!/usr/bin/env bash + set -euo pipefail + NS="istio-system" + NAME="my-gateway" + if kubectl get gw "$NAME" -n "$NS" >/dev/null 2>&1; then + echo "Verified: Gateway '$NAME' exists in namespace '$NS'." + else + echo "Gateway '$NAME' not found in namespace '$NS'." + exit 1 + fi + cleanup: + inline: |- + #!/usr/bin/env bash + set -euo pipefail + NS="istio-system" + NAME="my-gateway" + kubectl delete gw "$NAME" -n "$NS" --ignore-not-found + prompt: + inline: Create a Gateway named my-gateway in the istio-system namespace. \ No newline at end of file diff --git a/evals/tasks/kiali/istio-delete/istio-delete.yaml b/evals/tasks/kiali/istio-delete/istio-delete.yaml new file mode 100644 index 000000000..1f9f2e2e1 --- /dev/null +++ b/evals/tasks/kiali/istio-delete/istio-delete.yaml @@ -0,0 +1,70 @@ +kind: Task +metadata: + name: "Remove fault Injection" + category: "Istio Configuration & Management" + difficulty: medium +steps: + setup: + inline: |- + #!/usr/bin/env bash + set -euo pipefail + cat <<'EOF' | kubectl apply -f - + apiVersion: networking.istio.io/v1 + kind: DestinationRule + metadata: + namespace: bookinfo + name: ratings + labels: + gevals.kiali.io/test: delete-fault-injection + spec: + host: ratings.bookinfo.svc.cluster.local + subsets: + - name: v1 + labels: + version: v1 + --- + apiVersion: networking.istio.io/v1 + kind: VirtualService + metadata: + namespace: bookinfo + name: ratings + labels: + gevals.kiali.io/test: delete-fault-injection + spec: + hosts: + - ratings.bookinfo.svc.cluster.local + http: + - route: + - destination: + host: ratings.bookinfo.svc.cluster.local + subset: v1 + weight: 100 + fault: + abort: + percentage: + value: 100 + httpStatus: 503 + EOF + verify: + inline: |- + #!/usr/bin/env bash + vs_fault_names="$(kubectl get virtualservice -n "${NAMESPACE}" -o json \ + | jq -r '[.items[] | select(any(.spec.http[]?; has("fault"))) | .metadata.name] | .[]?')" + if [[ -n "${vs_fault_names}" ]]; then + exit 1 + fi + + # Verify DestinationRule 'ratings' does not exist (created during setup) + if kubectl get destinationrule ratings -n "${NAMESPACE}" >/dev/null 2>&1; then + exit 1 + fi + cleanup: + inline: |- + #!/usr/bin/env bash + set -euo pipefail + NS="bookinfo" + LABEL="gevals.kiali.io/test=gevals-testing" + kubectl delete virtualservice -n "$NS" -l "$LABEL" --ignore-not-found + kubectl delete destinationrule -n "$NS" -l "$LABEL" --ignore-not-found + prompt: + inline: Fix my namespace bookinfo to remove the fault injection. \ No newline at end of file diff --git a/evals/tasks/kiali/istio-list/istio-list.yaml b/evals/tasks/kiali/istio-list/istio-list.yaml new file mode 100644 index 000000000..84247ddbb --- /dev/null +++ b/evals/tasks/kiali/istio-list/istio-list.yaml @@ -0,0 +1,47 @@ +kind: Task +metadata: + name: "List all VS in bookinfo namespace" + category: "Istio Configuration & Management" + difficulty: easy +steps: + setup: + inline: |- + #!/usr/bin/env bash + set -euo pipefail + cat <<'EOF' | kubectl apply -f - + apiVersion: networking.istio.io/v1alpha3 + kind: VirtualService + metadata: + name: productpage-forced-failure + namespace: bookinfo + labels: + gevals.kiali.io/test: delete-fault-injection + spec: + hosts: + - "productpage" + - "productpage-bookinfo.apps-crc.testing" + gateways: + - bookinfo-gateway + - mesh # Para que afecte también si un microservicio llama a otro internamente + http: + - fault: + abort: + httpStatus: 500 + percentage: + value: 100 + route: + - destination: + host: productpage + subset: v1 + EOF + verify: + contains: "productpage-forced-failure" + cleanup: + inline: |- + #!/usr/bin/env bash + set -euo pipefail + NS="bookinfo" + LABEL="gevals.kiali.io/test=gevals-testing" + kubectl delete virtualservice -n "$NS" -l "$LABEL" --ignore-not-found + prompt: + inline: List all VirtualServices in the bookinfo namespace and check if they have any validation errors \ No newline at end of file diff --git a/evals/tasks/kiali/istio-patch/istio-patch.yaml b/evals/tasks/kiali/istio-patch/istio-patch.yaml new file mode 100644 index 000000000..07c31b052 --- /dev/null +++ b/evals/tasks/kiali/istio-patch/istio-patch.yaml @@ -0,0 +1,110 @@ +kind: Task +metadata: + name: "Patch my traffic" + category: "Istio Configuration & Management" + difficulty: medium +steps: + setup: + inline: |- + #!/usr/bin/env bash + set -euo pipefail + cat <<'EOF' | kubectl apply -f - + kind: DestinationRule + apiVersion: networking.istio.io/v1 + metadata: + namespace: bookinfo + name: reviews + labels: + gevals.kiali.io/test: gevals-testing + annotations: ~ + spec: + host: reviews.bookinfo.svc.cluster.local + subsets: + - name: v1 + labels: + version: v1 + - name: v2 + labels: + version: v2 + - name: v3 + labels: + version: v3 + trafficPolicy: ~ + + --- + + kind: VirtualService + apiVersion: networking.istio.io/v1 + metadata: + namespace: bookinfo + name: reviews + labels: + gevals.kiali.io/test: gevals-testing + spec: + http: + - route: + - destination: + host: reviews.bookinfo.svc.cluster.local + subset: v1 + weight: 0 + - destination: + host: reviews.bookinfo.svc.cluster.local + subset: v2 + weight: 0 + - destination: + host: reviews.bookinfo.svc.cluster.local + subset: v3 + weight: 100 + hosts: + - reviews.bookinfo.svc.cluster.local + gateways: ~ + EOF + verify: + inline: |- + #!/usr/bin/env bash + set -euo pipefail + NS="bookinfo" + LABEL="gevals.kiali.io/test=gevals-testing" + NAME="reviews" + + if ! command -v jq >/dev/null 2>&1; then + echo "jq is required for verification" + exit 1 + fi + + # Fetch the VirtualService by label and name + vs_json="$(kubectl get virtualservice -n "$NS" -l "$LABEL" -o json)" + found="$(echo "$vs_json" | jq -r --arg name "$NAME" '.items[]? | select(.metadata.name==$name) | .metadata.name' | head -n1)" + if [[ "$found" != "$NAME" ]]; then + echo "VirtualService '$NAME' with label '$LABEL' not found in namespace '$NS'" + exit 1 + fi + + # Verify there is a route to subset v2 with weight 50 + ok="$(echo "$vs_json" | jq -e --arg name "$NAME" ' + .items[]? | select(.metadata.name==$name) + | any(.spec.http[]?.route[]?; (.destination.subset=="v2") and ((.weight // 0) == 50)) + ' >/dev/null && echo yes || echo no)" + + if [[ "$ok" != "yes" ]]; then + echo "VirtualService '$NAME' does not route subset v2 with weight 50" + echo "Current routes:" + echo "$vs_json" | jq -r --arg name "$NAME" ' + .items[]? | select(.metadata.name==$name) + | .spec.http[]?.route[]? | {subset: .destination.subset, weight: .weight} + ' + exit 1 + fi + echo "Verified: VirtualService '$NAME' routes subset v2 with weight 50." + cleanup: + inline: |- + #!/usr/bin/env bash + set -euo pipefail + NS="bookinfo" + LABEL="gevals.kiali.io/test=gevals-testing" + kubectl delete virtualservice -n "$NS" -l "$LABEL" --ignore-not-found + kubectl delete destinationrule -n "$NS" -l "$LABEL" --ignore-not-found + prompt: + inline: I need to shift 50% of traffic to v2 of the reviews service. Apply a patch to the existing VirtualService. + + \ No newline at end of file diff --git a/evals/tasks/kiali/mesh-status/mesh-status.yaml b/evals/tasks/kiali/mesh-status/mesh-status.yaml new file mode 100644 index 000000000..24c64d24b --- /dev/null +++ b/evals/tasks/kiali/mesh-status/mesh-status.yaml @@ -0,0 +1,58 @@ +kind: Task +metadata: + name: "mesh-status" + difficulty: easy +steps: + setup: + inline: |- + #!/usr/bin/env bash + set -euo pipefail + cat <<'EOF' | kubectl apply -f - + apiVersion: networking.istio.io/v1 + kind: DestinationRule + metadata: + namespace: bookinfo + name: ratings + labels: + gevals.kiali.io/test: delete-fault-injection + spec: + host: ratings.bookinfo.svc.cluster.local + subsets: + - name: v1 + labels: + version: v1 + --- + apiVersion: networking.istio.io/v1 + kind: VirtualService + metadata: + namespace: bookinfo + name: ratings + labels: + gevals.kiali.io/test: delete-fault-injection + spec: + hosts: + - ratings.bookinfo.svc.cluster.local + http: + - route: + - destination: + host: ratings.bookinfo.svc.cluster.local + subset: v1 + weight: 100 + fault: + abort: + percentage: + value: 100 + httpStatus: 503 + EOF + verify: + contains: "healthy" + cleanup: + inline: |- + #!/usr/bin/env bash + set -euo pipefail + NS="bookinfo" + LABEL="gevals.kiali.io/test=delete-fault-injection" + kubectl delete virtualservice -n "$NS" -l "$LABEL" --ignore-not-found + kubectl delete destinationrule -n "$NS" -l "$LABEL" --ignore-not-found + prompt: + inline: Check my mesh. \ No newline at end of file diff --git a/evals/tasks/kiali/obs-unhealthy-namespaces/obs-unhealthy-namespaces.yaml b/evals/tasks/kiali/obs-unhealthy-namespaces/obs-unhealthy-namespaces.yaml new file mode 100644 index 000000000..4640a04ae --- /dev/null +++ b/evals/tasks/kiali/obs-unhealthy-namespaces/obs-unhealthy-namespaces.yaml @@ -0,0 +1,62 @@ +kind: Task +metadata: + name: "Unhealthy Namespaces" + category: "High-Level Observability & Health" + difficulty: easy +steps: + setup: + inline: |- + #!/usr/bin/env bash + set -euo pipefail + cat <<'EOF' | kubectl apply -f - + kind: DestinationRule + apiVersion: networking.istio.io/v1 + metadata: + namespace: bookinfo + name: ratings + labels: + gevals.kiali.io/test: delete-fault-injection + annotations: ~ + spec: + host: ratings.bookinfo.svc.cluster.local + subsets: + - name: v1 + labels: + version: v1 + trafficPolicy: ~ + + --- + apiVersion: networking.istio.io/v1 + kind: VirtualService + metadata: + name: ratings + namespace: bookinfo + labels: + gevals.kiali.io/test: fault-abort + spec: + hosts: + - ratings + http: + - fault: + abort: + httpStatus: 500 + percentage: + value: 100 + route: + - destination: + host: ratings + subset: v1 + EOF + verify: + contains: "1 unhealthy namespace" + cleanup: + inline: |- + #!/usr/bin/env bash + set -euo pipefail + NS="bookinfo" + LABEL="gevals.kiali.io/test=fault-abort" + kubectl delete virtualservice -n "$NS" -l "$LABEL" --ignore-not-found + kubectl delete destinationrule -n "$NS" -l "$LABEL" --ignore-not-found + prompt: + inline: Are there any unhealthy namespaces in my mesh right now? + diff --git a/evals/tasks/kiali/resource-get-namespaces/resource-get-namespaces.yaml b/evals/tasks/kiali/resource-get-namespaces/resource-get-namespaces.yaml new file mode 100644 index 000000000..da9cce67c --- /dev/null +++ b/evals/tasks/kiali/resource-get-namespaces/resource-get-namespaces.yaml @@ -0,0 +1,16 @@ +kind: Task +metadata: + name: "Get mesh namespaces" + category: "Resource Inspection" + difficulty: easy +steps: + setup: + inline: |- + #!/usr/bin/env bash + verify: + contains: "namespaces in your mesh" + cleanup: + inline: |- + #!/usr/bin/env bash + prompt: + inline: Check namespaces in my mesh. \ No newline at end of file diff --git a/evals/tasks/kiali/resource-get-service-detail/resource-get-service-detail.yaml b/evals/tasks/kiali/resource-get-service-detail/resource-get-service-detail.yaml new file mode 100644 index 000000000..2f6dc2483 --- /dev/null +++ b/evals/tasks/kiali/resource-get-service-detail/resource-get-service-detail.yaml @@ -0,0 +1,16 @@ +kind: Task +metadata: + name: "Get service detail" + category: "Resource Inspection" + difficulty: easy +steps: + setup: + inline: |- + #!/usr/bin/env bash + verify: + contains: "health status" + cleanup: + inline: |- + #!/usr/bin/env bash + prompt: + inline: Get the full details and health status for the details service \ No newline at end of file diff --git a/evals/tasks/kiali/resource-list-workloads/resource-list-workloads.yaml b/evals/tasks/kiali/resource-list-workloads/resource-list-workloads.yaml new file mode 100644 index 000000000..4fc515f51 --- /dev/null +++ b/evals/tasks/kiali/resource-list-workloads/resource-list-workloads.yaml @@ -0,0 +1,18 @@ +kind: Task +metadata: + name: "List workloads without sidecar" + category: "Resource Inspection" + difficulty: easy +steps: + setup: + inline: |- + #!/usr/bin/env bash + oc patch deployment details-v1 -n bookinfo -p '{"spec": {"template": {"metadata": {"annotations": {"sidecar.istio.io/inject": "false"}}}}}' + verify: + contains: "details-v1" + cleanup: + inline: |- + #!/usr/bin/env bash + oc patch deployment details-v1 -n bookinfo -p '{"spec": {"template": {"metadata": {"annotations": {"sidecar.istio.io/inject": "true"}}}}}' + prompt: + inline: List all workloads in the bookinfo namespace that have missing sidecars. \ No newline at end of file diff --git a/evals/tasks/kiali/resource-mesh-status/resource-mesh-status.yaml b/evals/tasks/kiali/resource-mesh-status/resource-mesh-status.yaml new file mode 100644 index 000000000..c4eb284ab --- /dev/null +++ b/evals/tasks/kiali/resource-mesh-status/resource-mesh-status.yaml @@ -0,0 +1,60 @@ +kind: Task +metadata: + name: "Status of my mesh" + category: "Resource Inspection" + difficulty: easy +steps: + setup: + inline: |- + #!/usr/bin/env bash + set -euo pipefail + cat <<'EOF' | kubectl apply -f - + apiVersion: networking.istio.io/v1 + kind: DestinationRule + metadata: + namespace: bookinfo + name: ratings + labels: + gevals.kiali.io/test: delete-fault-injection + spec: + host: ratings.bookinfo.svc.cluster.local + subsets: + - name: v1 + labels: + version: v1 + --- + apiVersion: networking.istio.io/v1 + kind: VirtualService + metadata: + namespace: bookinfo + name: ratings + labels: + gevals.kiali.io/test: delete-fault-injection + spec: + hosts: + - ratings.bookinfo.svc.cluster.local + http: + - route: + - destination: + host: ratings.bookinfo.svc.cluster.local + subset: v1 + weight: 100 + fault: + abort: + percentage: + value: 100 + httpStatus: 503 + EOF + verify: + inline: |- + #!/usr/bin/env bash + cleanup: + inline: |- + #!/usr/bin/env bash + set -euo pipefail + NS="bookinfo" + LABEL="gevals.kiali.io/test=gevals-testing" + kubectl delete virtualservice -n "$NS" -l "$LABEL" --ignore-not-found + kubectl delete destinationrule -n "$NS" -l "$LABEL" --ignore-not-found + prompt: + inline: Check my mesh. \ No newline at end of file diff --git a/evals/tasks/kiali/scripts/update_tasks.sh b/evals/tasks/kiali/scripts/update_tasks.sh new file mode 100755 index 000000000..31b03dad8 --- /dev/null +++ b/evals/tasks/kiali/scripts/update_tasks.sh @@ -0,0 +1,144 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +README_FILE="${1:-$ROOT_DIR/README.md}" +sanitize_section() { + local start_mark="$1" + local end_mark="$2" + local infile="$3" + local outfile="$4" + awk -v start="$start_mark" -v end="$end_mark" ' + BEGIN { inblock=0 } + { + if ($0 == start) { + print; + inblock=1; + next + } + if ($0 == end) { + print; + inblock=0; + next + } + if (inblock==1) next; + print + } + ' "$infile" > "$outfile" +} + +TMP_TASKS_CONTENT="$(mktemp)" +trap 'rm -f "$TMP_TASKS_CONTENT"' EXIT +TMP_TASKS_RAW="$(mktemp)" +trap 'rm -f "$TMP_TASKS_RAW"' EXIT +# Build Tasks summary section (grouped by category, ordered by difficulty) +{ + TASKS_DIR="$ROOT_DIR/" + if [[ ! -d "$TASKS_DIR" ]]; then + echo "_No tasks directory found at tests/tasks._" + else + : > "$TMP_TASKS_RAW" + for d in "$TASKS_DIR"/*; do + [[ -d "$d" ]] || continue + base="$(basename "$d")" + shopt -s nullglob + yaml_candidates=("$d"/*.yaml) + shopt -u nullglob + if [[ ${#yaml_candidates[@]} -eq 0 ]]; then + continue + fi + yaml="${yaml_candidates[0]}" + metaName="$(awk ' + /^[ \t]*metadata:/ {inm=1; next} + /^[^ \t]/ && inm==1 {inm=0} + inm && $1 == "name:" { + $1=""; sub(/^[ \t]+/, ""); + gsub(/^"|"$/, "", $0); + print; exit + } + ' "$yaml" 2>/dev/null || true)" + category="$(awk ' + /^[ \t]*metadata:/ {inm=1; next} + /^[^ \t]/ && inm==1 {inm=0} + inm && $1 == "category:" { + $1=""; sub(/^[ \t]+/, ""); + gsub(/^"|"$/, "", $0); + print; exit + } + ' "$yaml" 2>/dev/null || true)" + difficulty="$(awk ' + /^[ \t]*metadata:/ {inm=1; next} + /^[^ \t]/ && inm==1 {inm=0} + inm && $1 == "difficulty:" { + $1=""; sub(/^[ \t]+/, ""); + gsub(/^"|"$/, "", $0); + print; exit + } + ' "$yaml" 2>/dev/null || true)" + if [[ -z "$difficulty" ]]; then + difficulty="$(awk ' + /^[ \t]*difficulty:[ \t]*/ { sub(/^[ \t]*difficulty:[ \t]*/, ""); gsub(/^"|"$/, "", $0); print; exit } + ' "$yaml" 2>/dev/null || true)" + fi + prompt="$(awk ' + /^[ \t]*prompt:/ {pin=1; next} + pin && /^[^ \t]/ {pin=0} + pin && /^[ \t]*inline:/ { + sub(/^[ \t]*inline:[ \t]*/, ""); + print; exit + } + ' "$yaml" 2>/dev/null || true)" + [[ -n "$metaName" ]] || metaName="$base" + [[ -n "$prompt" ]] || prompt="(no prompt)" + [[ -n "$category" ]] || category="Uncategorized" + dnorm="$(echo "$difficulty" | tr '[:upper:]' '[:lower:]')" + case "$dnorm" in + easy) dorder=1 ;; + medium) dorder=2 ;; + hard) dorder=3 ;; + *) dorder=9 ;; + esac + printf "%s\t%02d\t%s\t%s\t%s\t%s\n" "$category" "$dorder" "$dnorm" "$base" "$metaName" "$prompt" >> "$TMP_TASKS_RAW" + done + current_cat="" + sort -t $'\t' -k1,1 -k2,2n -k4,4 "$TMP_TASKS_RAW" | while IFS=$'\t' read -r category dorder dlabel base metaName prompt; do + if [[ "$category" != "$current_cat" ]]; then + echo "- $category" + current_cat="$category" + fi + echo " - [${dlabel:-easy}] ${base} (${metaName})" + echo " **Prompt:** *${prompt}*" + done + fi +} > "$TMP_TASKS_CONTENT" + +# Update Tasks section in README +TASKS_START="" +TASKS_END="" +if ! grep -q "$TASKS_START" "$README_FILE" || ! grep -q "$TASKS_END" "$README_FILE"; then + echo "README tasks markers not found. Please ensure $TASKS_START and $TASKS_END exist." + exit 1 +fi +sanitize_section "$TASKS_START" "$TASKS_END" "$README_FILE" "$README_FILE.tmp" +mv "$README_FILE.tmp" "$README_FILE" +awk -v start="$TASKS_START" -v end="$TASKS_END" -v content_file="$TMP_TASKS_CONTENT" ' + BEGIN { inblock=0 } + { + if ($0 == start) { + print; + while ((getline line < content_file) > 0) print line; + inblock=1; + next + } + if ($0 == end) { + print; + inblock=0; + next + } + if (inblock==1) next; + print + } +' "$README_FILE" > "$README_FILE.tmp" +mv "$README_FILE.tmp" "$README_FILE" +echo "Updated tasks section in $README_FILE" + diff --git a/evals/tasks/kiali/show-topology/show-topology.yaml b/evals/tasks/kiali/show-topology/show-topology.yaml new file mode 100644 index 000000000..5b2f72860 --- /dev/null +++ b/evals/tasks/kiali/show-topology/show-topology.yaml @@ -0,0 +1,16 @@ +kind: Task +metadata: + name: "Show topology bookinfo" + category: "High-Level Observability & Health" + difficulty: easy +steps: + setup: + inline: |- + #!/usr/bin/env bash + verify: + contains: "topology of the **bookinfo**" + cleanup: + inline: |- + #!/usr/bin/env bash + prompt: + inline: Show me the topology of the bookinfo namespace. \ No newline at end of file diff --git a/evals/tasks/kiali/status-kiali-istio/status-kiali-istio.yaml b/evals/tasks/kiali/status-kiali-istio/status-kiali-istio.yaml new file mode 100644 index 000000000..b399e76a0 --- /dev/null +++ b/evals/tasks/kiali/status-kiali-istio/status-kiali-istio.yaml @@ -0,0 +1,16 @@ +kind: Task +metadata: + name: "Status Kiali and Istio" + category: "High-Level Observability & Health" + difficulty: easy +steps: + setup: + inline: |- + #!/usr/bin/env bash + verify: + contains: "Istio Version" + cleanup: + inline: |- + #!/usr/bin/env bash + prompt: + inline: Give me a status report on the interaction between Kiali and Istio components \ No newline at end of file diff --git a/evals/tasks/kiali/troubleshooting-latency-traces/troubleshooting-latency-traces.yaml b/evals/tasks/kiali/troubleshooting-latency-traces/troubleshooting-latency-traces.yaml new file mode 100644 index 000000000..f1d604eba --- /dev/null +++ b/evals/tasks/kiali/troubleshooting-latency-traces/troubleshooting-latency-traces.yaml @@ -0,0 +1,16 @@ +kind: Task +metadata: + name: "Get latency workload" + category: "Troubleshooting & Debugging" + difficulty: easy +steps: + setup: + inline: |- + #!/usr/bin/env bash + verify: + contains: "30 minutes" + cleanup: + inline: |- + #!/usr/bin/env bash + prompt: + inline: Analyze the latency for the reviews workload over the last 30 minutes? \ No newline at end of file diff --git a/evals/tasks/kiali/troubleshooting-log/troubleshooting-log.yaml b/evals/tasks/kiali/troubleshooting-log/troubleshooting-log.yaml new file mode 100644 index 000000000..cc9d92c94 --- /dev/null +++ b/evals/tasks/kiali/troubleshooting-log/troubleshooting-log.yaml @@ -0,0 +1,44 @@ +kind: Task +metadata: + name: "Get log productpage due 500" + category: "Troubleshooting & Debugging" + difficulty: easy +steps: + setup: + inline: |- + #!/usr/bin/env bash + set -euo pipefail + cat <<'EOF' | kubectl apply -f - + apiVersion: networking.istio.io/v1alpha3 + kind: VirtualService + metadata: + name: productpage-fault + namespace: bookinfo + labels: + gevals.kiali.io/test: gevals-testing + spec: + hosts: + - "productpage" + - "productpage.bookinfo.svc.cluster.local" + - "productpage-bookinfo.apps-crc.testing" # <--- ¡AQUÍ ESTÁ LA CLAVE! + http: + - fault: + abort: + httpStatus: 500 + percentage: + value: 100 + route: + - destination: + host: productpage + EOF + verify: + contains: "fault injection" + cleanup: + inline: |- + #!/usr/bin/env bash + set -euo pipefail + NS="bookinfo" + LABEL="gevals.kiali.io/test=gevals-testing" + kubectl delete virtualservice -n "$NS" -l "$LABEL" --ignore-not-found + prompt: + inline: Why is the productpage service returning 500 errors? \ No newline at end of file diff --git a/evals/tasks/kiali/troubleshooting-trace-lagging/troubleshooting-trace-lagging.yaml b/evals/tasks/kiali/troubleshooting-trace-lagging/troubleshooting-trace-lagging.yaml new file mode 100644 index 000000000..8af03eb17 --- /dev/null +++ b/evals/tasks/kiali/troubleshooting-trace-lagging/troubleshooting-trace-lagging.yaml @@ -0,0 +1,68 @@ +kind: Task +metadata: + name: "Check traces for a service" + category: "Troubleshooting & Debugging" + difficulty: easy +steps: + setup: + inline: |- + #!/usr/bin/env bash + set -euo pipefail + cat <<'EOF' | kubectl apply -f - + kind: DestinationRule + apiVersion: networking.istio.io/v1 + metadata: + namespace: bookinfo + name: ratings + labels: + gevals.kiali.io/test: delete-fault-injection + annotations: ~ + spec: + host: ratings.bookinfo.svc.cluster.local + subsets: + - name: v1 + labels: + version: v1 + trafficPolicy: ~ + + --- + + kind: VirtualService + apiVersion: networking.istio.io/v1 + metadata: + namespace: bookinfo + name: ratings + labels: + gevals.kiali.io/test: delete-fault-injection + spec: + http: + - route: + - destination: + host: ratings.bookinfo.svc.cluster.local + subset: v1 + weight: 100 + fault: + delay: + percentage: + value: 100 + fixedDelay: 5s + hosts: + - ratings.bookinfo.svc.cluster.local + gateways: ~ + EOF + verify: + contains: "ratings" + cleanup: + inline: |- + #!/usr/bin/env bash + set -euo pipefail + NS="bookinfo" + LABEL="gevals.kiali.io/test=gevals-testing" + kubectl delete virtualservice -n "$NS" -l "$LABEL" --ignore-not-found + kubectl delete destinationrule -n "$NS" -l "$LABEL" --ignore-not-found + prompt: + inline: I see a spike in duration for ratings. Can you check the traces to see which span is lagging? + + + + \ No newline at end of file diff --git a/evals/tasks/create-canary-deployment/artifacts/deployment-v1.yaml b/evals/tasks/kubernetes/create-canary-deployment/artifacts/deployment-v1.yaml similarity index 100% rename from evals/tasks/create-canary-deployment/artifacts/deployment-v1.yaml rename to evals/tasks/kubernetes/create-canary-deployment/artifacts/deployment-v1.yaml diff --git a/evals/tasks/create-canary-deployment/artifacts/service.yaml b/evals/tasks/kubernetes/create-canary-deployment/artifacts/service.yaml similarity index 100% rename from evals/tasks/create-canary-deployment/artifacts/service.yaml rename to evals/tasks/kubernetes/create-canary-deployment/artifacts/service.yaml diff --git a/evals/tasks/create-canary-deployment/cleanup.sh b/evals/tasks/kubernetes/create-canary-deployment/cleanup.sh similarity index 100% rename from evals/tasks/create-canary-deployment/cleanup.sh rename to evals/tasks/kubernetes/create-canary-deployment/cleanup.sh diff --git a/evals/tasks/create-canary-deployment/create-canary-deployment.yaml b/evals/tasks/kubernetes/create-canary-deployment/create-canary-deployment.yaml similarity index 100% rename from evals/tasks/create-canary-deployment/create-canary-deployment.yaml rename to evals/tasks/kubernetes/create-canary-deployment/create-canary-deployment.yaml diff --git a/evals/tasks/create-canary-deployment/setup.sh b/evals/tasks/kubernetes/create-canary-deployment/setup.sh similarity index 100% rename from evals/tasks/create-canary-deployment/setup.sh rename to evals/tasks/kubernetes/create-canary-deployment/setup.sh diff --git a/evals/tasks/create-canary-deployment/verify.sh b/evals/tasks/kubernetes/create-canary-deployment/verify.sh similarity index 100% rename from evals/tasks/create-canary-deployment/verify.sh rename to evals/tasks/kubernetes/create-canary-deployment/verify.sh diff --git a/evals/tasks/create-network-policy/artifacts/desired-policy.yaml b/evals/tasks/kubernetes/create-network-policy/artifacts/desired-policy.yaml similarity index 100% rename from evals/tasks/create-network-policy/artifacts/desired-policy.yaml rename to evals/tasks/kubernetes/create-network-policy/artifacts/desired-policy.yaml diff --git a/evals/tasks/create-network-policy/cleanup.sh b/evals/tasks/kubernetes/create-network-policy/cleanup.sh similarity index 100% rename from evals/tasks/create-network-policy/cleanup.sh rename to evals/tasks/kubernetes/create-network-policy/cleanup.sh diff --git a/evals/tasks/create-network-policy/create-network-policy.yaml b/evals/tasks/kubernetes/create-network-policy/create-network-policy.yaml similarity index 100% rename from evals/tasks/create-network-policy/create-network-policy.yaml rename to evals/tasks/kubernetes/create-network-policy/create-network-policy.yaml diff --git a/evals/tasks/create-network-policy/setup.sh b/evals/tasks/kubernetes/create-network-policy/setup.sh similarity index 100% rename from evals/tasks/create-network-policy/setup.sh rename to evals/tasks/kubernetes/create-network-policy/setup.sh diff --git a/evals/tasks/create-network-policy/verify.sh b/evals/tasks/kubernetes/create-network-policy/verify.sh similarity index 100% rename from evals/tasks/create-network-policy/verify.sh rename to evals/tasks/kubernetes/create-network-policy/verify.sh diff --git a/evals/tasks/create-pod-mount-configmaps/cleanup.sh b/evals/tasks/kubernetes/create-pod-mount-configmaps/cleanup.sh similarity index 100% rename from evals/tasks/create-pod-mount-configmaps/cleanup.sh rename to evals/tasks/kubernetes/create-pod-mount-configmaps/cleanup.sh diff --git a/evals/tasks/create-pod-mount-configmaps/create-pod-mount-configmaps.yaml b/evals/tasks/kubernetes/create-pod-mount-configmaps/create-pod-mount-configmaps.yaml similarity index 100% rename from evals/tasks/create-pod-mount-configmaps/create-pod-mount-configmaps.yaml rename to evals/tasks/kubernetes/create-pod-mount-configmaps/create-pod-mount-configmaps.yaml diff --git a/evals/tasks/create-pod-mount-configmaps/setup.sh b/evals/tasks/kubernetes/create-pod-mount-configmaps/setup.sh similarity index 100% rename from evals/tasks/create-pod-mount-configmaps/setup.sh rename to evals/tasks/kubernetes/create-pod-mount-configmaps/setup.sh diff --git a/evals/tasks/create-pod-mount-configmaps/verify.sh b/evals/tasks/kubernetes/create-pod-mount-configmaps/verify.sh similarity index 100% rename from evals/tasks/create-pod-mount-configmaps/verify.sh rename to evals/tasks/kubernetes/create-pod-mount-configmaps/verify.sh diff --git a/evals/tasks/create-pod-resources-limits/cleanup.sh b/evals/tasks/kubernetes/create-pod-resources-limits/cleanup.sh similarity index 100% rename from evals/tasks/create-pod-resources-limits/cleanup.sh rename to evals/tasks/kubernetes/create-pod-resources-limits/cleanup.sh diff --git a/evals/tasks/create-pod-resources-limits/create-pod-resources-limits.yaml b/evals/tasks/kubernetes/create-pod-resources-limits/create-pod-resources-limits.yaml similarity index 100% rename from evals/tasks/create-pod-resources-limits/create-pod-resources-limits.yaml rename to evals/tasks/kubernetes/create-pod-resources-limits/create-pod-resources-limits.yaml diff --git a/evals/tasks/create-pod-resources-limits/setup.sh b/evals/tasks/kubernetes/create-pod-resources-limits/setup.sh similarity index 100% rename from evals/tasks/create-pod-resources-limits/setup.sh rename to evals/tasks/kubernetes/create-pod-resources-limits/setup.sh diff --git a/evals/tasks/create-pod-resources-limits/verify.sh b/evals/tasks/kubernetes/create-pod-resources-limits/verify.sh similarity index 100% rename from evals/tasks/create-pod-resources-limits/verify.sh rename to evals/tasks/kubernetes/create-pod-resources-limits/verify.sh diff --git a/evals/tasks/create-pod/cleanup.sh b/evals/tasks/kubernetes/create-pod/cleanup.sh similarity index 100% rename from evals/tasks/create-pod/cleanup.sh rename to evals/tasks/kubernetes/create-pod/cleanup.sh diff --git a/evals/tasks/create-pod/create-pod.yaml b/evals/tasks/kubernetes/create-pod/create-pod.yaml similarity index 100% rename from evals/tasks/create-pod/create-pod.yaml rename to evals/tasks/kubernetes/create-pod/create-pod.yaml diff --git a/evals/tasks/create-pod/setup.sh b/evals/tasks/kubernetes/create-pod/setup.sh similarity index 100% rename from evals/tasks/create-pod/setup.sh rename to evals/tasks/kubernetes/create-pod/setup.sh diff --git a/evals/tasks/create-pod/verify.sh b/evals/tasks/kubernetes/create-pod/verify.sh similarity index 100% rename from evals/tasks/create-pod/verify.sh rename to evals/tasks/kubernetes/create-pod/verify.sh diff --git a/evals/tasks/create-simple-rbac/cleanup.sh b/evals/tasks/kubernetes/create-simple-rbac/cleanup.sh similarity index 100% rename from evals/tasks/create-simple-rbac/cleanup.sh rename to evals/tasks/kubernetes/create-simple-rbac/cleanup.sh diff --git a/evals/tasks/create-simple-rbac/create-simple-rbac.yaml b/evals/tasks/kubernetes/create-simple-rbac/create-simple-rbac.yaml similarity index 100% rename from evals/tasks/create-simple-rbac/create-simple-rbac.yaml rename to evals/tasks/kubernetes/create-simple-rbac/create-simple-rbac.yaml diff --git a/evals/tasks/create-simple-rbac/setup.sh b/evals/tasks/kubernetes/create-simple-rbac/setup.sh similarity index 100% rename from evals/tasks/create-simple-rbac/setup.sh rename to evals/tasks/kubernetes/create-simple-rbac/setup.sh diff --git a/evals/tasks/create-simple-rbac/verify.sh b/evals/tasks/kubernetes/create-simple-rbac/verify.sh similarity index 100% rename from evals/tasks/create-simple-rbac/verify.sh rename to evals/tasks/kubernetes/create-simple-rbac/verify.sh diff --git a/evals/tasks/debug-app-logs/artifacts/calc-app-pod.yaml b/evals/tasks/kubernetes/debug-app-logs/artifacts/calc-app-pod.yaml similarity index 100% rename from evals/tasks/debug-app-logs/artifacts/calc-app-pod.yaml rename to evals/tasks/kubernetes/debug-app-logs/artifacts/calc-app-pod.yaml diff --git a/evals/tasks/debug-app-logs/artifacts/calc-app.py b/evals/tasks/kubernetes/debug-app-logs/artifacts/calc-app.py similarity index 100% rename from evals/tasks/debug-app-logs/artifacts/calc-app.py rename to evals/tasks/kubernetes/debug-app-logs/artifacts/calc-app.py diff --git a/evals/tasks/debug-app-logs/cleanup.sh b/evals/tasks/kubernetes/debug-app-logs/cleanup.sh similarity index 100% rename from evals/tasks/debug-app-logs/cleanup.sh rename to evals/tasks/kubernetes/debug-app-logs/cleanup.sh diff --git a/evals/tasks/debug-app-logs/debug-app-logs.yaml b/evals/tasks/kubernetes/debug-app-logs/debug-app-logs.yaml similarity index 100% rename from evals/tasks/debug-app-logs/debug-app-logs.yaml rename to evals/tasks/kubernetes/debug-app-logs/debug-app-logs.yaml diff --git a/evals/tasks/debug-app-logs/setup.sh b/evals/tasks/kubernetes/debug-app-logs/setup.sh similarity index 100% rename from evals/tasks/debug-app-logs/setup.sh rename to evals/tasks/kubernetes/debug-app-logs/setup.sh diff --git a/evals/tasks/deployment-traffic-switch/artifacts/blue-deployment.yaml b/evals/tasks/kubernetes/deployment-traffic-switch/artifacts/blue-deployment.yaml similarity index 100% rename from evals/tasks/deployment-traffic-switch/artifacts/blue-deployment.yaml rename to evals/tasks/kubernetes/deployment-traffic-switch/artifacts/blue-deployment.yaml diff --git a/evals/tasks/deployment-traffic-switch/artifacts/green-deployment.yaml b/evals/tasks/kubernetes/deployment-traffic-switch/artifacts/green-deployment.yaml similarity index 100% rename from evals/tasks/deployment-traffic-switch/artifacts/green-deployment.yaml rename to evals/tasks/kubernetes/deployment-traffic-switch/artifacts/green-deployment.yaml diff --git a/evals/tasks/deployment-traffic-switch/artifacts/service.yaml b/evals/tasks/kubernetes/deployment-traffic-switch/artifacts/service.yaml similarity index 100% rename from evals/tasks/deployment-traffic-switch/artifacts/service.yaml rename to evals/tasks/kubernetes/deployment-traffic-switch/artifacts/service.yaml diff --git a/evals/tasks/deployment-traffic-switch/cleanup.sh b/evals/tasks/kubernetes/deployment-traffic-switch/cleanup.sh similarity index 100% rename from evals/tasks/deployment-traffic-switch/cleanup.sh rename to evals/tasks/kubernetes/deployment-traffic-switch/cleanup.sh diff --git a/evals/tasks/deployment-traffic-switch/deployment-traffix-switch.yaml b/evals/tasks/kubernetes/deployment-traffic-switch/deployment-traffix-switch.yaml similarity index 100% rename from evals/tasks/deployment-traffic-switch/deployment-traffix-switch.yaml rename to evals/tasks/kubernetes/deployment-traffic-switch/deployment-traffix-switch.yaml diff --git a/evals/tasks/deployment-traffic-switch/setup.sh b/evals/tasks/kubernetes/deployment-traffic-switch/setup.sh similarity index 100% rename from evals/tasks/deployment-traffic-switch/setup.sh rename to evals/tasks/kubernetes/deployment-traffic-switch/setup.sh diff --git a/evals/tasks/deployment-traffic-switch/verify.sh b/evals/tasks/kubernetes/deployment-traffic-switch/verify.sh similarity index 100% rename from evals/tasks/deployment-traffic-switch/verify.sh rename to evals/tasks/kubernetes/deployment-traffic-switch/verify.sh diff --git a/evals/tasks/fix-crashloop/cleanup.sh b/evals/tasks/kubernetes/fix-crashloop/cleanup.sh similarity index 100% rename from evals/tasks/fix-crashloop/cleanup.sh rename to evals/tasks/kubernetes/fix-crashloop/cleanup.sh diff --git a/evals/tasks/fix-crashloop/fix-crashloop.yaml b/evals/tasks/kubernetes/fix-crashloop/fix-crashloop.yaml similarity index 100% rename from evals/tasks/fix-crashloop/fix-crashloop.yaml rename to evals/tasks/kubernetes/fix-crashloop/fix-crashloop.yaml diff --git a/evals/tasks/fix-crashloop/setup.sh b/evals/tasks/kubernetes/fix-crashloop/setup.sh similarity index 100% rename from evals/tasks/fix-crashloop/setup.sh rename to evals/tasks/kubernetes/fix-crashloop/setup.sh diff --git a/evals/tasks/fix-crashloop/verify.sh b/evals/tasks/kubernetes/fix-crashloop/verify.sh similarity index 100% rename from evals/tasks/fix-crashloop/verify.sh rename to evals/tasks/kubernetes/fix-crashloop/verify.sh diff --git a/evals/tasks/fix-image-pull/cleanup.sh b/evals/tasks/kubernetes/fix-image-pull/cleanup.sh similarity index 100% rename from evals/tasks/fix-image-pull/cleanup.sh rename to evals/tasks/kubernetes/fix-image-pull/cleanup.sh diff --git a/evals/tasks/fix-image-pull/fix-image-pull.yaml b/evals/tasks/kubernetes/fix-image-pull/fix-image-pull.yaml similarity index 100% rename from evals/tasks/fix-image-pull/fix-image-pull.yaml rename to evals/tasks/kubernetes/fix-image-pull/fix-image-pull.yaml diff --git a/evals/tasks/fix-image-pull/setup.sh b/evals/tasks/kubernetes/fix-image-pull/setup.sh similarity index 100% rename from evals/tasks/fix-image-pull/setup.sh rename to evals/tasks/kubernetes/fix-image-pull/setup.sh diff --git a/evals/tasks/fix-image-pull/verify.sh b/evals/tasks/kubernetes/fix-image-pull/verify.sh similarity index 100% rename from evals/tasks/fix-image-pull/verify.sh rename to evals/tasks/kubernetes/fix-image-pull/verify.sh diff --git a/evals/tasks/fix-pending-pod/artifacts/homepage-pod.yaml b/evals/tasks/kubernetes/fix-pending-pod/artifacts/homepage-pod.yaml similarity index 100% rename from evals/tasks/fix-pending-pod/artifacts/homepage-pod.yaml rename to evals/tasks/kubernetes/fix-pending-pod/artifacts/homepage-pod.yaml diff --git a/evals/tasks/fix-pending-pod/artifacts/homepage-pvc.yaml b/evals/tasks/kubernetes/fix-pending-pod/artifacts/homepage-pvc.yaml similarity index 100% rename from evals/tasks/fix-pending-pod/artifacts/homepage-pvc.yaml rename to evals/tasks/kubernetes/fix-pending-pod/artifacts/homepage-pvc.yaml diff --git a/evals/tasks/fix-pending-pod/cleanup.sh b/evals/tasks/kubernetes/fix-pending-pod/cleanup.sh similarity index 100% rename from evals/tasks/fix-pending-pod/cleanup.sh rename to evals/tasks/kubernetes/fix-pending-pod/cleanup.sh diff --git a/evals/tasks/fix-pending-pod/fix-pending-pod.yaml b/evals/tasks/kubernetes/fix-pending-pod/fix-pending-pod.yaml similarity index 100% rename from evals/tasks/fix-pending-pod/fix-pending-pod.yaml rename to evals/tasks/kubernetes/fix-pending-pod/fix-pending-pod.yaml diff --git a/evals/tasks/fix-pending-pod/setup.sh b/evals/tasks/kubernetes/fix-pending-pod/setup.sh similarity index 100% rename from evals/tasks/fix-pending-pod/setup.sh rename to evals/tasks/kubernetes/fix-pending-pod/setup.sh diff --git a/evals/tasks/fix-pending-pod/verify.sh b/evals/tasks/kubernetes/fix-pending-pod/verify.sh similarity index 100% rename from evals/tasks/fix-pending-pod/verify.sh rename to evals/tasks/kubernetes/fix-pending-pod/verify.sh diff --git a/evals/tasks/fix-probes/cleanup.sh b/evals/tasks/kubernetes/fix-probes/cleanup.sh similarity index 100% rename from evals/tasks/fix-probes/cleanup.sh rename to evals/tasks/kubernetes/fix-probes/cleanup.sh diff --git a/evals/tasks/fix-probes/fix-probes.yaml b/evals/tasks/kubernetes/fix-probes/fix-probes.yaml similarity index 100% rename from evals/tasks/fix-probes/fix-probes.yaml rename to evals/tasks/kubernetes/fix-probes/fix-probes.yaml diff --git a/evals/tasks/fix-probes/setup.sh b/evals/tasks/kubernetes/fix-probes/setup.sh similarity index 100% rename from evals/tasks/fix-probes/setup.sh rename to evals/tasks/kubernetes/fix-probes/setup.sh diff --git a/evals/tasks/fix-probes/verify.sh b/evals/tasks/kubernetes/fix-probes/verify.sh similarity index 100% rename from evals/tasks/fix-probes/verify.sh rename to evals/tasks/kubernetes/fix-probes/verify.sh diff --git a/evals/tasks/fix-rbac-wrong-resource/cleanup.sh b/evals/tasks/kubernetes/fix-rbac-wrong-resource/cleanup.sh similarity index 100% rename from evals/tasks/fix-rbac-wrong-resource/cleanup.sh rename to evals/tasks/kubernetes/fix-rbac-wrong-resource/cleanup.sh diff --git a/evals/tasks/fix-rbac-wrong-resource/fix-rbac-wrong-resource.yaml b/evals/tasks/kubernetes/fix-rbac-wrong-resource/fix-rbac-wrong-resource.yaml similarity index 100% rename from evals/tasks/fix-rbac-wrong-resource/fix-rbac-wrong-resource.yaml rename to evals/tasks/kubernetes/fix-rbac-wrong-resource/fix-rbac-wrong-resource.yaml diff --git a/evals/tasks/fix-rbac-wrong-resource/setup.sh b/evals/tasks/kubernetes/fix-rbac-wrong-resource/setup.sh similarity index 100% rename from evals/tasks/fix-rbac-wrong-resource/setup.sh rename to evals/tasks/kubernetes/fix-rbac-wrong-resource/setup.sh diff --git a/evals/tasks/fix-rbac-wrong-resource/verify.sh b/evals/tasks/kubernetes/fix-rbac-wrong-resource/verify.sh similarity index 100% rename from evals/tasks/fix-rbac-wrong-resource/verify.sh rename to evals/tasks/kubernetes/fix-rbac-wrong-resource/verify.sh diff --git a/evals/tasks/fix-service-routing/cleanup.sh b/evals/tasks/kubernetes/fix-service-routing/cleanup.sh similarity index 100% rename from evals/tasks/fix-service-routing/cleanup.sh rename to evals/tasks/kubernetes/fix-service-routing/cleanup.sh diff --git a/evals/tasks/fix-service-routing/fix-service-routing.yaml b/evals/tasks/kubernetes/fix-service-routing/fix-service-routing.yaml similarity index 100% rename from evals/tasks/fix-service-routing/fix-service-routing.yaml rename to evals/tasks/kubernetes/fix-service-routing/fix-service-routing.yaml diff --git a/evals/tasks/fix-service-routing/setup.sh b/evals/tasks/kubernetes/fix-service-routing/setup.sh similarity index 100% rename from evals/tasks/fix-service-routing/setup.sh rename to evals/tasks/kubernetes/fix-service-routing/setup.sh diff --git a/evals/tasks/fix-service-routing/verify.sh b/evals/tasks/kubernetes/fix-service-routing/verify.sh similarity index 100% rename from evals/tasks/fix-service-routing/verify.sh rename to evals/tasks/kubernetes/fix-service-routing/verify.sh diff --git a/evals/tasks/fix-service-with-no-endpoints/artifacts/deployment.yaml b/evals/tasks/kubernetes/fix-service-with-no-endpoints/artifacts/deployment.yaml similarity index 100% rename from evals/tasks/fix-service-with-no-endpoints/artifacts/deployment.yaml rename to evals/tasks/kubernetes/fix-service-with-no-endpoints/artifacts/deployment.yaml diff --git a/evals/tasks/fix-service-with-no-endpoints/artifacts/service.yaml b/evals/tasks/kubernetes/fix-service-with-no-endpoints/artifacts/service.yaml similarity index 100% rename from evals/tasks/fix-service-with-no-endpoints/artifacts/service.yaml rename to evals/tasks/kubernetes/fix-service-with-no-endpoints/artifacts/service.yaml diff --git a/evals/tasks/fix-service-with-no-endpoints/cleanup.sh b/evals/tasks/kubernetes/fix-service-with-no-endpoints/cleanup.sh similarity index 100% rename from evals/tasks/fix-service-with-no-endpoints/cleanup.sh rename to evals/tasks/kubernetes/fix-service-with-no-endpoints/cleanup.sh diff --git a/evals/tasks/fix-service-with-no-endpoints/fix-service-with-no-endpoints.yaml b/evals/tasks/kubernetes/fix-service-with-no-endpoints/fix-service-with-no-endpoints.yaml similarity index 100% rename from evals/tasks/fix-service-with-no-endpoints/fix-service-with-no-endpoints.yaml rename to evals/tasks/kubernetes/fix-service-with-no-endpoints/fix-service-with-no-endpoints.yaml diff --git a/evals/tasks/fix-service-with-no-endpoints/setup.sh b/evals/tasks/kubernetes/fix-service-with-no-endpoints/setup.sh similarity index 100% rename from evals/tasks/fix-service-with-no-endpoints/setup.sh rename to evals/tasks/kubernetes/fix-service-with-no-endpoints/setup.sh diff --git a/evals/tasks/fix-service-with-no-endpoints/verify.sh b/evals/tasks/kubernetes/fix-service-with-no-endpoints/verify.sh similarity index 100% rename from evals/tasks/fix-service-with-no-endpoints/verify.sh rename to evals/tasks/kubernetes/fix-service-with-no-endpoints/verify.sh diff --git a/evals/tasks/horizontal-pod-autoscaler/cleanup.sh b/evals/tasks/kubernetes/horizontal-pod-autoscaler/cleanup.sh similarity index 100% rename from evals/tasks/horizontal-pod-autoscaler/cleanup.sh rename to evals/tasks/kubernetes/horizontal-pod-autoscaler/cleanup.sh diff --git a/evals/tasks/horizontal-pod-autoscaler/horizontal-pod-autoscaler.yaml b/evals/tasks/kubernetes/horizontal-pod-autoscaler/horizontal-pod-autoscaler.yaml similarity index 100% rename from evals/tasks/horizontal-pod-autoscaler/horizontal-pod-autoscaler.yaml rename to evals/tasks/kubernetes/horizontal-pod-autoscaler/horizontal-pod-autoscaler.yaml diff --git a/evals/tasks/horizontal-pod-autoscaler/setup.sh b/evals/tasks/kubernetes/horizontal-pod-autoscaler/setup.sh similarity index 100% rename from evals/tasks/horizontal-pod-autoscaler/setup.sh rename to evals/tasks/kubernetes/horizontal-pod-autoscaler/setup.sh diff --git a/evals/tasks/horizontal-pod-autoscaler/verify.sh b/evals/tasks/kubernetes/horizontal-pod-autoscaler/verify.sh similarity index 100% rename from evals/tasks/horizontal-pod-autoscaler/verify.sh rename to evals/tasks/kubernetes/horizontal-pod-autoscaler/verify.sh diff --git a/evals/tasks/list-images-for-pods/artifacts/manifest.yaml b/evals/tasks/kubernetes/list-images-for-pods/artifacts/manifest.yaml similarity index 100% rename from evals/tasks/list-images-for-pods/artifacts/manifest.yaml rename to evals/tasks/kubernetes/list-images-for-pods/artifacts/manifest.yaml diff --git a/evals/tasks/list-images-for-pods/cleanup.sh b/evals/tasks/kubernetes/list-images-for-pods/cleanup.sh similarity index 100% rename from evals/tasks/list-images-for-pods/cleanup.sh rename to evals/tasks/kubernetes/list-images-for-pods/cleanup.sh diff --git a/evals/tasks/list-images-for-pods/list-images-for-pods.yaml b/evals/tasks/kubernetes/list-images-for-pods/list-images-for-pods.yaml similarity index 100% rename from evals/tasks/list-images-for-pods/list-images-for-pods.yaml rename to evals/tasks/kubernetes/list-images-for-pods/list-images-for-pods.yaml diff --git a/evals/tasks/list-images-for-pods/setup.sh b/evals/tasks/kubernetes/list-images-for-pods/setup.sh similarity index 100% rename from evals/tasks/list-images-for-pods/setup.sh rename to evals/tasks/kubernetes/list-images-for-pods/setup.sh diff --git a/evals/tasks/multi-container-pod-communication/cleanup.sh b/evals/tasks/kubernetes/multi-container-pod-communication/cleanup.sh similarity index 100% rename from evals/tasks/multi-container-pod-communication/cleanup.sh rename to evals/tasks/kubernetes/multi-container-pod-communication/cleanup.sh diff --git a/evals/tasks/multi-container-pod-communication/multi-container-pod-communication.yaml b/evals/tasks/kubernetes/multi-container-pod-communication/multi-container-pod-communication.yaml similarity index 100% rename from evals/tasks/multi-container-pod-communication/multi-container-pod-communication.yaml rename to evals/tasks/kubernetes/multi-container-pod-communication/multi-container-pod-communication.yaml diff --git a/evals/tasks/multi-container-pod-communication/setup.sh b/evals/tasks/kubernetes/multi-container-pod-communication/setup.sh similarity index 100% rename from evals/tasks/multi-container-pod-communication/setup.sh rename to evals/tasks/kubernetes/multi-container-pod-communication/setup.sh diff --git a/evals/tasks/multi-container-pod-communication/verify.sh b/evals/tasks/kubernetes/multi-container-pod-communication/verify.sh similarity index 100% rename from evals/tasks/multi-container-pod-communication/verify.sh rename to evals/tasks/kubernetes/multi-container-pod-communication/verify.sh diff --git a/evals/tasks/resize-pvc/artifacts/storage-pod.yaml b/evals/tasks/kubernetes/resize-pvc/artifacts/storage-pod.yaml similarity index 100% rename from evals/tasks/resize-pvc/artifacts/storage-pod.yaml rename to evals/tasks/kubernetes/resize-pvc/artifacts/storage-pod.yaml diff --git a/evals/tasks/resize-pvc/artifacts/storage-pvc.yaml b/evals/tasks/kubernetes/resize-pvc/artifacts/storage-pvc.yaml similarity index 100% rename from evals/tasks/resize-pvc/artifacts/storage-pvc.yaml rename to evals/tasks/kubernetes/resize-pvc/artifacts/storage-pvc.yaml diff --git a/evals/tasks/resize-pvc/cleanup.sh b/evals/tasks/kubernetes/resize-pvc/cleanup.sh similarity index 100% rename from evals/tasks/resize-pvc/cleanup.sh rename to evals/tasks/kubernetes/resize-pvc/cleanup.sh diff --git a/evals/tasks/resize-pvc/resize-pvc.yaml b/evals/tasks/kubernetes/resize-pvc/resize-pvc.yaml similarity index 100% rename from evals/tasks/resize-pvc/resize-pvc.yaml rename to evals/tasks/kubernetes/resize-pvc/resize-pvc.yaml diff --git a/evals/tasks/resize-pvc/setup.sh b/evals/tasks/kubernetes/resize-pvc/setup.sh similarity index 100% rename from evals/tasks/resize-pvc/setup.sh rename to evals/tasks/kubernetes/resize-pvc/setup.sh diff --git a/evals/tasks/resize-pvc/verify.sh b/evals/tasks/kubernetes/resize-pvc/verify.sh similarity index 100% rename from evals/tasks/resize-pvc/verify.sh rename to evals/tasks/kubernetes/resize-pvc/verify.sh diff --git a/evals/tasks/rolling-update-deployment/cleanup.sh b/evals/tasks/kubernetes/rolling-update-deployment/cleanup.sh similarity index 100% rename from evals/tasks/rolling-update-deployment/cleanup.sh rename to evals/tasks/kubernetes/rolling-update-deployment/cleanup.sh diff --git a/evals/tasks/rolling-update-deployment/rolling-update-deployment.yaml b/evals/tasks/kubernetes/rolling-update-deployment/rolling-update-deployment.yaml similarity index 100% rename from evals/tasks/rolling-update-deployment/rolling-update-deployment.yaml rename to evals/tasks/kubernetes/rolling-update-deployment/rolling-update-deployment.yaml diff --git a/evals/tasks/rolling-update-deployment/setup.sh b/evals/tasks/kubernetes/rolling-update-deployment/setup.sh similarity index 100% rename from evals/tasks/rolling-update-deployment/setup.sh rename to evals/tasks/kubernetes/rolling-update-deployment/setup.sh diff --git a/evals/tasks/rolling-update-deployment/verify.sh b/evals/tasks/kubernetes/rolling-update-deployment/verify.sh similarity index 100% rename from evals/tasks/rolling-update-deployment/verify.sh rename to evals/tasks/kubernetes/rolling-update-deployment/verify.sh diff --git a/evals/tasks/scale-deployment/cleanup.sh b/evals/tasks/kubernetes/scale-deployment/cleanup.sh similarity index 100% rename from evals/tasks/scale-deployment/cleanup.sh rename to evals/tasks/kubernetes/scale-deployment/cleanup.sh diff --git a/evals/tasks/scale-deployment/scale-deployment.yaml b/evals/tasks/kubernetes/scale-deployment/scale-deployment.yaml similarity index 100% rename from evals/tasks/scale-deployment/scale-deployment.yaml rename to evals/tasks/kubernetes/scale-deployment/scale-deployment.yaml diff --git a/evals/tasks/scale-deployment/setup.sh b/evals/tasks/kubernetes/scale-deployment/setup.sh similarity index 100% rename from evals/tasks/scale-deployment/setup.sh rename to evals/tasks/kubernetes/scale-deployment/setup.sh diff --git a/evals/tasks/scale-deployment/verify.sh b/evals/tasks/kubernetes/scale-deployment/verify.sh similarity index 100% rename from evals/tasks/scale-deployment/verify.sh rename to evals/tasks/kubernetes/scale-deployment/verify.sh diff --git a/evals/tasks/scale-down-deployment/cleanup.sh b/evals/tasks/kubernetes/scale-down-deployment/cleanup.sh similarity index 100% rename from evals/tasks/scale-down-deployment/cleanup.sh rename to evals/tasks/kubernetes/scale-down-deployment/cleanup.sh diff --git a/evals/tasks/scale-down-deployment/scale-down-deployment.yaml b/evals/tasks/kubernetes/scale-down-deployment/scale-down-deployment.yaml similarity index 100% rename from evals/tasks/scale-down-deployment/scale-down-deployment.yaml rename to evals/tasks/kubernetes/scale-down-deployment/scale-down-deployment.yaml diff --git a/evals/tasks/scale-down-deployment/setup.sh b/evals/tasks/kubernetes/scale-down-deployment/setup.sh similarity index 100% rename from evals/tasks/scale-down-deployment/setup.sh rename to evals/tasks/kubernetes/scale-down-deployment/setup.sh diff --git a/evals/tasks/scale-down-deployment/verify.sh b/evals/tasks/kubernetes/scale-down-deployment/verify.sh similarity index 100% rename from evals/tasks/scale-down-deployment/verify.sh rename to evals/tasks/kubernetes/scale-down-deployment/verify.sh diff --git a/evals/tasks/setup-dev-cluster/cleanup.sh b/evals/tasks/kubernetes/setup-dev-cluster/cleanup.sh similarity index 100% rename from evals/tasks/setup-dev-cluster/cleanup.sh rename to evals/tasks/kubernetes/setup-dev-cluster/cleanup.sh diff --git a/evals/tasks/setup-dev-cluster/setup-dev-cluster.md b/evals/tasks/kubernetes/setup-dev-cluster/setup-dev-cluster.md similarity index 100% rename from evals/tasks/setup-dev-cluster/setup-dev-cluster.md rename to evals/tasks/kubernetes/setup-dev-cluster/setup-dev-cluster.md diff --git a/evals/tasks/setup-dev-cluster/setup-dev-cluster.yaml b/evals/tasks/kubernetes/setup-dev-cluster/setup-dev-cluster.yaml similarity index 100% rename from evals/tasks/setup-dev-cluster/setup-dev-cluster.yaml rename to evals/tasks/kubernetes/setup-dev-cluster/setup-dev-cluster.yaml diff --git a/evals/tasks/setup-dev-cluster/setup.sh b/evals/tasks/kubernetes/setup-dev-cluster/setup.sh similarity index 100% rename from evals/tasks/setup-dev-cluster/setup.sh rename to evals/tasks/kubernetes/setup-dev-cluster/setup.sh diff --git a/evals/tasks/setup-dev-cluster/verify.sh b/evals/tasks/kubernetes/setup-dev-cluster/verify.sh similarity index 100% rename from evals/tasks/setup-dev-cluster/verify.sh rename to evals/tasks/kubernetes/setup-dev-cluster/verify.sh diff --git a/evals/tasks/statefulset-lifecycle/cleanup.sh b/evals/tasks/kubernetes/statefulset-lifecycle/cleanup.sh similarity index 100% rename from evals/tasks/statefulset-lifecycle/cleanup.sh rename to evals/tasks/kubernetes/statefulset-lifecycle/cleanup.sh diff --git a/evals/tasks/statefulset-lifecycle/setup.sh b/evals/tasks/kubernetes/statefulset-lifecycle/setup.sh similarity index 100% rename from evals/tasks/statefulset-lifecycle/setup.sh rename to evals/tasks/kubernetes/statefulset-lifecycle/setup.sh diff --git a/evals/tasks/statefulset-lifecycle/statefulset-lifecycle.yaml b/evals/tasks/kubernetes/statefulset-lifecycle/statefulset-lifecycle.yaml similarity index 100% rename from evals/tasks/statefulset-lifecycle/statefulset-lifecycle.yaml rename to evals/tasks/kubernetes/statefulset-lifecycle/statefulset-lifecycle.yaml diff --git a/evals/tasks/statefulset-lifecycle/verify.sh b/evals/tasks/kubernetes/statefulset-lifecycle/verify.sh similarity index 100% rename from evals/tasks/statefulset-lifecycle/verify.sh rename to evals/tasks/kubernetes/statefulset-lifecycle/verify.sh