# ModelReplica Source: /reference/modelreplicas/ Apply instances as `apiVersion: modelplane.ai/v1alpha1`, `kind: ModelReplica`. ## Example ```yaml # ModelReplicas are composed automatically by ModelDeployment, with the # scheduler's placement resolved onto each member (nodePoolName, deviceRequests). # Avoid creating them directly. apiVersion: modelplane.ai/v1alpha1 kind: ModelReplica metadata: name: qwen3-8b-eks-l4-0 namespace: ml-team spec: clusterName: eks-l4 engines: - name: qwen3-8b members: - role: Standalone nodePoolName: gpu-l4 deviceRequests: - name: gpu count: 1 deviceClassName: gpu.nvidia.com template: spec: containers: - name: engine image: vllm/vllm-openai:v0.23.0 args: - --model=Qwen/Qwen3-8B ``` ## Definition The CompositeResourceDefinition this reference is generated from, with the complete OpenAPI schema, validation rules, and defaults: ```yaml apiVersion: apiextensions.crossplane.io/v2 kind: CompositeResourceDefinition metadata: name: modelreplicas.modelplane.ai spec: group: modelplane.ai names: categories: [crossplane, modelplane, models] kind: ModelReplica plural: modelreplicas shortNames: [mr] scope: Namespaced versions: - name: v1alpha1 served: true referenceable: true additionalPrinterColumns: - name: CLUSTER type: string description: The InferenceCluster the replica runs on. jsonPath: .spec.clusterName - name: SERVING type: string description: Whether at least one copy of every engine is ready and routed. jsonPath: .status.conditions[?(@.type=='ModelServing')].status schema: openAPIV3Schema: type: object required: [spec] properties: spec: type: object required: [clusterName, engines] # A ModelReplica is machine-generated by the ModelDeployment compose # function, but the API server still serves and accepts it, so these # rules let the consuming function (and its backends) trust the # resolved shape rather than re-checking it in code or crashing on a # malformed hand-written replica. PrefillDecode serving runs one # Prefill and one Decode engine, each marking its phase; serving is # optional and defaults to Unified, under which no engine may carry a # phase. x-kubernetes-validations: - rule: "!has(self.serving) || self.serving.mode != 'PrefillDecode' || size(self.engines) == 2" message: "PrefillDecode serving requires exactly two engines (one Prefill, one Decode)" - rule: "!has(self.serving) || self.serving.mode != 'PrefillDecode' || self.engines.exists_one(e, has(e.phase) && e.phase == 'Prefill')" message: "PrefillDecode serving requires exactly one engine with phase Prefill" - rule: "!has(self.serving) || self.serving.mode != 'PrefillDecode' || self.engines.exists_one(e, has(e.phase) && e.phase == 'Decode')" message: "PrefillDecode serving requires exactly one engine with phase Decode" - rule: "(has(self.serving) && self.serving.mode == 'PrefillDecode') || !self.engines.exists(e, has(e.phase))" message: "engine phase may only be set when serving.mode is PrefillDecode" # The ports of a serving pod: a Standalone member or a Leader. A Worker # doesn't serve, so these skip it. The serving port is serving.port, 8000 # by default. Keep in sync with apis/modeldeployments/definition.yaml. # - At most one container declares the serving port, where requests arrive. # - If a container beside the engine declares the serving port, the engine # declares its http port elsewhere, and if the engine's http port is # elsewhere, a container beside it declares the serving port. # - The engine may name the serving port only http. - rule: "self.engines.all(e, e.members.all(m, (has(m.role) && m.role == 'Worker') || !has(m.template.spec) || m.template.spec.containers.filter(c, has(c.ports) && c.ports.exists(p, p.containerPort == self.?serving.?port.orValue(8000))).size() <= 1))" message: "at most one container in a serving pod may declare the serving port (serving.port, 8000 by default)" - rule: "self.engines.all(e, e.members.all(m, (has(m.role) && m.role == 'Worker') || !has(m.template.spec) || !m.template.spec.containers.exists(c, c.name != 'engine' && has(c.ports) && c.ports.exists(p, p.containerPort == self.?serving.?port.orValue(8000))) || m.template.spec.containers.exists(c, c.name == 'engine' && has(c.ports) && c.ports.exists(p, has(p.name) && p.name == 'http' && p.containerPort != self.?serving.?port.orValue(8000)))))" message: "when another container declares the serving port, the engine container must declare its 'http' port on a different port" - rule: "self.engines.all(e, e.members.all(m, (has(m.role) && m.role == 'Worker') || !has(m.template.spec) || !m.template.spec.containers.exists(c, c.name == 'engine' && has(c.ports) && c.ports.exists(p, has(p.name) && p.name == 'http' && p.containerPort != self.?serving.?port.orValue(8000))) || m.template.spec.containers.exists(c, c.name != 'engine' && has(c.ports) && c.ports.exists(p, p.containerPort == self.?serving.?port.orValue(8000)))))" message: "when the engine container's 'http' port isn't the serving port, another container in its pod must declare the serving port" - rule: "self.engines.all(e, e.members.all(m, (has(m.role) && m.role == 'Worker') || !has(m.template.spec) || !m.template.spec.containers.exists(c, c.name == 'engine' && has(c.ports) && c.ports.exists(p, p.containerPort == self.?serving.?port.orValue(8000) && has(p.name) && p.name != 'http'))))" message: "the engine container may name the serving port only 'http'" properties: clusterName: type: string minLength: 1 description: >- Name of the InferenceCluster this replica is pinned to. Replicas are pinned at creation time. If the cluster is temporarily unavailable the replica stays pinned and the parent ModelDeployment surfaces the degraded state via its conditions. If the cluster is deleted entirely the parent ModelDeployment re-places the replica on another viable cluster. modelCacheRef: type: object required: [name] description: >- Optional reference to a ModelCache mounted into the engine pods. Inherited verbatim from the parent ModelDeployment. properties: name: type: string minLength: 1 serving: type: object x-kubernetes-validations: - rule: "!has(self.endpointPicker) || (has(self.router) && self.router == 'EndpointPicker')" message: "serving.endpointPicker may only be set when serving.router is EndpointPicker" - rule: "!has(self.router) || has(self.endpointPicker)" message: "serving.router EndpointPicker requires serving.endpointPicker with its template" - rule: "self.mode != 'PrefillDecode' || (has(self.router) && self.router == 'EndpointPicker')" message: "PrefillDecode serving requires serving.router EndpointPicker and serving.endpointPicker" description: >- Serving, inherited verbatim from the parent ModelDeployment. Both modes front the serving pods with an InferencePool and an endpoint picker, which endpointPicker can configure. PrefillDecode also role-labels the engine each marks with its phase. properties: mode: type: string enum: [Unified, PrefillDecode] default: Unified router: type: string enum: [EndpointPicker] port: type: integer minimum: 1 maximum: 65535 description: >- The port every serving pod receives requests on, where the InferencePool sends them. The engine serves its API there unless a container in front of it, such as a sidecar, declares this port. Defaults to 8000. endpointPicker: type: object required: [template] properties: template: type: object required: [spec] properties: spec: type: object required: [containers] properties: containers: type: array minItems: 1 maxItems: 1 x-kubernetes-list-type: map x-kubernetes-list-map-keys: [name] x-kubernetes-validations: - rule: "self.all(c, c.name == 'epp')" message: "the endpoint picker's container must be named 'epp'" items: type: object required: [name, image] properties: name: type: string minLength: 1 maxLength: 63 image: type: string minLength: 1 command: type: array items: type: string args: type: array items: type: string env: type: array items: type: object required: [name] properties: name: type: string value: type: string valueFrom: type: object properties: secretKeyRef: type: object required: [name, key] properties: name: type: string key: type: string optional: type: boolean configMapKeyRef: type: object required: [name, key] properties: name: type: string key: type: string optional: type: boolean fieldRef: type: object required: [fieldPath] properties: fieldPath: type: string apiVersion: type: string resources: type: object x-kubernetes-validations: - rule: "(!has(self.requests) || self.requests.all(k, k in ['cpu', 'memory', 'ephemeral-storage'])) && (!has(self.limits) || self.limits.all(k, k in ['cpu', 'memory', 'ephemeral-storage']))" message: "resources take only cpu, memory and ephemeral-storage; a member gets devices from its node pool's device requests" properties: requests: type: object maxProperties: 3 additionalProperties: anyOf: [{type: integer}, {type: string}] x-kubernetes-int-or-string: true limits: type: object maxProperties: 3 additionalProperties: anyOf: [{type: integer}, {type: string}] x-kubernetes-int-or-string: true imagePullSecrets: type: array items: type: object required: [name] properties: name: type: string config: type: object x-kubernetes-preserve-unknown-fields: true minProperties: 1 engines: type: array description: >- The replica's inference engines, mirroring the parent ModelDeployment's engines with the scheduler's placement resolved onto each member: the pool the scheduler pinned the member to (nodePoolName) and the DRA device requests resolved from that pool's InferenceClass devices (deviceRequests). The members of one replica are co-scheduled onto a single cluster, but may land on different pools - the scheduler prefers one pool per engine and splits members only when no pool satisfies them all. minItems: 1 maxItems: 8 x-kubernetes-list-type: map x-kubernetes-list-map-keys: [name] items: type: object required: [name, members] # An engine is exactly one Standalone member, or one Leader # and one Worker member - the two shapes the backends compose # (a Deployment, or a LeaderWorkerSet or Grove PodCliqueSet # depending on the cluster's stack). A multi-node gang's # size lives in the Worker member's worker.nodes, not in # repeated Worker members, so members caps at two. # select_backend switches on Standalone-present, and the # Grove backend dereferences the Leader and Worker members, # so any other role combination would crash or compose the # wrong workload. # # The scheduler also only places an engine where at least one # member resolves a claimable device; an engine where no # member carries deviceRequests would compose pods with no # ResourceClaim at all - no GPUs anywhere. x-kubernetes-validations: - rule: >- (self.members.size() == 1 && self.members[0].role == 'Standalone') || (self.members.exists_one(m, m.role == 'Leader') && self.members.exists(m, m.role == 'Worker') && !self.members.exists(m, m.role == 'Standalone')) message: >- an engine must be either a single Standalone member or one Leader and one Worker member - rule: "self.members.exists(m, has(m.deviceRequests))" message: >- at least one member of an engine must carry deviceRequests properties: name: type: string minLength: 1 maxLength: 63 copies: type: integer default: 1 minimum: 1 maximum: 64 phase: type: string enum: [Prefill, Decode] members: type: array minItems: 1 maxItems: 2 x-kubernetes-list-type: atomic items: type: object required: [nodePoolName, template] # worker is meaningful only for a Worker member: a # Standalone and a Leader are always exactly one pod on # one node. The llm-d backend reads worker.nodes to size # the gang, so worker on any other role is uncomposable. x-kubernetes-validations: - rule: "!has(self.worker) || self.role == 'Worker'" message: "worker may only be set on a Worker member" - rule: "!has(self.role) || self.role != 'Worker' || !has(self.template.spec) || self.template.spec.containers.all(c, c.name != 'engine' || !has(c.readinessProbe) || ((!has(c.readinessProbe.httpGet) || type(c.readinessProbe.httpGet.port) != string || (has(c.ports) && c.ports.exists(p, has(p.name) && p.name == c.readinessProbe.httpGet.port))) && (!has(c.readinessProbe.tcpSocket) || type(c.readinessProbe.tcpSocket.port) != string || (has(c.ports) && c.ports.exists(p, has(p.name) && p.name == c.readinessProbe.tcpSocket.port)))))" message: "a Worker's engine readinessProbe may name only a port its engine container declares; Modelplane gives a Worker no http port" properties: role: type: string enum: [Standalone, Leader, Worker] default: Standalone nodePoolName: type: string minLength: 1 description: >- Name of the node pool on the pinned InferenceCluster the scheduler selected for this member. The scheduler pins every member to a specific pool, so this is always set - a member with no device requests of its own is pinned to its engine's pool. deviceRequests: type: array description: >- Resolved DRA device requests for the member's matched pool. The parent ModelDeployment's compose function joins the member's nodeSelector requests with the matched InferenceClass devices and stamps the claim: DRA devices here. This function turns each into a DeviceRequest in the member's DRA ResourceClaimTemplate. Absent when the member claims no devices - it carried no nodeSelector, or its requests matched only synthetic devices - in which case its pods get no ResourceClaim and only the pool pin places them. Never present and empty. At least one member of each engine always carries requests: the scheduler only places an engine where some member yields a claimable device. minItems: 1 maxItems: 16 x-kubernetes-list-type: map x-kubernetes-list-map-keys: [name] items: type: object required: [name, deviceClassName] properties: name: type: string description: >- Request name; becomes the DeviceRequest name. minLength: 1 maxLength: 63 deviceClassName: type: string description: >- Cluster-scoped DRA DeviceClass to claim through, from the matched InferenceClass device. minLength: 1 maxLength: 253 count: type: integer description: How many devices to claim. default: 1 minimum: 1 maximum: 64 selectors: type: array description: >- DRA CEL selectors copied verbatim from the nodeSelector request, ANDed in the DeviceRequest. maxItems: 8 x-kubernetes-list-type: atomic items: type: object # A selector must carry a selector kind # (today only cel). An empty {} selector would # match any device, silently widening the DRA # claim the serving pod binds through. minProperties: 1 properties: cel: type: string minLength: 1 maxLength: 10240 worker: type: object # No schema default, mirroring the ModelDeployment # XRD: defaults apply before CEL validation there, # and the composed member should round-trip the # deployment member unchanged. required: [nodes] properties: nodes: type: integer minimum: 1 maximum: 63 template: type: object properties: metadata: type: object # Mirrors the ModelDeployment member metadata # rules. Replicas are composed from an # already-validated deployment, so this is a # backstop for directly authored ModelReplicas; # keeping the schemas identical preserves the # round-trip contract noted on worker above. x-kubernetes-validations: - rule: "!has(self.labels) || self.labels.all(k, !k.startsWith('modelplane.ai/'))" message: "member template.metadata.labels keys must not use the reserved modelplane.ai/ prefix" - rule: "!has(self.labels) || self.labels.all(k, k.matches('^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\\\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?[A-Za-z0-9]([-A-Za-z0-9_.]{0,61}[A-Za-z0-9])?$'))" message: "member template.metadata.labels keys must be valid Kubernetes label keys" - rule: "!has(self.labels) || self.labels.all(k, k.split('/')[0].size() <= 253)" message: "member template.metadata.labels key prefixes must be DNS subdomains of at most 253 characters" - rule: "!has(self.labels) || self.labels.all(k, self.labels[k].matches('^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$'))" message: "member template.metadata.labels values must be valid Kubernetes label values" - rule: "!has(self.annotations) || self.annotations.all(k, k.matches('^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\\\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?[A-Za-z0-9]([-A-Za-z0-9_.]{0,61}[A-Za-z0-9])?$'))" message: "member template.metadata.annotations keys must be valid Kubernetes annotation keys" - rule: "!has(self.annotations) || self.annotations.all(k, k.split('/')[0].size() <= 253)" message: "member template.metadata.annotations key prefixes must be DNS subdomains of at most 253 characters" properties: labels: type: object maxProperties: 32 additionalProperties: type: string maxLength: 63 annotations: type: object maxProperties: 32 additionalProperties: type: string maxLength: 8192 spec: type: object required: [containers] properties: containers: type: array minItems: 1 maxItems: 8 x-kubernetes-list-type: map x-kubernetes-list-map-keys: [name] x-kubernetes-validations: - rule: "self.exists_one(c, c.name == 'engine')" message: "exactly one container must be named 'engine'" items: type: object required: [name, image] properties: name: type: string minLength: 1 maxLength: 63 pattern: '^[a-z0-9]([-a-z0-9]*[a-z0-9])?$' image: type: string minLength: 1 command: type: array items: type: string args: type: array items: type: string env: type: array items: type: object required: [name] properties: name: type: string value: type: string valueFrom: type: object properties: secretKeyRef: type: object required: [name, key] properties: name: type: string key: type: string optional: type: boolean configMapKeyRef: type: object required: [name, key] properties: name: type: string key: type: string optional: type: boolean fieldRef: type: object description: >- Reference a pod field via the downward API, e.g. status.podIP, metadata.name, or metadata.namespace. required: [fieldPath] properties: fieldPath: type: string apiVersion: type: string resources: type: object description: >- CPU and memory requests and limits. On the engine container, Modelplane adds the member's device claim beside them. x-kubernetes-validations: - rule: "(!has(self.requests) || self.requests.all(k, k in ['cpu', 'memory', 'ephemeral-storage'])) && (!has(self.limits) || self.limits.all(k, k in ['cpu', 'memory', 'ephemeral-storage']))" message: "resources take only cpu, memory and ephemeral-storage; a member gets devices from its node pool's device requests" properties: requests: type: object maxProperties: 3 additionalProperties: anyOf: [{type: integer}, {type: string}] x-kubernetes-int-or-string: true limits: type: object maxProperties: 3 additionalProperties: anyOf: [{type: integer}, {type: string}] x-kubernetes-int-or-string: true readinessProbe: type: object description: >- How Kubernetes checks that the container is ready. Without one, Modelplane probes the engine container of a Standalone or Leader member with GET /health on its http port. A probe set here replaces that default whole, and Kubernetes defaults apply to any field it leaves out. A Worker has no http port, so its engine's probe may name only a port the engine container declares. x-kubernetes-validations: - rule: "[has(self.httpGet), has(self.tcpSocket), has(self.grpc)].filter(h, h).size() == 1" message: "readinessProbe sets exactly one of httpGet, tcpSocket and grpc" properties: httpGet: type: object required: [port] x-kubernetes-validations: - rule: "type(self.port) == string || (self.port >= 1 && self.port <= 65535)" message: "httpGet.port must be a port name or a number from 1 to 65535" properties: path: type: string pattern: '^/' port: anyOf: [{type: integer}, {type: string}] x-kubernetes-int-or-string: true tcpSocket: type: object required: [port] x-kubernetes-validations: - rule: "type(self.port) == string || (self.port >= 1 && self.port <= 65535)" message: "tcpSocket.port must be a port name or a number from 1 to 65535" properties: port: anyOf: [{type: integer}, {type: string}] x-kubernetes-int-or-string: true grpc: type: object required: [port] properties: port: type: integer minimum: 1 maximum: 65535 service: type: string initialDelaySeconds: type: integer minimum: 0 periodSeconds: type: integer minimum: 1 timeoutSeconds: type: integer minimum: 1 successThreshold: type: integer minimum: 1 failureThreshold: type: integer minimum: 1 ports: type: array maxItems: 8 x-kubernetes-list-type: map x-kubernetes-list-map-keys: [containerPort] x-kubernetes-validations: - rule: "self.all(p, !has(p.name) || self.exists_one(q, has(q.name) && q.name == p.name))" message: "port names must be unique within a container" items: type: object required: [containerPort] properties: name: type: string minLength: 1 maxLength: 15 pattern: '^[a-z0-9]([-a-z0-9]*[a-z0-9])?$' containerPort: type: integer minimum: 1 maximum: 65535 imagePullSecrets: type: array items: type: object required: [name] properties: name: type: string status: type: object ```