CRI: clarify purpose of annotations

Add language to make it explicit that annotations are not to be altered
by runtimes, and should only be used for features that are opaque to the
Kubernetes APIs. Unfortunately there are currently exceptions
introduced in [1][1], but this change makes it clear that they are to be
changed and that no more such semantic-affecting annotations should be
introduced.

In the spirit of the discussion and conclusion in [2][2].

Also captures the link between the annotations returned by various
status queries and those supplied in associated configs.

[1]: kubernetes#34819
[2]: kubernetes#30819 (comment)

Author: Jonathan Boulle

pkg/kubelet/api/v1alpha1/runtime/api.pb.go

@@ -234,14 +234,29 @@ message PodSandboxConfig {
     repeated PortMapping port_mappings = 5;
     // Key-value pairs that may be used to scope and select individual resources.
     map<string, string> labels = 6;
-    // Annotations is an unstructured key value map that may be set by external
-    // tools to store and retrieve arbitrary metadata. There are a few features are
-    // driven by annotations, Runtimes could support them optionally:
+    // Unstructured key-value map that may be set by the kubelet to store and
+    // retrieve arbitrary metadata. This will include any annotations set on a
+    // pod through the Kubernetes API.
+    //
+    // Annotations MUST NOT be altered by the runtime; the annotations stored
+    // here MUST be returned in the PodSandboxStatus associated with the pod
+    // this PodSandboxConfig creates.
+    //
+    // In general, in order to preserve a well-defined interface between the
+    // kubelet and the container runtime, annotations SHOULD NOT influence
+    // runtime behaviour. For legacy reasons, there are some annotations which
+    // currently explicitly break this rule, listed below; in future versions
+    // of the interface these will be promoted to typed features.
+    //
+    // Annotations can also be useful for runtime authors to experiment with
+    // new features that are opaque to the Kubernetes APIs (both user-facing
+    // and the CRI). Whenever possible, however, runtime authors SHOULD
+    // consider proposing new typed fields for any new features instead.
     //
     // 1. AppArmor
     //
     //    key: container.apparmor.security.beta.kubernetes.io/<container_name>
-    //    description: apparmor profile for the container.
+    //    description: apparmor profile for a container in this pod.
     //    value:
     //      * runtime/default: equivalent to not specifying a profile.
     //      * localhost/<profile_name>: profile loaded on the node
@@ -255,8 +270,8 @@ message PodSandboxConfig {
     //      value: see below.
     //
     //      key: security.alpha.kubernetes.io/seccomp/container/<container name>
-    //      description: the seccomp profile for the container (overides pod).
-    //      values: see below
+    //      description: the seccomp profile for the container (overrides pod).
+    //      value: see below
     //
     //      The value of seccomp is runtime agnostic:
     //      * runtime/default: the default profile for the container runtime
@@ -348,10 +363,12 @@ message PodSandboxStatus {
     optional PodSandboxNetworkStatus network = 5;
     // Linux-specific status to a pod sandbox.
     optional LinuxPodSandboxStatus linux = 6;
-    // Labels are key value pairs that may be used to scope and select individual resources.
+    // Labels are key-value pairs that may be used to scope and select individual resources.
     map<string, string> labels = 7;
-    // Annotations is an unstructured key value map that may be set by external
-    // tools to store and retrieve arbitrary metadata.
+    // Unstructured key-value map holding arbitrary metadata.
+    // Annotations MUST NOT be altered by the runtime; the value of this field
+    // MUST be identical to that of the corresponding PodSandboxConfig used to
+    // instantiate the pod sandbox this status represents.
     map<string, string> annotations = 8;
 }
 
@@ -391,8 +408,10 @@ message PodSandbox {
     optional int64 created_at = 4;
     // Labels of the PodSandbox.
     map<string, string> labels = 5;
-    // Annotations is an unstructured key value map that may be set by external
-    // tools to store and retrieve arbitrary metadata.
+    // Unstructured key-value map holding arbitrary metadata.
+    // Annotations MUST NOT be altered by the runtime; the value of this field
+    // MUST be identical to that of the corresponding PodSandboxConfig used to
+    // instantiate this PodSandbox.
     map<string, string> annotations = 6;
 }
 
@@ -551,8 +570,16 @@ message ContainerConfig {
     //     prefix ::= DNS_SUBDOMAIN
     //     name ::= DNS_LABEL
     map<string, string> labels = 9;
-    // Annotations is an unstructured key value map that may be set by external
-    // tools to store and retrieve arbitrary metadata.
+    // Unstructured key-value map that may be used by the kubelet to store and
+    // retrieve arbitrary metadata.
+    //
+    // Annotations MUST NOT be altered by the runtime; the annotations stored
+    // here MUST be returned in the ContainerStatus associated with the container
+    // this ContainerConfig creates.
+    //
+    // In general, in order to preserve a well-defined interface between the
+    // kubelet and the container runtime, annotations SHOULD NOT influence
+    // runtime behaviour.
     map<string, string> annotations = 10;
     // Path relative to PodSandboxConfig.LogDirectory for container to store
     // the log (STDOUT and STDERR) on the host.
@@ -665,8 +692,10 @@ message Container {
     optional int64 created_at = 7;
     // Key-value pairs that may be used to scope and select individual resources.
     map<string, string> labels = 8;
-    // Annotations is an unstructured key value map that may be set by external
-    // tools to store and retrieve arbitrary metadata.
+    // Unstructured key-value map holding arbitrary metadata.
+    // Annotations MUST NOT be altered by the runtime; the value of this field
+    // MUST be identical to that of the corresponding ContainerConfig used to
+    // instantiate this Container.
     map<string, string> annotations = 9;
 }
 
@@ -708,7 +737,10 @@ message ContainerStatus {
     optional string message = 11;
     // Key-value pairs that may be used to scope and select individual resources.
     map<string,string> labels = 12;
-    // Annotations is an unstructured key value map.
+    // Unstructured key-value map holding arbitrary metadata.
+    // Annotations MUST NOT be altered by the runtime; the value of this field
+    // MUST be identical to that of the corresponding ContainerConfig used to
+    // instantiate the Container this status represents.
     map<string,string> annotations = 13;
     // Mounts for the container.
     repeated Mount mounts = 14;