Skip to content

Commit ec34222

Browse files
klihubklihub
committed
device-injector: add support for CDI injection.
Add support for injecting annotated CDI devices using the new native NRI CDI injection API. Signed-off-by: Krisztian Litkey <[email protected]>
1 parent 646502f commit ec34222

File tree

5 files changed

+439
-77
lines changed

5 files changed

+439
-77
lines changed

‎plugins/device-injector/README.md‎

Lines changed: 38 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,19 @@
11
## Device Injector Plugin
22

3-
This sample plugin can inject devices and mounts into containers using
4-
pod annotations.
3+
This sample plugin can inject Linux device nodes, CDI devices, and mounts into
4+
containers using pod annotations.
55

66
### Device Annotations
77

88
Devices are annotated using the `devices.nri.io` annotation key prefix.
99
The key `devices.nri.io/container.$CONTAINER_NAME` annotates devices to
1010
be injected into `$CONTAINER_NAME`. The keys `devices.nri.io` and
11-
`devices.nri.io/pod` annotate devices to be injected into all containers.
11+
`devices.nri.io/pod` annotate devices to be injected into containers
12+
without any other, container-specific device annotations. Only one of
13+
these latter two annotations will be ever taken into account. If both are
14+
present, it is unspecified which one is used.
1215

13-
The annotation syntax for device injection is
16+
The annotation value syntax for device injection is
1417

1518
```
1619
- path: /dev/dev0
@@ -26,10 +29,40 @@ The annotation syntax for device injection is
2629

2730
`file_mode`, `uid` and `gid` can be omitted, the rest are mandatory.
2831

32+
### CDI Device Annotations
33+
34+
CDI devices are annotated in a similar manner to devices, but using the
35+
`cdi-devices.nri.io` annotation key prefix. The annotation value for CDI
36+
devices is the list of CDI device names to inject.
37+
38+
For instance, the following annotation
39+
40+
```
41+
metadata:
42+
name: bash
43+
annotations:
44+
cdi-devices.nri.io/container.c0: |
45+
- vendor0.com/device=null
46+
cdi-devices.nri.io/container.c1: |
47+
- vendor0.com/device=zero
48+
cdi-devices.nri.io/container.c2: |
49+
- vendor0.com/device=dev0
50+
- vendor1.com/device=dev0
51+
- vendor1.com/device=dev1
52+
cdi-devices.nri.io/container.mgmt: |
53+
- vendor0.com/device=all
54+
```
55+
56+
requests the injection of the CDI device vendor0.com/device=null to container
57+
c0, the injection of the CDI device vendor0.com/device=zero to container c1,
58+
the injection of the CDI devices vendor0.com/device=dev0, vendor1.com/device=dev0
59+
and vendor1.com/device=dev1 to container c2, and the injection of the CDI device
60+
vendor0.com/device=all to container mgmt.
61+
2962
### Mount Annotations
3063

3164
Mounts are annotated in a similar manner to devices, but using the
32-
`mounts.nri.io` annotation key prefix. The annotation syntax for mount
65+
`mounts.nri.io` annotation key prefix. The annotation value syntax for mount
3366
injection is
3467

3568
```

‎plugins/device-injector/device-injector.go‎

Lines changed: 126 additions & 72 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,8 @@ const (
3535
deviceKey = "devices.nri.io"
3636
// Prefix of the key used for mount annotations.
3737
mountKey = "mounts.nri.io"
38+
// Prefix of the key used for CDI device annotations.
39+
cdiDeviceKey = "cdi-devices.nri.io"
3840
)
3941

4042
var (
@@ -67,131 +69,183 @@ type plugin struct {
6769
}
6870

6971
// CreateContainer handles container creation requests.
70-
func (p *plugin) CreateContainer(_ context.Context, pod *api.PodSandbox, container *api.Container) (*api.ContainerAdjustment, []*api.ContainerUpdate, error) {
71-
var (
72-
ctrName string
73-
devices []device
74-
mounts []mount
75-
err error
76-
)
77-
78-
ctrName = containerName(pod, container)
79-
72+
func (p *plugin) CreateContainer(_ context.Context, pod *api.PodSandbox, ctr *api.Container) (*api.ContainerAdjustment, []*api.ContainerUpdate, error) {
8073
if verbose {
81-
dump("CreateContainer", "pod", pod, "container", container)
74+
dump("CreateContainer", "pod", pod, "container", ctr)
8275
}
8376

8477
adjust := &api.ContainerAdjustment{}
8578

86-
// inject devices to container
87-
devices, err = parseDevices(container.Name, pod.Annotations)
88-
if err != nil {
79+
if err := injectDevices(pod, ctr, adjust); err != nil {
8980
return nil, nil, err
9081
}
9182

92-
if len(devices) == 0 {
93-
log.Infof("%s: no devices annotated...", ctrName)
94-
} else {
95-
if verbose {
96-
dump(ctrName, "annotated devices", devices)
97-
}
98-
99-
for _, d := range devices {
100-
adjust.AddDevice(d.toNRI())
101-
if !verbose {
102-
log.Infof("%s: injected device %q...", ctrName, d.Path)
103-
}
104-
}
83+
if err := injectCDIDevices(pod, ctr, adjust); err != nil {
84+
return nil, nil, err
10585
}
10686

107-
// inject mounts to container
108-
mounts, err = parseMounts(container.Name, pod.Annotations)
109-
if err != nil {
87+
if err := injectMounts(pod, ctr, adjust); err != nil {
11088
return nil, nil, err
11189
}
11290

113-
if len(mounts) == 0 {
114-
log.Infof("%s: no mounts annotated...", ctrName)
115-
} else {
116-
if verbose {
117-
dump(ctrName, "annotated mounts", mounts)
118-
}
91+
return adjust, nil, nil
11992

120-
for _, m := range mounts {
121-
adjust.AddMount(m.toNRI())
122-
if !verbose {
123-
log.Infof("%s: injected mount %q -> %q...", ctrName, m.Source, m.Destination)
124-
}
125-
}
93+
if verbose {
94+
dump(containerName(pod, ctr), "ContainerAdjustment", adjust)
95+
}
96+
97+
return adjust, nil, nil
98+
}
99+
100+
func injectDevices(pod *api.PodSandbox, ctr *api.Container, a *api.ContainerAdjustment) error {
101+
devices, err := parseDevices(ctr.Name, pod.Annotations)
102+
if err != nil {
103+
return err
104+
}
105+
106+
if len(devices) == 0 {
107+
log.Debugf("%s: no devices annotated...", containerName(pod, ctr))
108+
return nil
126109
}
127110

128111
if verbose {
129-
dump(ctrName, "ContainerAdjustment", adjust)
112+
dump(containerName(pod, ctr), "annotated devices", devices)
130113
}
131114

132-
return adjust, nil, nil
115+
for _, d := range devices {
116+
a.AddDevice(d.toNRI())
117+
if !verbose {
118+
log.Infof("%s: injected device %q...", containerName(pod, ctr), d.Path)
119+
}
120+
}
121+
122+
return nil
133123
}
134124

135125
func parseDevices(ctr string, annotations map[string]string) ([]device, error) {
136126
var (
137-
key string
138-
annotation []byte
139-
devices []device
127+
devices []device
140128
)
141129

142-
// look up effective device annotation and unmarshal devices
143-
for _, key = range []string{
144-
deviceKey + "/container." + ctr,
145-
deviceKey + "/pod",
146-
deviceKey,
147-
} {
148-
if value, ok := annotations[key]; ok {
149-
annotation = []byte(value)
150-
break
151-
}
130+
annotation := getAnnotation(annotations, deviceKey, ctr)
131+
if annotation == nil {
132+
return nil, nil
152133
}
153134

154135
if annotation == nil {
155136
return nil, nil
156137
}
157138

158139
if err := yaml.Unmarshal(annotation, &devices); err != nil {
159-
return nil, fmt.Errorf("invalid device annotation %q: %w", key, err)
140+
return nil, fmt.Errorf("invalid device annotation %q: %w", string(annotation), err)
160141
}
161142

162143
return devices, nil
163144
}
164145

165-
func parseMounts(ctr string, annotations map[string]string) ([]mount, error) {
146+
func injectCDIDevices(pod *api.PodSandbox, ctr *api.Container, a *api.ContainerAdjustment) error {
147+
devices, err := parseCDIDevices(ctr.Name, pod.Annotations)
148+
if err != nil {
149+
return err
150+
}
151+
152+
if len(devices) == 0 {
153+
log.Debugf("%s: no CDI devices annotated...", containerName(pod, ctr))
154+
return nil
155+
}
156+
157+
if verbose {
158+
dump(containerName(pod, ctr), "annotated CDI devices", devices)
159+
}
160+
161+
for _, name := range devices {
162+
a.AddCDIDevice(
163+
&api.CDIDevice{
164+
Name: name,
165+
},
166+
)
167+
if !verbose {
168+
log.Infof("%s: injected CDI device %q...", containerName(pod, ctr), name)
169+
}
170+
}
171+
172+
return nil
173+
}
174+
175+
func parseCDIDevices(ctr string, annotations map[string]string) ([]string, error) {
166176
var (
167-
key string
168-
annotation []byte
169-
mounts []mount
177+
cdiDevices []string
170178
)
171179

172-
// look up effective device annotation and unmarshal devices
173-
for _, key = range []string{
174-
mountKey + "/container." + ctr,
175-
mountKey + "/pod",
176-
mountKey,
177-
} {
178-
if value, ok := annotations[key]; ok {
179-
annotation = []byte(value)
180-
break
180+
annotation := getAnnotation(annotations, cdiDeviceKey, ctr)
181+
if annotation == nil {
182+
return nil, nil
183+
}
184+
185+
if err := yaml.Unmarshal(annotation, &cdiDevices); err != nil {
186+
return nil, fmt.Errorf("invalid CDI device annotation %q: %w", string(annotation), err)
187+
}
188+
189+
return cdiDevices, nil
190+
}
191+
192+
func injectMounts(pod *api.PodSandbox, ctr *api.Container, a *api.ContainerAdjustment) error {
193+
mounts, err := parseMounts(ctr.Name, pod.Annotations)
194+
if err != nil {
195+
return err
196+
}
197+
198+
if len(mounts) == 0 {
199+
log.Debugf("%s: no mounts annotated...", containerName(pod, ctr))
200+
return nil
201+
}
202+
203+
if verbose {
204+
dump(containerName(pod, ctr), "annotated mounts", mounts)
205+
}
206+
207+
for _, m := range mounts {
208+
a.AddMount(m.toNRI())
209+
if !verbose {
210+
log.Infof("%s: injected mount %q -> %q...", containerName(pod, ctr),
211+
m.Source, m.Destination)
181212
}
182213
}
183214

215+
return nil
216+
}
217+
218+
func parseMounts(ctr string, annotations map[string]string) ([]mount, error) {
219+
var (
220+
mounts []mount
221+
)
222+
223+
annotation := getAnnotation(annotations, mountKey, ctr)
184224
if annotation == nil {
185225
return nil, nil
186226
}
187227

188228
if err := yaml.Unmarshal(annotation, &mounts); err != nil {
189-
return nil, fmt.Errorf("invalid mount annotation %q: %w", key, err)
229+
return nil, fmt.Errorf("invalid mount annotation %q: %w", string(annotation), err)
190230
}
191231

192232
return mounts, nil
193233
}
194234

235+
func getAnnotation(annotations map[string]string, mainKey, ctr string) []byte {
236+
for _, key := range []string{
237+
mainKey + "/container." + ctr,
238+
mainKey + "/pod",
239+
mainKey,
240+
} {
241+
if value, ok := annotations[key]; ok {
242+
return []byte(value)
243+
}
244+
}
245+
246+
return nil
247+
}
248+
195249
// Convert a device to the NRI API representation.
196250
func (d *device) toNRI() *api.LinuxDevice {
197251
apiDev := &api.LinuxDevice{

0 commit comments

Comments
 (0)