[DO NOT MERGE] Use OCI store as blob cache

.github/workflows/build-deploy.yaml

@@ -5,6 +5,7 @@ on:
   push:
     branches:
     - main
+    - test
   workflow_dispatch:
 
 jobs:

.github/workflows/helm.yaml

@@ -45,4 +45,4 @@ jobs:
         PKG_RESPONSE=$(helm package ./charts)
         echo "$PKG_RESPONSE"
         CHART_TAR_GZ=$(basename "$PKG_RESPONSE") 
-        helm push "$CHART_TAR_GZ" oci://ghcr.io/converged-computing/oras-csi-helm
+        helm push "$CHART_TAR_GZ" oci://ghcr.io/billy-playground/oras-csi-helm

Makefile

@@ -3,7 +3,7 @@ DEVTAG=$(DRIVER_VERSION)-dev
 HELM_PLUGIN_NAME=oras-csi
 BATS := bats
 NAME=oras-csi-plugin
-DOCKER_REGISTRY=ghcr.io/converged-computing
+DOCKER_REGISTRY=ghcr.io/billy-playground
 
 .PHONY: help
 help: ## Generates help for all targets

charts/values.yaml

@@ -16,7 +16,7 @@ node:
     env:
       csiEndpoint: unix:///csi/csi.sock
     image:
-      repository: ghcr.io/converged-computing/oras-csi-plugin
+      repository: ghcr.io/billy-playground/oras-csi-plugin
       tag: latest
     imagePullPolicy: Always
   driverRegistrar:

cmd/oras-csi-plugin/main.go

@@ -3,8 +3,8 @@ package main
 import (
 	"flag"
 
-	"github.com/converged-computing/oras-csi/pkg/driver"
-	"github.com/converged-computing/oras-csi/pkg/oras"
+	"github.com/billy-playground/oras-csi/pkg/driver"
+	"github.com/billy-playground/oras-csi/pkg/oras"
 	log "github.com/sirupsen/logrus"
 )
 

deploy/dev-driver.yaml

@@ -161,7 +161,7 @@ spec:
             capabilities:
               add: ["SYS_ADMIN"]
             allowPrivilegeEscalation: true
-          image: ghcr.io/converged-computing/oras-csi-plugin:0.1.0-dev
+          image: ghcr.io/billy-playground/oras-csi-plugin:0.1.0-dev
           args:
             - "--mode=node"
             - "--csi-endpoint=$(CSI_ENDPOINT)"

deploy/driver-csi-oras.yaml

@@ -161,7 +161,7 @@ spec:
             capabilities:
               add: ["SYS_ADMIN"]
             allowPrivilegeEscalation: true
-          image: ghcr.io/converged-computing/oras-csi-plugin:latest
+          image: ghcr.io/billy-playground/oras-csi-plugin:latest
           args:
             - "--mode=node"
             - "--csi-endpoint=$(CSI_ENDPOINT)"

docs/usage.md

@@ -130,30 +130,37 @@ $ kubectl logs -n kube-system csi-oras-node-bb7mw csi-oras-plugin -f
 <summary>More verbose output</summary>
 
 ```console
-time="2023-04-15T03:31:19Z" level=info msg="Preparing artifact cache (mode: node; node-id: minikube; root-dir: /; plugin-data-dir: pv_data enforce-namespaces: %!s(bool=true))"
-time="2023-04-15T03:31:19Z" level=info msg="NewNodeService creation (rootDir /, pluginDataDir pv_data, nodeId minikube, handlersCount 1)"
-time="2023-04-15T03:31:19Z" level=info msg="StartService - endpoint unix:///csi/csi.sock"
-time="2023-04-15T03:31:19Z" level=info msg=CreategRPCServer
-time="2023-04-15T03:31:19Z" level=info msg="CreateListener - endpoint unix:///csi/csi.sock"
-time="2023-04-15T03:31:19Z" level=info msg="CreateListener - Removing socket /csi/csi.sock"
-time="2023-04-15T03:31:19Z" level=info msg="StartService - Registering node service"
-time="2023-04-15T03:31:19Z" level=info msg="StartService - Starting to serve!"
-time="2023-04-15T03:31:20Z" level=info msg=GetPluginInfo
-time="2023-04-15T03:31:22Z" level=info msg=NodeGetInfo
-time="2023-04-15T03:32:26Z" level=info msg="NodePublishVolume - VolumeId: csi-16c0ab68018efd3a4f540655a119f3af7955bdd0d3f8d0882ef749757e154d0d, Readonly: true, VolumeContext map[csi.storage.k8s.io/ephemeral:true csi.storage.k8s.io/pod.name:my-csi-app-inline csi.storage.k8s.io/pod.namespace:default csi.storage.k8s.io/pod.uid:47933acc-6ba8-4e11-b7ba-71837f6cd0ea csi.storage.k8s.io/serviceAccount.name:default oras.artifact.reference:ghcr.io/singularityhub/github-ci:latest], PublishContext map[], VolumeCapability mount:<> access_mode:<mode:SINGLE_NODE_WRITER >  TargetPath /var/lib/kubelet/pods/47933acc-6ba8-4e11-b7ba-71837f6cd0ea/volumes/kubernetes.io~csi/oras-inline/mount"
-time="2023-04-15T03:32:26Z" level=info msg="Looking for volume context...."
-time="2023-04-15T03:32:26Z" level=info msg="map[csi.storage.k8s.io/ephemeral:true csi.storage.k8s.io/pod.name:my-csi-app-inline csi.storage.k8s.io/pod.namespace:default csi.storage.k8s.io/pod.uid:47933acc-6ba8-4e11-b7ba-71837f6cd0ea csi.storage.k8s.io/serviceAccount.name:default oras.artifact.reference:ghcr.io/singularityhub/github-ci:latest]"
-time="2023-04-15T03:32:27Z" level=info msg="volume source directory:/pv_data/ghcr-io-singularityhub-github-ci-latest"
-time="2023-04-15T03:32:27Z" level=info msg="volume target directory:/var/lib/kubelet/pods/47933acc-6ba8-4e11-b7ba-71837f6cd0ea/volumes/kubernetes.io~csi/oras-inline/mount"
-time="2023-04-15T03:32:27Z" level=info msg="volume options:[ro]"
-time="2023-04-15T03:33:01Z" level=info msg="NodeUnpublishVolume - VolumeId: csi-16c0ab68018efd3a4f540655a119f3af7955bdd0d3f8d0882ef749757e154d0d, TargetPath: /var/lib/kubelet/pods/47933acc-6ba8-4e11-b7ba-71837f6cd0ea/volumes/kubernetes.io~csi/oras-inline/mount)"
-time="2023-04-15T03:33:08Z" level=info msg="NodePublishVolume - VolumeId: csi-c53cdb53dc3045deec22489a48716a714dd6b2beef2dc2657234b317b92e93bb, Readonly: true, VolumeContext map[csi.storage.k8s.io/ephemeral:true csi.storage.k8s.io/pod.name:my-csi-app-inline csi.storage.k8s.io/pod.namespace:default csi.storage.k8s.io/pod.uid:e0dfd3c8-66e4-4a8c-8501-99cf50bb094e csi.storage.k8s.io/serviceAccount.name:default oras.artifact.reference:ghcr.io/singularityhub/github-ci:latest], PublishContext map[], VolumeCapability mount:<> access_mode:<mode:SINGLE_NODE_WRITER >  TargetPath /var/lib/kubelet/pods/e0dfd3c8-66e4-4a8c-8501-99cf50bb094e/volumes/kubernetes.io~csi/oras-inline/mount"
-time="2023-04-15T03:33:08Z" level=info msg="Looking for volume context...."
-time="2023-04-15T03:33:08Z" level=info msg="map[csi.storage.k8s.io/ephemeral:true csi.storage.k8s.io/pod.name:my-csi-app-inline csi.storage.k8s.io/pod.namespace:default csi.storage.k8s.io/pod.uid:e0dfd3c8-66e4-4a8c-8501-99cf50bb094e csi.storage.k8s.io/serviceAccount.name:default oras.artifact.reference:ghcr.io/singularityhub/github-ci:latest]"
-time="2023-04-15T03:33:08Z" level=info msg="volume source directory:/pv_data/ghcr-io-singularityhub-github-ci-latest"
-time="2023-04-15T03:33:08Z" level=info msg="volume target directory:/var/lib/kubelet/pods/e0dfd3c8-66e4-4a8c-8501-99cf50bb094e/volumes/kubernetes.io~csi/oras-inline/mount"
-time="2023-04-15T03:33:08Z" level=info msg="volume options:[ro]"
+time="2023-05-17T09:59:58Z" level=info msg="Preparing artifact cache (mode: node; node-id: oras-csi-control-plane; root-dir: /; plugin-data-dir: pv_data enforce-namespaces: true)"
+time="2023-05-17T09:59:58Z" level=info msg="NewNodeService creation (rootDir /, pluginDataDir pv_data, nodeId oras-csi-control-plane, handlersCount 1)"
+time="2023-05-17T09:59:58Z" level=info msg="Setting up ORAS Logging. ORAS path: /pv_data/logs"
+time="2023-05-17T09:59:58Z" level=info msg="ORAS Logging set up!"
+time="2023-05-17T09:59:58Z" level=info msg="StartService - endpoint unix:///csi/csi.sock"
+time="2023-05-17T09:59:58Z" level=info msg=CreategRPCServer
+time="2023-05-17T09:59:58Z" level=info msg="CreateListener - endpoint unix:///csi/csi.sock"
+time="2023-05-17T09:59:58Z" level=info msg="CreateListener - Removing socket /csi/csi.sock"
+time="2023-05-17T09:59:58Z" level=info msg="StartService - Registering node service"
+time="2023-05-17T09:59:58Z" level=info msg="StartService - Starting to serve!"
+time="2023-05-17T09:59:58Z" level=info msg=GetPluginInfo
+time="2023-05-17T09:59:58Z" level=info msg=NodeGetInfo
+time="2023-05-17T10:01:49Z" level=info msg="NodePublishVolume - VolumeId: csi-b2e6bcaddfcc84d4434e8bc126bf2a225009e95cef2215e57a1b4e1f277bf900, Readonly: true, VolumeCapability mount:<> access_mode:<mode:SINGLE_NODE_WRITER >  TargetPath /var/lib/kubelet/pods/44bb7b41-7854-4bdf-a2c5-d3552aee35f5/volumes/kubernetes.io~csi/oras-inline/mount"
+time="2023-05-17T10:01:49Z" level=info msg="Looking for volume context...."
+time="2023-05-17T10:01:49Z" level=info msg="Oras - container: ghcr.io/singularityhub/github-ci, target: /mnt/oras-csi-control-plane"
+time="2023-05-17T10:01:49Z" level=info msg="Enforce namespaces: true"
+time="2023-05-17T10:01:49Z" level=info msg="Enforcing artifact namespace to be under default"
+time="2023-05-17T10:01:49Z" level=info msg="Remote repository ghcr.io/singularityhub/github-ci:latest will be proxied by /pv_data/default/ghcr-io-singularityhub-github-ci-latest"
+time="2023-05-17T10:01:49Z" level=info msg="Resolving manifest descriptor for ghcr.io/singularityhub/github-ci:latest"
+time="2023-05-17T10:01:50Z" level=info msg="Fetching manifest {\"MediaType\":\"application/vnd.oci.image.manifest.v1+json\",\"Digest\":\"sha256:5d6742ff0b10c1196202765dafb43275259bcbdbd3868c19ba1d19476c088867\",\"Size\":402}"
+time="2023-05-17T10:01:50Z" level=info msg="Uncached fetching : {\"MediaType\":\"application/vnd.oci.image.manifest.v1+json\",\"Digest\":\"sha256:5d6742ff0b10c1196202765dafb43275259bcbdbd3868c19ba1d19476c088867\",\"Size\":402}"
+time="2023-05-17T10:01:50Z" level=info msg="Pulling sha256:acb1ec674e686f4ba7a0e5c0ce1d41b6c2a5f5f1b9b9baca9c612f794faa3f8e, 1 of 1"
+time="2023-05-17T10:01:50Z" level=info msg="Uncached fetching : {\"MediaType\":\"application/vnd.sylabs.sif.layer.v1.sif\",\"Digest\":\"sha256:acb1ec674e686f4ba7a0e5c0ce1d41b6c2a5f5f1b9b9baca9c612f794faa3f8e\",\"Size\":798720}"
+time="2023-05-17T10:01:50Z" level=info msg="OCI: Writing sha256:acb1ec674e686f4ba7a0e5c0ce1d41b6c2a5f5f1b9b9baca9c612f794faa3f8e to /pv_data/default/ghcr-io-singularityhub-github-ci-latest/container.sif"
+time="2023-05-17T10:01:50Z" level=info msg="Oras artifact root: /pv_data/default/ghcr-io-singularityhub-github-ci-latest"
+time="2023-05-17T10:01:50Z" level=info msg="Found artifact asset: container.sif"
+time="2023-05-17T10:01:50Z" level=info msg="volume source directory:/pv_data/default/ghcr-io-singularityhub-github-ci-latest"
+time="2023-05-17T10:01:50Z" level=info msg="volume target directory:/var/lib/kubelet/pods/44bb7b41-7854-4bdf-a2c5-d3552aee35f5/volumes/kubernetes.io~csi/oras-inline/mount"
+time="2023-05-17T10:01:50Z" level=info msg="volume options:[ro]"
 ```
+Notice that the `Uncached fetching` indicates that manifest and sif layer blob are both pulled from the remote registry.
 
 </details>
 
@@ -215,18 +222,24 @@ root@my-second-csi-app-inline:/# ls /mnt/second-oras/
 container.sif
 ```
 
-Importantly, in the logs we see an indication that the container was not re-pulled (our original goal): `Artifact root already exists, no need to re-create!`
+Importantly, in the logs we see an indication that the manifest and sif blob are pulled from the OCI layout cache (our original goal): `Cached fetching :`
 
 ```console
-time="2023-04-12T20:34:37Z" level=info msg="Artifact root already exists, no need to re-create!"
-time="2023-04-12T20:34:37Z" level=info msg="Oras artifact root: /pv_data/ghcr-io-singularityhub-github-ci-latest"
-time="2023-04-12T20:34:37Z" level=info msg="Found artifact asset: container.sif"
-time="2023-04-12T20:34:37Z" level=info msg="volume source directory:/pv_data/ghcr-io-singularityhub-github-ci-latest"
-time="2023-04-12T20:34:37Z" level=info msg="volume target directory:/var/lib/kubelet/pods/eda9a3b5-b6ce-41c5-84d2-ea7a1ea677bb/volumes/kubernetes.io~csi/oras-inline/mount"
-time="2023-04-12T20:34:37Z" level=info msg="volume options:[ro]"
-time="2023-04-12T20:34:37Z" level=info msg="BindMount - source: /pv_data/ghcr-io-singularityhub-github-ci-latest, target: /var/lib/kubelet/pods/eda9a3b5-b6ce-41c5-84d2-ea7a1ea677bb/volumes/kubernetes.io~csi/oras-inline/mount, options: [ro]"
-time="2023-04-12T20:34:37Z" level=info msg="mount -o bind /pv_data/ghcr-io-singularityhub-github-ci-latest /var/lib/kubelet/pods/eda9a3b5-b6ce-41c5-84d2-ea7a1ea677bb/volumes/kubernetes.io~csi/oras-inline/mount"
-time="2023-04-12T20:34:37Z" level=info msg="Successfully mounted /pv_data/ghcr-io-singularityhub-github-ci-latest to /var/lib/kubelet/pods/eda9a3b5-b6ce-41c5-84d2-ea7a1ea677bb/volumes/kubernetes.io~csi/oras-inline/mount"
+time="2023-05-17T10:03:11Z" level=info msg="Remote repository ghcr.io/singularityhub/github-ci:latest will be proxied by /pv_data/default/ghcr-io-singularityhub-github-ci-latest"
+time="2023-05-17T10:03:11Z" level=info msg="Manifest cached for ghcr.io/singularityhub/github-ci:latest"
+time="2023-05-17T10:03:11Z" level=info msg="Fetching manifest {\"MediaType\":\"application/vnd.oci.image.manifest.v1+json\",\"Digest\":\"sha256:5d6742ff0b10c1196202765dafb43275259bcbdbd3868c19ba1d19476c088867\",\"Size\":402}"
+time="2023-05-17T10:03:11Z" level=info msg="Cached fetching : {\"MediaType\":\"application/vnd.oci.image.manifest.v1+json\",\"Digest\":\"sha256:5d6742ff0b10c1196202765dafb43275259bcbdbd3868c19ba1d19476c088867\",\"Size\":402}"
+time="2023-05-17T10:03:11Z" level=info msg="Pulling sha256:acb1ec674e686f4ba7a0e5c0ce1d41b6c2a5f5f1b9b9baca9c612f794faa3f8e, 1 of 1"
+time="2023-05-17T10:03:11Z" level=info msg="Cached fetching : {\"MediaType\":\"application/vnd.sylabs.sif.layer.v1.sif\",\"Digest\":\"sha256:acb1ec674e686f4ba7a0e5c0ce1d41b6c2a5f5f1b9b9baca9c612f794faa3f8e\",\"Size\":798720}"
+time="2023-05-17T10:03:11Z" level=info msg="OCI: Writing sha256:acb1ec674e686f4ba7a0e5c0ce1d41b6c2a5f5f1b9b9baca9c612f794faa3f8e to /pv_data/default/ghcr-io-singularityhub-github-ci-latest/container.sif"
+time="2023-05-17T10:03:11Z" level=info msg="Oras artifact root: /pv_data/default/ghcr-io-singularityhub-github-ci-latest"
+time="2023-05-17T10:03:11Z" level=info msg="Found artifact asset: container.sif"
+time="2023-05-17T10:03:11Z" level=info msg="volume source directory:/pv_data/default/ghcr-io-singularityhub-github-ci-latest"
+time="2023-05-17T10:03:11Z" level=info msg="volume target directory:/var/lib/kubelet/pods/eff62b75-3449-470c-af17-c4ba06adcf41/volumes/kubernetes.io~csi/oras-inline/mount"
+time="2023-05-17T10:03:11Z" level=info msg="volume options:[ro]"
+time="2023-05-17T10:03:11Z" level=info msg="BindMount - source: /pv_data/default/ghcr-io-singularityhub-github-ci-latest, target: /var/lib/kubelet/pods/eff62b75-3449-470c-af17-c4ba06adcf41/volumes/kubernetes.io~csi/oras-inline/mount, options: [ro]"
+time="2023-05-17T10:03:11Z" level=info msg="mount -o bind /pv_data/default/ghcr-io-singularityhub-github-ci-latest /var/lib/kubelet/pods/eff62b75-3449-470c-af17-c4ba06adcf41/volumes/kubernetes.io~csi/oras-inline/mount"
+time="2023-05-17T10:03:11Z" level=info msg="Successfully mounted /pv_data/default/ghcr-io-singularityhub-github-ci-latest to /var/lib/kubelet/pods/eff62b75-3449-470c-af17-c4ba06adcf41/volumes/kubernetes.io~csi/oras-inline/mount"
 ```
 
 Let's try testing that the artifact remains persistent on the node and delete both pods, and also
@@ -264,7 +277,11 @@ Now we can theoretically create a pod again, and that same container.sif should
 $ kubectl apply -f examples/basic/pod/pod.yaml
 ```
 ```console
-time="2023-04-12T20:52:16Z" level=info msg="Artifact root already exists, no need to re-create!"
+time="2023-05-17T10:03:11Z" level=info msg="Manifest cached for ghcr.io/singularityhub/github-ci:latest"
+time="2023-05-17T10:03:11Z" level=info msg="Fetching manifest {\"MediaType\":\"application/vnd.oci.image.manifest.v1+json\",\"Digest\":\"sha256:5d6742ff0b10c1196202765dafb43275259bcbdbd3868c19ba1d19476c088867\",\"Size\":402}"
+time="2023-05-17T10:03:11Z" level=info msg="Cached fetching : {\"MediaType\":\"application/vnd.oci.image.manifest.v1+json\",\"Digest\":\"sha256:5d6742ff0b10c1196202765dafb43275259bcbdbd3868c19ba1d19476c088867\",\"Size\":402}"
+time="2023-05-17T10:03:11Z" level=info msg="Pulling sha256:acb1ec674e686f4ba7a0e5c0ce1d41b6c2a5f5f1b9b9baca9c612f794faa3f8e, 1 of 1"
+time="2023-05-17T10:03:11Z" level=info msg="Cached fetching : {\"MediaType\":\"application/vnd.sylabs.sif.layer.v1.sif\",\"Digest\":\"sha256:acb1ec674e686f4ba7a0e5c0ce1d41b6c2a5f5f1b9b9baca9c612f794faa3f8e\",\"Size\":798720}"
 ```
 
 And that's it! We very likely should have an attribute that specifies for this to be cleaned up and re-created. E.g., you

go.mod

@@ -1,34 +1,24 @@
-module github.com/converged-computing/oras-csi
+module github.com/billy-playground/oras-csi
 
 go 1.18
 
 require (
 	github.com/container-storage-interface/spec v1.6.0
 	github.com/golang/protobuf v1.5.2
+	github.com/opencontainers/image-spec v1.1.0-rc.3
 	github.com/sirupsen/logrus v1.8.0
 	google.golang.org/grpc v1.47.0
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0
+	oras.land/oras-go/v2 v2.1.0
 )
 
 require (
-	github.com/fsnotify/fsnotify v1.4.9 // indirect
-	github.com/go-logr/logr v1.2.0 // indirect
-	github.com/google/uuid v1.3.0 // indirect
-	github.com/kubernetes-csi/csi-test/v4 v4.4.0 // indirect
 	github.com/magefile/mage v1.10.0 // indirect
-	github.com/nxadm/tail v1.4.8 // indirect
-	github.com/onsi/ginkgo v1.16.5 // indirect
-	github.com/onsi/gomega v1.19.0 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
-	github.com/opencontainers/image-spec v1.1.0-rc2 // indirect
-	golang.org/x/net v0.0.0-20220225172249-27dd8689420f // indirect
+	golang.org/x/net v0.0.0-20201021035429-f5854403a974 // indirect
 	golang.org/x/sync v0.1.0 // indirect
-	golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e // indirect
-	golang.org/x/text v0.3.7 // indirect
-	google.golang.org/genproto v0.0.0-20201209185603-f92720507ed4 // indirect
+	golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4 // indirect
+	golang.org/x/text v0.3.3 // indirect
+	google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 // indirect
 	google.golang.org/protobuf v1.27.1 // indirect
-	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
-	gopkg.in/yaml.v2 v2.4.0 // indirect
-	k8s.io/klog/v2 v2.60.1 // indirect
-	oras.land/oras-go/v2 v2.0.2 // indirect
 )