make mappings more custom

Author: ChristianKniep

.drone.yml

@@ -1,6 +1,6 @@
 workspace:
-  base: /drone/src
-  path: /usr/local/src/github.com/qnib/doxy
+  base: /usr/local/
+  path: src/github.com/qnib/doxy
 
 pipeline:
   alpine:

GPU.md

@@ -2,6 +2,54 @@
 
 As a Proof-of-Concept how to implement GPU (and InfiniBand) support, `doxy` was extended to allow for injection of payload to the `docker create` call.
 
+## CentOS7 box
+
+### Check for CUDA devices
+
+The BOX has 10 Tesla K80 devices (`/dev/nvidia[0-9]`).
+
+```
+[root@odin001 ~]# nvidia-smi -L
+GPU 0: Tesla K80 (UUID: GPU-4095713a-1f9b-791d-841d-8b35143127d4)
+GPU 1: Tesla K80 (UUID: GPU-ab541226-7c4f-ab59-3927-1535f68a3a8f)
+GPU 2: Tesla K80 (UUID: GPU-8310202d-1d32-bac5-cc36-2add9e21d9d6)
+GPU 3: Tesla K80 (UUID: GPU-cb3d675d-ba3b-5cdb-2331-9534bfd20679)
+GPU 4: Tesla K80 (UUID: GPU-8f1511d6-5326-e718-c682-cd2377bbf7cf)
+GPU 5: Tesla K80 (UUID: GPU-997c3b02-765c-7cde-daff-f19dadeb6894)
+GPU 6: Tesla K80 (UUID: GPU-bb1a7162-859a-4e9c-eeac-150bd35ff767)
+GPU 7: Tesla K80 (UUID: GPU-9698fbda-39fd-5f1f-1691-626c7e780f36)
+GPU 8: Tesla K80 (UUID: GPU-243c49d3-5dc3-26cc-2f6e-42e2baf6ac93)
+GPU 9: Tesla K80 (UUID: GPU-0fc004de-3f1a-030a-3931-33043b895514)
+```
+
+### Starting the proxy
+```
+[root@odin001 ~]# docker run -v /var/run:/var/run/ -ti --rm qnib/doxy:gpu doxy --pattern-key=hpc --debug --proxy-socket=/var/run/hpc.sock --gpu
+> execute CMD 'doxy --pattern-key=hpc --debug --proxy-socket=/var/run/hpc.sock --gpu'
+2018/02/25 00:07:17 [II] Start Version: 0.2.4
+2018/02/25 00:07:17 Error reading patterns file 'open /etc/doxy.pattern: no such file or directory', using hpc patterns
+2018/02/25 00:07:17 [doxy] Listening on /var/run/hpc.sock
+2018/02/25 00:07:17 Serving proxy on '/var/run/hpc.sock'
+[negroni] 2018-02-25T00:07:52Z | 200 |   843.548µs | docker | GET /_ping
+Add GPU stuff
+New device: /dev/nvidia0:/dev/nvidia0:rwm
+New device: /dev/nvidiactl:/dev/nvidiactl:rwm
+```
+
+### Running a CUDA image:
+
+Since passing through the log-API calls gave me some headache, the container is created using the proxy, but started using the proper docker API.
+
+```
+$ docker start -ai $(docker -H unix:///var/run/hpc.sock create \
+                            -ti -e SKIP_ENTRYPOINTS=true \
+                            qnib/cplain-cuda nvidia-smi -L)
+[II] qnib/init-plain script v0.4.28
+> execute CMD 'nvidia-smi -L'
+GPU 0: Tesla K80 (UUID: GPU-4095713a-1f9b-791d-841d-8b35143127d4)
+```
+
+## AWS p2.xlarge
 On an `p2.xlarge` instance with CUDA support, `doxy:gpu` was started like that.
 
 ```bash

main.go

@@ -64,6 +64,11 @@ var (
 		Usage: "Overwrite `--user` with given value (if pin-user is set)",
 		EnvVar: "DOXY_USER",
 	}
+	cudaLibPathFlag = cli.StringFlag{
+		Name:  "cuda-lib-path",
+		Usage: "Path to cuda libraries.",
+		EnvVar: "DOXY_CUDA_LIB_PATH",
+	}
 	deviceFileFlag = cli.StringFlag{
 		Name:  "device-file",
 		Value: proxy.DEVICE_FILE,
@@ -86,6 +91,8 @@ func EvalOptions(cfg *config.Config) (po []proxy.ProxyOption) {
 	pinUser, _ := cfg.String("user")
 	pinUserBool, _ := cfg.Bool("pin-user")
 	po = append(po, proxy.WithPinUser(pinUserBool, pinUser))
+	cudalibPath, _ := cfg.String("cuda-lib-path")
+	po = append(po, proxy.WithCudaLibPath(cudalibPath))
 	return
 }
 
@@ -115,7 +122,7 @@ func EvalDevicesOpts(cfg *config.Config) (proxy.ProxyOption) {
 	defer reader.Close()
 	devices := []string{}
 	if err != nil {
-		return proxy.WithDevMappings(proxy.DEVICES)
+		return proxy.WithDevMappings([]string{})
 	}
 	devices, err  = proxy.ReadLineFile(reader)
 	if err != nil {
@@ -158,6 +165,7 @@ func main() {
 		bindAddFlag,
 		pinUserBool,
 		pinUserFlag,
+		cudaLibPathFlag,
 	}
 	app.Action = RunApp
 	app.Run(os.Args)

proxy/main.go

@@ -30,15 +30,14 @@ var (
 	}
 	HPC_PAT = []string{
 		`^/(v\d\.\d+/)?containers(/\w+)?/(json|stats|top|logs|wait|create|start|run|kill)$`,
-		`^/(v\d\.\d+/)?images(/\w+)?/(json|pull)$`,
+		`^/(v\d\.\d+/)?images(/\w+)?/(json|pull|create)$`,
 		`^/(v\d\.\d+/)?info$`,
 		`^/(v\d\.\d+/)?images/(pull|create)$`,
 		`^/(v\d\.\d+/)?version$`,
 		"^/_ping$",
 	}
-	DEVICES = []string{
+	GPUS = []string{
 		"/dev/nvidia0:/dev/nvidia0:rwm",
-		"/dev/nvidia-uvm:/dev/nvidia-uvm:rwm",
 		"/dev/nvidiactl:/dev/nvidiactl:rwm",
 	}
 	PATTERNS = map[string][]string{
@@ -49,10 +48,10 @@ var (
 
 type Proxy struct {
 	po ProxyOptions
-	dockerSocket, newSocket, pinUser 	string
-	debug, gpu, pinUserEnabled			bool
-	patterns 							[]string
-	bindMounts,devMappings				[]string
+	dockerSocket, newSocket, pinUser,cudaLibPath	string
+	debug, gpu, pinUserEnabled						bool
+	patterns 										[]string
+	bindMounts,devMappings							[]string
 }
 
 func NewProxy(opts ...ProxyOption) Proxy {
@@ -71,6 +70,7 @@ func NewProxy(opts ...ProxyOption) Proxy {
 		patterns: options.Patterns,
 		bindMounts: options.BindMounts,
 		devMappings: options.DevMappings,
+		cudaLibPath: options.CudaLibPath,
 	}
 }
 
@@ -86,7 +86,7 @@ func (p *Proxy) GetOptions() map[string]interface{} {
 }
 
 func (p *Proxy) Run() {
-	upstream := NewUpstream(p.dockerSocket, p.patterns, p.bindMounts, p.devMappings, p.gpu, p.pinUser, p.pinUserEnabled)
+	upstream := NewUpstream(p.dockerSocket, p.patterns, p.bindMounts, p.devMappings, p.gpu, p.pinUser, p.pinUserEnabled, p.cudaLibPath)
 	sigc := make(chan os.Signal, 1)
 	signal.Notify(sigc, os.Interrupt, os.Kill, syscall.SIGTERM)
 	l, err := ListenToNewSock(p.newSocket, sigc)

proxy/options.go

@@ -2,9 +2,9 @@ package proxy
 
 
 type ProxyOptions struct {
-	DockerSocket,ProxySocket,PinUser 	string
-	Debug,Gpu,PinUserEnabled 			bool
-	Patterns,BindMounts,DevMappings 	[]string
+	DockerSocket,ProxySocket,PinUser,CudaLibPath	string
+	Debug,Gpu,PinUserEnabled 						bool
+	Patterns,BindMounts,DevMappings 				[]string
 }
 
 var defaultProxyOptions = ProxyOptions{
@@ -14,6 +14,7 @@ var defaultProxyOptions = ProxyOptions{
 	PinUserEnabled: false,
 	Debug: false,
 	Gpu: false,
+	CudaLibPath: "",
 	Patterns: []string{},
 	BindMounts: []string{},
 	DevMappings: []string{},
@@ -32,6 +33,11 @@ func WithDockerSocket(s string) ProxyOption {
 		o.DockerSocket = s
 	}
 }
+func WithCudaLibPath(s string) ProxyOption {
+	return func(o *ProxyOptions) {
+		o.CudaLibPath = s
+	}
+}
 
 func WithProxySocket(s string) ProxyOption {
 	return func(o *ProxyOptions) {

proxy/proxy.go

@@ -21,15 +21,15 @@ import (
 
 // UpStream creates upstream handler struct
 type UpStream struct {
-	Name  			string
-	proxy 			http.Handler
+	Name  				string
+	proxy 				http.Handler
 	// TODO: Kick out separat config options and use more generic one
-	allowed 		[]*regexp.Regexp
-	bindMounts 		[]string
-	devMappings 	[]string
-	gpu 			bool
-	pinUser 		string
-	pinUserEnabled	bool
+	allowed 			[]*regexp.Regexp
+	bindMounts 			[]string
+	devMappings 		[]string
+	gpu 				bool
+	pinUser,cudaLibPath	string
+	pinUserEnabled		bool
 }
 
 // UnixSocket just provides the path, so that I can test it
@@ -81,11 +81,12 @@ func NewUpstreamPO(po ProxyOptions) *UpStream {
 		gpu: po.Gpu,
 		pinUser: po.PinUser,
 		pinUserEnabled: po.PinUserEnabled,
+		cudaLibPath: po.CudaLibPath,
 	}
 	return upstream
 }
 // NewUpstream returns a new socket (magic)
-func NewUpstream(socket string, regs []string, binds []string, devs []string, gpu bool, pinUser string, pinUserB bool) *UpStream {
+func NewUpstream(socket string, regs []string, binds []string, devs []string, gpu bool, pinUser string, pinUserB bool, cudaLibPath string) *UpStream {
 	us := NewUnixSocket(socket)
 	a := []*regexp.Regexp{}
 	for _, r := range regs {
@@ -101,6 +102,7 @@ func NewUpstream(socket string, regs []string, binds []string, devs []string, gp
 		gpu: gpu,
 		pinUser: pinUser,
 		pinUserEnabled: pinUserB,
+		cudaLibPath: cudaLibPath,
 	}
 	return upstream
 }
@@ -151,16 +153,30 @@ func (u *UpStream) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 		}
 		// prepare devMappings
 		devMappings := []string{}
-		for _, dev := range u.devMappings {
+		for _, dev := range GPUS {
 			devMappings = append(devMappings, dev)
 		}
 		// In case GPU support is enabled add devices and mounts
 		if u.gpu {
 			fmt.Println("Add GPU stuff")
 			// TODO: Be smarter about the version of the driver
-			hostConfig.Binds = append(hostConfig.Binds, "/usr/lib/nvidia-384/:/usr/local/nvidia/")
+			if u.cudaLibPath != "" {
+				hostConfig.Binds = append(hostConfig.Binds, fmt.Sprintf("%s:/usr/local/nvidia/", u.cudaLibPath))
+			}
 			config.Env = append(config.Env, "PATH=/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin")
 			config.Env = append(config.Env, "LD_LIBRARY_PATH=/usr/local/nvidia/")
+			for _, dev := range devMappings {
+				if dev == "" {
+					continue
+				}
+				fmt.Printf("New device: %s\n", dev)
+
+				dm, err := createDevMapping(dev)
+				if err != nil {
+					continue
+				}
+				hostConfig.Devices = append(hostConfig.Devices, dm)
+			}
 		}
 		if u.pinUserEnabled {
 			fmt.Print("Alter User setting ")
@@ -184,18 +200,7 @@ func (u *UpStream) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 			fmt.Printf("New bindmount: %s\n", bMount)
 			hostConfig.Binds = append(hostConfig.Binds, bMount)
 		}
-		for _, dev := range devMappings {
-			if dev == "" {
-				continue
-			}
-			fmt.Printf("New device: %s\n", dev)
 
-			dm, err := createDevMapping(dev)
-			if err != nil {
-				continue
-			}
-			hostConfig.Devices = append(hostConfig.Devices, dm)
-		}
 		fmt.Printf("Mounts: %v\n", hostConfig.Binds)
 		cfgBody := configWrapper{
 			Config:           config,