Merge pull request #373 from gliderlabs/master

release v3.2.4

Author: michaelshobbs

CHANGELOG.md

@@ -10,6 +10,22 @@ All notable changes to this project will be documented in this file.
 
 ### Changed
 
+## [v3.2.4] - 2018-01-16
+### Fixed
+- @michaelshobbs fix working_directory so we don't duplicate test runs
+
+### Added
+- @chris7444 take the hostname from /etc/host_hostname if the file is there
+- @chris7444 update README.md for swarm deployments PR #329
+- @nvanheuverzwijn strip \r and \n when reading the file /etc/host_hostname
+- @lucassabreu toJSON with examples
+
+### Changed
+- @michaelshobbs pass debug to test container
+- @jgreat Strip header bytes from log stream
+- @trondvh chmod +x build.sh
+- @develar alpine 3.7 + golang 1.9.2
+
 ## [v3.2.3] - 2017-09-23
 ### Added
 - @guigouz guigouz Add `RAW_FORMAT` to the documentation
@@ -162,8 +178,9 @@ All notable changes to this project will be documented in this file.
 - Base container is now Alpine
 - Moved to gliderlabs organization
 
-[unreleased]: https://github.com/gliderlabs/logspout/compare/v3.2.3...HEAD
-[v3.2.3]: https://github.com/gliderlabs/logspout/compare/v3.2.3...v3.2.3
+[unreleased]: https://github.com/gliderlabs/logspout/compare/v3.2.4...HEAD
+[v3.2.4]: https://github.com/gliderlabs/logspout/compare/v3.2.3...v3.2.4
+[v3.2.3]: https://github.com/gliderlabs/logspout/compare/v3.2.2...v3.2.3
 [v3.2.2]: https://github.com/gliderlabs/logspout/compare/v3.2.1...v3.2.2
 [v3.2.1]: https://github.com/gliderlabs/logspout/compare/v3.2...v3.2.1
 [v3.2]: https://github.com/gliderlabs/logspout/compare/v3.1...v3.2

Dockerfile

@@ -1,4 +1,4 @@
-FROM alpine:3.5
+FROM alpine:3.7
 ENTRYPOINT ["/bin/logspout"]
 VOLUME /mnt/routes
 EXPOSE 80
@@ -8,4 +8,4 @@ RUN cd /src && ./build.sh "$(cat VERSION)"
 
 ONBUILD COPY ./build.sh /src/build.sh
 ONBUILD COPY ./modules.go /src/modules.go
-ONBUILD RUN cd /src && ./build.sh "$(cat VERSION)-custom"
+ONBUILD RUN cd /src && chmod +x ./build.sh && ./build.sh "$(cat VERSION)-custom"

Dockerfile.dev

@@ -1,4 +1,4 @@
-FROM alpine:3.5
+FROM alpine:3.6
 VOLUME /mnt/routes
 EXPOSE 80
 

Makefile

@@ -49,6 +49,7 @@ test: build-dev
 		-v /var/run/docker.sock:/var/run/docker.sock \
 		-v $(PWD):/go/src/github.com/gliderlabs/logspout \
 		-e TEST_ARGS="" \
+		-e DEBUG=$(DEBUG) \
 		$(NAME):dev make -e test-direct
 
 test-direct:

README.md

@@ -59,17 +59,17 @@ You can tell logspout to only include certain containers by setting filter param
 		--volume=/var/run/docker.sock:/var/run/docker.sock \
 		gliderlabs/logspout \
 		raw://192.168.10.10:5000?filter.name=*_db
-
+	
 	$ docker run \
 		--volume=/var/run/docker.sock:/var/run/docker.sock \
 		gliderlabs/logspout \
 		raw://192.168.10.10:5000?filter.id=3b6ba57db54a
-
+	
 	$ docker run \
 		--volume=/var/run/docker.sock:/var/run/docker.sock \
 		gliderlabs/logspout \
 		raw://192.168.10.10:5000?filter.sources=stdout%2Cstderr
-
+	
 	# Forward logs from containers with both label 'a' starting with 'x', and label 'b' ending in 'y'.
 	$ docker run \
 		--volume=/var/run/docker.sock:/var/run/docker.sock \
@@ -156,6 +156,85 @@ Logspout relies on the Docker API to retrieve container logs. A failure in the A
 * `SYSLOG_TAG` - datum for tag field (default `{{.ContainerName}}+route.Options["append_tag"]`)
 * `SYSLOG_TIMESTAMP` - datum for timestamp field (default `{{.Timestamp}}`)
 
+#### Raw Format
+
+The raw adapter has a function `toJSON` that can be used to format the message/fields to generate JSON-like output in a simple way, or full JSON output.
+
+Use examples:
+
+##### Mixed JSON + generic:
+```
+{{ .Time.Format "2006-01-02T15:04:05Z07:00" }} { "container" : "{{ .Container.Name }}", "labels": {{ toJSON .Container.Config.Labels }}, "timestamp": "{{ .Time.Format "2006-01-02T15:04:05Z07:00" }}", "source" : "{{ .Source }}", "message": {{ toJSON .Data }} }
+```
+
+```
+2017-10-26T11:59:32Z { "container" : "/catalogo_worker_1", "image": "sha256:e9bce6c17c80c603c4c8dbac2ad2285982d218f6ea0332f8b0fb84572941b773", "labels": {"com.docker.compose.config-hash":"4f9c3d3bfb2f65e29a4bc8a4a1b3f0a1c8a42323106a5e9106fe9279f8031321","com.docker.compose.container-number":"1","com.docker.compose.oneoff":"False","com.docker.compose.project":"catalogo","com.docker.compose.service":"worker","com.docker.compose.version":"1.16.1","logging":"true"}, "timestamp": "2017-10-26T11:59:32Z", "source" : "stdout", "message": "2017-10-26 11:59:32,950 INFO success: command_bus_0 entered RUNNING state, process has stayed up for \u003e than 1 seconds (startsecs)" }
+```
+
+##### Full JSON like:
+
+```
+{ "container" : "{{ .Container.Name }}", "labels": {{ toJSON .Container.Config.Labels }}, "timestamp": "{{ .Time.Format "2006-01-02T15:04:05Z07:00" }}", "source" : "{{ .Source }}", "message": {{ toJSON .Data }} }
+```
+
+```json
+{
+  "container": "/a_container",
+  "image": "sha256:e9bce6c17c80c603c4c8dbac2ad2285982d218f6ea0332f8b0fb84572941b773",
+  "labels": {
+    "com.docker.compose.config-hash": "4f9c3d3bfb2f65e29a4bc8a4a1b3f0a1c8a42323106a5e9106fe9279f8031321",
+    "com.docker.compose.container-number": "1",
+    "com.docker.compose.oneoff": "False",
+    "com.docker.compose.project": "a_project",
+    "com.docker.compose.service": "worker",
+    "com.docker.compose.version": "1.16.1",
+    "logging": "true"
+  },
+  "timestamp": "2017-10-26T11:59:32Z",
+  "source": "stdout",
+  "message": "2017-10-26 11:59:32,950 INFO success: command_bus_0 entered RUNNING state, process has stayed up for > than 1 seconds (startsecs)"
+}
+
+```
+
+#### Using Logspout in a swarm
+
+In a swarm, logspout is best deployed as a global service.  When running logspout with 'docker run', you can change the value of the hostname field using the `SYSLOG_HOSTNAME` environment variable as explained above. However, this does not work in a compose file because the value for `SYSLOG_HOSTNAME` will be the same for all logspout "tasks", regardless of the docker host on which they run. To support this mode of deployment, the syslog adapter will look for the file `/etc/host_hostname` and, if the file exists and it is not empty, will configure the hostname field with the content of this file. You can then use a volume mount to map a file on the docker hosts with the file `/etc/host_hostname` in the container.  The sample compose file below illustrates how this can be done
+
+```
+version: "3"
+networks:
+  logging:
+services:
+  logspout:
+  image: gliderlabs/logspout:latest
+  networks:
+    - logging
+  volumes:
+    - /etc/hostname:/etc/host_hostname:ro
+    - /var/run/docker.sock:/var/run/docker.sock
+  command:
+    syslog://svt2-logger.am2.cloudra.local:514
+  deploy:
+    mode: global
+    resources:
+      limits:
+        cpus: '0.20'
+        memory: 256M
+      reservations:
+        cpus: '0.10'
+      memory: 128M
+```
+
+logspout can then be deployed as a global service in the swam with the following command
+
+```bash
+docker stack deploy --compose-file <name of your compose file>
+```
+
+More information about services and their mode of deployment can be found here:
+https://docs.docker.com/engine/swarm/how-swarm-mode-works/services/ 
+
 ## Modules
 
 The standard distribution of logspout comes with all modules defined in this repository. You can remove or add new modules with custom builds of logspout. In the `custom` dir, edit the `modules.go` file and do a `docker build`.

VERSION

@@ -1 +1 @@
-v3.2.3
+v3.2.4

adapters/raw/raw.go

@@ -2,6 +2,7 @@ package raw
 
 import (
 	"bytes"
+	"encoding/json"
 	"errors"
 	"log"
 	"net"
@@ -16,6 +17,17 @@ func init() {
 	router.AdapterFactories.Register(NewRawAdapter, "raw")
 }
 
+var funcs = template.FuncMap{
+	"toJSON": func(value interface{}) string {
+		bytes, err := json.Marshal(value)
+		if err != nil {
+			log.Println("error marshalling to JSON: ", err)
+			return "null"
+		}
+		return string(bytes)
+	},
+}
+
 // NewRawAdapter returns a configured raw.Adapter
 func NewRawAdapter(route *router.Route) (router.LogAdapter, error) {
 	transport, found := router.AdapterTransports.Lookup(route.AdapterTransport("udp"))
@@ -30,7 +42,7 @@ func NewRawAdapter(route *router.Route) (router.LogAdapter, error) {
 	if os.Getenv("RAW_FORMAT") != "" {
 		tmplStr = os.Getenv("RAW_FORMAT")
 	}
-	tmpl, err := template.New("raw").Parse(tmplStr)
+	tmpl, err := template.New("raw").Funcs(funcs).Parse(tmplStr)
 	if err != nil {
 		return nil, err
 	}

adapters/syslog/syslog.go

@@ -4,11 +4,13 @@ import (
 	"bytes"
 	"errors"
 	"fmt"
+	"io/ioutil"
 	"log"
 	"log/syslog"
 	"net"
 	"os"
 	"strconv"
+	"strings"
 	"syscall"
 	"text/template"
 	"time"
@@ -54,6 +56,16 @@ func debug(v ...interface{}) {
 	}
 }
 
+func getHostname() string {
+	content, err := ioutil.ReadFile("/etc/host_hostname")
+	if err == nil && len(content) > 0 {
+		hostname = strings.TrimRight(string(content), "\r\n")
+	} else {
+		hostname = getopt("SYSLOG_HOSTNAME", "{{.Container.Config.Hostname}}")
+	}
+	return hostname
+}
+
 // NewSyslogAdapter returnas a configured syslog.Adapter
 func NewSyslogAdapter(route *router.Route) (router.LogAdapter, error) {
 	transport, found := router.AdapterTransports.Lookup(route.AdapterTransport("udp"))
@@ -67,8 +79,9 @@ func NewSyslogAdapter(route *router.Route) (router.LogAdapter, error) {
 
 	format := getopt("SYSLOG_FORMAT", "rfc5424")
 	priority := getopt("SYSLOG_PRIORITY", "{{.Priority}}")
-	hostname := getopt("SYSLOG_HOSTNAME", "{{.Container.Config.Hostname}}")
 	pid := getopt("SYSLOG_PID", "{{.Container.State.Pid}}")
+	hostname = getHostname()
+
 	tag := getopt("SYSLOG_TAG", "{{.ContainerName}}"+route.Options["append_tag"])
 	structuredData := getopt("SYSLOG_STRUCTURED_DATA", "")
 	if route.Options["structured_data"] != "" {

adapters/syslog/syslog_test.go

@@ -4,10 +4,12 @@ import (
 	"bufio"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"log"
 	"net"
 	"os"
 	"strconv"
+	"strings"
 	"sync"
 	"testing"
 	"text/template"
@@ -41,6 +43,9 @@ var (
 	}
 	testTmplStr = fmt.Sprintf("<%s>%s %s %s[%s]: %s\n",
 		testPriority, testTimestamp, testHostname, testTag, testPid, testData)
+	hostHostnameFilename = "/etc/host_hostname"
+	hostnameContent      = "hostname"
+	badHostnameContent   = "hostname\r\n"
 )
 
 func TestSyslogRetryCount(t *testing.T) {
@@ -102,6 +107,16 @@ func TestSyslogReconnectOnClose(t *testing.T) {
 	}
 }
 
+func TestHostnameDoesNotHaveLineFeed(t *testing.T) {
+	if err := ioutil.WriteFile(hostHostnameFilename, []byte(badHostnameContent), 0777); err != nil {
+		t.Fatal(err)
+	}
+	testHostname := getHostname()
+	if strings.Contains(testHostname, badHostnameContent) {
+		t.Errorf("expected hostname to be %s. got %s in hostname %s", hostnameContent, badHostnameContent, testHostname)
+	}
+}
+
 func startServer(n, la string, done chan<- string) (addr string, sock io.Closer, wg *sync.WaitGroup) {
 	if n == "udp" || n == "tcp" {
 		la = "127.0.0.1:0"

circle.yml

@@ -2,10 +2,9 @@ version: 2
 jobs:
   build:
     machine: true
-    working_directory: ~/.go_workspace/src/github.com/gliderlabs/
+    working_directory: ~/.go_workspace/src/github.com/gliderlabs/logspout
     environment:
       DEBUG: true
-      GO_PROJECT: ../go/src/github.com/gliderlabs/$CIRCLE_PROJECT_REPONAME
     steps:
       - checkout
       - run: |