From 23d97b1c352a2fa45a9b3a8d0ce652e7155c8383 Mon Sep 17 00:00:00 2001 From: Micah Parks <66095735+MicahParks@users.noreply.github.com> Date: Mon, 4 Aug 2025 14:05:58 -0400 Subject: [PATCH 1/8] Remove default port option to allow port 80 to be used for grpc scheme --- diode/client.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/diode/client.go b/diode/client.go index 4e4dd7c..c2dba72 100644 --- a/diode/client.go +++ b/diode/client.go @@ -74,9 +74,6 @@ func parseTarget(target string) (string, string, bool, error) { } authority := u.Host - if u.Port() == "" { - authority += ":443" - } path := u.Path if path == "/" { From 8e466f085f601bc638da43b6ff679eefd6a51560 Mon Sep 17 00:00:00 2001 From: Micah Parks <66095735+MicahParks@users.noreply.github.com> Date: Mon, 4 Aug 2025 14:29:12 -0400 Subject: [PATCH 2/8] Allow HTTP and HTTPS schemes and attach default port if necessary --- diode/client.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/diode/client.go b/diode/client.go index c2dba72..a59bb6e 100644 --- a/diode/client.go +++ b/diode/client.go @@ -54,7 +54,7 @@ const ( defaultStreamName = "latest" ) -var allowedSchemesRe = regexp.MustCompile(`grpc|grpcs`) +var allowedSchemesRe = regexp.MustCompile(`grpc|grpcs|http|https`) // loadCerts loads the system x509 cert pool func loadCerts() *x509.CertPool { @@ -74,6 +74,14 @@ func parseTarget(target string) (string, string, bool, error) { } authority := u.Host + if u.Port() == "" { + switch u.Scheme { + case "grpc", "http": + authority += ":80" + case "grpcs", "https": + authority += ":443" + } + } path := u.Path if path == "/" { From 1a192f9ba72b9e66eeff47cdf465d01bc1947569 Mon Sep 17 00:00:00 2001 From: Micah Parks <66095735+MicahParks@users.noreply.github.com> Date: Mon, 4 Aug 2025 14:48:10 -0400 Subject: [PATCH 3/8] Update errors --- diode/client.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/diode/client.go b/diode/client.go index a59bb6e..83a3a15 100644 --- a/diode/client.go +++ b/diode/client.go @@ -70,7 +70,7 @@ func parseTarget(target string) (string, string, bool, error) { } if !allowedSchemesRe.MatchString(u.Scheme) { - return "", "", false, errors.New("target should start with grpc:// or grpcs://") + return "", "", false, errors.New("target should start with grpc:// or grpcs:// or http:// or https://") } authority := u.Host @@ -80,6 +80,8 @@ func parseTarget(target string) (string, string, bool, error) { authority += ":80" case "grpcs", "https": authority += ":443" + default: + return "", "", false, fmt.Errorf("missing port with unsupported scheme: %s", u.Scheme) } } From a6d2a536e2d299aab4b5fa8747690a8516d7a019 Mon Sep 17 00:00:00 2001 From: Micah Parks <66095735+MicahParks@users.noreply.github.com> Date: Fri, 8 Aug 2025 10:47:29 -0400 Subject: [PATCH 4/8] Add tests and correct TLS verify --- diode/client.go | 17 +++++++++++++---- diode/client_test.go | 18 ++++++++++++++++-- 2 files changed, 29 insertions(+), 6 deletions(-) diff --git a/diode/client.go b/diode/client.go index 83a3a15..6f0b53f 100644 --- a/diode/client.go +++ b/diode/client.go @@ -54,7 +54,12 @@ const ( defaultStreamName = "latest" ) -var allowedSchemesRe = regexp.MustCompile(`grpc|grpcs|http|https`) +var ( + // ErrInvalidTargetScheme is returned when the target URL does not start with a valid scheme. + ErrInvalidTargetScheme = errors.New("target should start with grpc:// or grpcs:// or http:// or https://") + + allowedSchemesRe = regexp.MustCompile(`grpc|grpcs|http|https`) +) // loadCerts loads the system x509 cert pool func loadCerts() *x509.CertPool { @@ -70,7 +75,7 @@ func parseTarget(target string) (string, string, bool, error) { } if !allowedSchemesRe.MatchString(u.Scheme) { - return "", "", false, errors.New("target should start with grpc:// or grpcs:// or http:// or https://") + return "", "", false, ErrInvalidTargetScheme } authority := u.Host @@ -81,7 +86,7 @@ func parseTarget(target string) (string, string, bool, error) { case "grpcs", "https": authority += ":443" default: - return "", "", false, fmt.Errorf("missing port with unsupported scheme: %s", u.Scheme) + return "", "", false, fmt.Errorf("missing port with unsupported scheme: %s: %w", u.Scheme, ErrInvalidTargetScheme) } } @@ -90,7 +95,11 @@ func parseTarget(target string) (string, string, bool, error) { path = "" } - tlsVerify := u.Scheme == "grpcs" + tlsVerify := false + switch u.Scheme { + case "grpcs", "https": + tlsVerify = true + } return authority, path, tlsVerify, nil } diff --git a/diode/client_test.go b/diode/client_test.go index 5c9a607..d65eda7 100644 --- a/diode/client_test.go +++ b/diode/client_test.go @@ -63,6 +63,20 @@ func TestParseTarget(t *testing.T) { tlsVerify: true, wantErr: nil, }, + { + desc: "valid HTTP target", + target: "http://localhost:8081", + authority: "localhost:8081", + tlsVerify: false, + wantErr: nil, + }, + { + desc: "valid HTTP target with tls", + target: "https://localhost:8081", + authority: "localhost:8081", + tlsVerify: true, + wantErr: nil, + }, { desc: "valid target empty path on grpc://localhost:8081/", target: "grpc://localhost:8081/", @@ -81,11 +95,11 @@ func TestParseTarget(t *testing.T) { }, { desc: "invalid scheme in target", - target: "http://localhost:8081", + target: "ftp://localhost:8081", authority: "", path: "", tlsVerify: false, - wantErr: errors.New("target should start with grpc:// or grpcs://"), + wantErr: ErrInvalidTargetScheme, }, { desc: "invalid target", From 2bd38539cdcaa477c0b48834c671f0797b5bb3a5 Mon Sep 17 00:00:00 2001 From: Micah Parks <66095735+MicahParks@users.noreply.github.com> Date: Fri, 8 Aug 2025 10:54:40 -0400 Subject: [PATCH 5/8] Update other relevant test --- diode/client_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/diode/client_test.go b/diode/client_test.go index d65eda7..b796a16 100644 --- a/diode/client_test.go +++ b/diode/client_test.go @@ -331,14 +331,14 @@ func TestNewClient(t *testing.T) { }, { desc: "invalid target", - target: "http://localhost:8081", + target: "ftp://localhost:8081", appName: "my-producer", appVersion: "0.1.0", clientID: "client-id-123", clientSecret: "client-secret-456", clientIDEnvVarValue: "", clientSecretEnvVarValue: "", - wantErr: errors.New("target should start with grpc:// or grpcs://"), + wantErr: ErrInvalidTargetScheme, }, { desc: "missing clientID and clientSecret", From 612fa1a96dde4288d932bbd30f49e94ffe5f8618 Mon Sep 17 00:00:00 2001 From: Micah Parks <66095735+MicahParks@users.noreply.github.com> Date: Fri, 8 Aug 2025 11:05:41 -0400 Subject: [PATCH 6/8] Update target mention --- README.md | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 4161d62..0548ba1 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,7 @@ More information about Diode can be found at [https://netboxlabs.com/blog/introducing-diode-streamlining-data-ingestion-in-netbox/](https://netboxlabs.com/blog/introducing-diode-streamlining-data-ingestion-in-netbox/). ## Prerequisites + - Go 1.24 or later installed ## Installation @@ -30,8 +31,9 @@ go get github.com/netboxlabs/diode-sdk-go ### Example -* `target` should be the address of the Diode service, e.g. `grpc://localhost:8080/diode` for insecure connection - or `grpcs://example.com` for secure connection. +* `target` should be the address of the Diode service. + * Insecure connections: `grpc://localhost:8080/diode` or `http://localhost:8080/diode` + * Secure connections: `grpcs://example.com` or `https://example.com` ```go package main @@ -119,16 +121,17 @@ See all [examples](./examples/main.go) for reference. ### Dry run client -Use a `DryRunClient` to inspect what would be sent to Diode without actually sending any data. When a directory is provided a new JSON file is created for each ingest call. +Use a `DryRunClient` to inspect what would be sent to Diode without actually sending any data. When a directory is +provided a new JSON file is created for each ingest call. ```go // Write ingest payload to a timestamped file in /tmp client, err := diode.NewDryRunClient("example-app", "/tmp") if err != nil { - log.Fatal(err) +log.Fatal(err) } _, _ = client.Ingest(context.Background(), []diode.Entity{ - &diode.Device{Name: diode.String("Device A")}, +&diode.Device{Name: diode.String("Device A")}, }) _ = client.Close() ``` @@ -138,23 +141,23 @@ Loaded entities can later be ingested using a real client: ```go protoEntities, err := diode.LoadDryRunEntities("/tmp/example-app_1750106879725947344.json") if err != nil { - log.Fatal(err) +log.Fatal(err) } realClient, err := diode.NewClient( - "grpc://localhost:8080/diode", - "example-app", - "0.1.0", - diode.WithClientID("YOUR_CLIENT_ID"), - diode.WithClientSecret("YOUR_CLIENT_SECRET"), +"grpc://localhost:8080/diode", +"example-app", +"0.1.0", +diode.WithClientID("YOUR_CLIENT_ID"), +diode.WithClientSecret("YOUR_CLIENT_SECRET"), ) if err != nil { - log.Fatal(err) +log.Fatal(err) } _, err = realClient.IngestProto(context.Background(), protoEntities) if err != nil { - log.Fatal(err) +log.Fatal(err) } ``` From 9a7b24f178e36dff2250bf0229cd362afa2f272c Mon Sep 17 00:00:00 2001 From: Micah Parks <66095735+MicahParks@users.noreply.github.com> Date: Fri, 8 Aug 2025 11:29:50 -0400 Subject: [PATCH 7/8] Undo auto-format --- README.md | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 0548ba1..9f662fd 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,6 @@ More information about Diode can be found at [https://netboxlabs.com/blog/introducing-diode-streamlining-data-ingestion-in-netbox/](https://netboxlabs.com/blog/introducing-diode-streamlining-data-ingestion-in-netbox/). ## Prerequisites - - Go 1.24 or later installed ## Installation @@ -121,17 +120,16 @@ See all [examples](./examples/main.go) for reference. ### Dry run client -Use a `DryRunClient` to inspect what would be sent to Diode without actually sending any data. When a directory is -provided a new JSON file is created for each ingest call. +Use a `DryRunClient` to inspect what would be sent to Diode without actually sending any data. When a directory is provided a new JSON file is created for each ingest call. ```go // Write ingest payload to a timestamped file in /tmp client, err := diode.NewDryRunClient("example-app", "/tmp") if err != nil { -log.Fatal(err) + log.Fatal(err) } _, _ = client.Ingest(context.Background(), []diode.Entity{ -&diode.Device{Name: diode.String("Device A")}, + &diode.Device{Name: diode.String("Device A")}, }) _ = client.Close() ``` @@ -141,23 +139,23 @@ Loaded entities can later be ingested using a real client: ```go protoEntities, err := diode.LoadDryRunEntities("/tmp/example-app_1750106879725947344.json") if err != nil { -log.Fatal(err) + log.Fatal(err) } realClient, err := diode.NewClient( -"grpc://localhost:8080/diode", -"example-app", -"0.1.0", -diode.WithClientID("YOUR_CLIENT_ID"), -diode.WithClientSecret("YOUR_CLIENT_SECRET"), + "grpc://localhost:8080/diode", + "example-app", + "0.1.0", + diode.WithClientID("YOUR_CLIENT_ID"), + diode.WithClientSecret("YOUR_CLIENT_SECRET"), ) if err != nil { -log.Fatal(err) + log.Fatal(err) } _, err = realClient.IngestProto(context.Background(), protoEntities) if err != nil { -log.Fatal(err) + log.Fatal(err) } ``` From 9cf2439a47337feb19f36d15f98157e5a61f5981 Mon Sep 17 00:00:00 2001 From: Micah Parks <66095735+MicahParks@users.noreply.github.com> Date: Mon, 11 Aug 2025 11:39:52 -0400 Subject: [PATCH 8/8] Update diode/client.go Co-authored-by: Michal Fiedorowicz --- diode/client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/diode/client.go b/diode/client.go index 6f0b53f..db1a58c 100644 --- a/diode/client.go +++ b/diode/client.go @@ -56,7 +56,7 @@ const ( var ( // ErrInvalidTargetScheme is returned when the target URL does not start with a valid scheme. - ErrInvalidTargetScheme = errors.New("target should start with grpc:// or grpcs:// or http:// or https://") + ErrInvalidTargetScheme = errors.New("target should start with grpc://, grpcs://, http:// or https://") allowedSchemesRe = regexp.MustCompile(`grpc|grpcs|http|https`) )