Category Archives: golang

Force ipv4 for golang http client

Sometimes you want to prevent your http client from using ipv6/tcp6. That’s how you do it.

dialer := &net.Dialer{
	Timeout:   dialTimeout,
	KeepAlive: keepAliveTimeout,
}

transport := &http.Transport{
	Proxy:             http.ProxyFromEnvironment,
	ForceAttemptHTTP2: false,
	DialContext: func(ctx context.Context, network string, addr string) (net.Conn, error) {
		ipv4, err := resolveIPv4(addr)
		if err != nil {
			return nil, err
		}

		return dialer.DialContext(ctx, network, ipv4)
	},
}

httpClient := &http.Client{
	Transport: transport,
	Timeout:   args.HTTPTimeout,
}

// resolveIPv4 resolves an address to IPv4 address.
func resolveIPv4(addr string) (string, error) {
	url := strings.Split(addr, ":")

	m := new(dns.Msg)
	m.SetQuestion(dns.Fqdn(url[0]), dns.TypeA)
	m.RecursionDesired = true

	config, _ := dns.ClientConfigFromFile("/etc/resolv.conf")
	c := new(dns.Client)
	r, _, err := c.Exchange(m, net.JoinHostPort(config.Servers[0], config.Port))
	if err != nil {
		return "", err
	}
	for _, ans := range r.Answer {
		if a, ok := ans.(*dns.A); ok {
			url[0] = a.A.String()
		}
	}

	return strings.Join(url, ":"), nil
}

dialer := &net.Dialer{

Timeout: dialTimeout,

KeepAlive: keepAliveTimeout,

}

transport := &http.Transport{

Proxy: http.ProxyFromEnvironment,

ForceAttemptHTTP2: false,

DialContext: func(ctx context.Context, network string, addr string) (net.Conn, error) {

ipv4, err := resolveIPv4(addr)

if err != nil {

return nil, err

}

return dialer.DialContext(ctx, network, ipv4)

}

httpClient := &http.Client{

Transport: transport,

Timeout: args.HTTPTimeout,

}

// resolveIPv4 resolves an address to IPv4 address.

func resolveIPv4(addr string) (string, error) {

url := strings.Split(addr, ":")

m := new(dns.Msg)

m.SetQuestion(dns.Fqdn(url[0]), dns.TypeA)

m.RecursionDesired = true

config, _ := dns.ClientConfigFromFile("/etc/resolv.conf")

c := new(dns.Client)

r, _, err := c.Exchange(m, net.JoinHostPort(config.Servers[0], config.Port))

if err != nil {

return "", err

}

for _, ans := range r.Answer {

if a, ok := ans.(*dns.A); ok {

url[0] = a.A.String()

}

return strings.Join(url, ":"), nil

}

Mocking for unit-tests and e2e-tests in golang

Some patterns to test your golang app. Mocking an interface

	mockS3 := &mocks.S3API{}

	mockResultFn := func(input *s3.ListObjectsInput) *s3.ListObjectsOutput {
		output := &s3.ListObjectsOutput{}
		output.SetCommonPrefixes([]*s3.CommonPrefix{
			&s3.CommonPrefix{
				Prefix: aws.String("2017-01-01"),
			},
		})
		return output
	}

	// NB: .Return(...) must return the same signature as the method being mocked.
	//     In this case it's (*s3.ListObjectsOutput, error).
	mockS3.On("ListObjects", mock.MatchedBy(func(input *s3.ListObjectsInput) bool {
		return input.Delimiter != nil && *input.Delimiter == "/" && input.Prefix == nil
	})).Return(mockResultFn, nil)

	listingInput := &s3.ListObjectsInput{
		Bucket:    aws.String("foo"),
		Delimiter: aws.String("/"),
	}
	listingOutput, err := mockS3.ListObjects(listingInput)
	if err != nil {
		panic(err)
	}

	for _, x := range listingOutput.CommonPrefixes {
		fmt.Printf("common prefix: %+v\n", *x)
	}

mockS3 := &mocks.S3API{}

mockResultFn := func(input *s3.ListObjectsInput) *s3.ListObjectsOutput {

output := &s3.ListObjectsOutput{}

output.SetCommonPrefixes([]*s3.CommonPrefix{

&s3.CommonPrefix{

Prefix: aws.String("2017-01-01"),

})

return output

}

// NB: .Return(...) must return the same signature as the method being mocked.

// In this case it's (*s3.ListObjectsOutput, error).

mockS3.On("ListObjects", mock.MatchedBy(func(input *s3.ListObjectsInput) bool {

return input.Delimiter != nil && *input.Delimiter == "/" && input.Prefix == nil

})).Return(mockResultFn, nil)

listingInput := &s3.ListObjectsInput{

Bucket: aws.String("foo"),

Delimiter: aws.String("/"),

}

listingOutput, err := mockS3.ListObjects(listingInput)

if err != nil {

panic(err)

}

for _, x := range listingOutput.CommonPrefixes {

fmt.Printf("common prefix: %+v\n", *x)

}

https://github.com/vektra/mockery Mocking SQL database

// a successful case
func TestShouldUpdateStats(t *testing.T) {
	db, mock, err := sqlmock.New()
	if err != nil {
		t.Fatalf("an error '%s' was not expected when opening a stub database connection", err)
	}
	defer db.Close()

	mock.ExpectBegin()
	mock.ExpectExec("UPDATE products").WillReturnResult(sqlmock.NewResult(1, 1))
	mock.ExpectExec("INSERT INTO product_viewers").WithArgs(2, 3).WillReturnResult(sqlmock.NewResult(1, 1))
	mock.ExpectCommit()

	// now we execute our method
	if err = recordStats(db, 2, 3); err != nil {
		t.Errorf("error was not expected while updating stats: %s", err)
	}

	// we make sure that all expectations were met
	if err := mock.ExpectationsWereMet(); err != nil {
		t.Errorf("there were unfulfilled expectations: %s", err)
	}
}

// a successful case

func TestShouldUpdateStats(t *testing.T) {

db, mock, err := sqlmock.New()

if err != nil {

t.Fatalf("an error '%s' was not expected when opening a stub database connection", err)

}

defer db.Close()

mock.ExpectBegin()

mock.ExpectExec("UPDATE products").WillReturnResult(sqlmock.NewResult(1, 1))

mock.ExpectExec("INSERT INTO product_viewers").WithArgs(2, 3).WillReturnResult(sqlmock.NewResult(1, 1))

mock.ExpectCommit()

// now we execute our method

if err = recordStats(db, 2, 3); err != nil {

t.Errorf("error was not expected while updating stats: %s", err)

}

// we make sure that all expectations were met

if err := mock.ExpectationsWereMet(); err != nil {

t.Errorf("there were unfulfilled expectations: %s", err)

}

https://github.com/DATA-DOG/go-sqlmock Mocking HTTP server for unit-tests

	handler := func(w http.ResponseWriter, r *http.Request) {
		io.WriteString(w, "<html><body>Hello World!</body></html>")
	}

	req := httptest.NewRequest("GET", "http://example.com/foo", nil)
	w := httptest.NewRecorder()
	handler(w, req)

	resp := w.Result()
	body, _ := ioutil.ReadAll(resp.Body)

	fmt.Println(resp.StatusCode)
	fmt.Println(resp.Header.Get("Content-Type"))
	fmt.Println(string(body))

handler := func(w http.ResponseWriter, r *http.Request) {

io.WriteString(w, "<html><body>Hello World!</body></html>")

}

req := httptest.NewRequest("GET", "http://example.com/foo", nil)

w := httptest.NewRecorder()

handler(w, req)

resp := w.Result()

body, _ := ioutil.ReadAll(resp.Body)

fmt.Println(resp.StatusCode)

fmt.Println(resp.Header.Get("Content-Type"))

fmt.Println(string(body))

https://golang.org/pkg/net/http/httptest/ Stubbing HTTP server for e2e-tests

{
	"host": "example.com",
	"method": "GET|POST|PUT|PATCH|... (Mandatory)", 
	"path": "/your/path/:variable (Mandatory)",
	"queryStringParameters": {
		"name": ["value"],
		"name": ["value", "value"]
	},
	"headers": {
		"name": ["value"]
	},
	"cookies": {
		"name": "value"
	},
	"body": "Expected Body"
}

{

"host": "example.com",

"method": "GET|POST|PUT|PATCH|... (Mandatory)",

"path": "/your/path/:variable (Mandatory)",

"queryStringParameters": {

"name": ["value"],

"name": ["value", "value"]

"headers": {

"name": ["value"]

"cookies": {

"name": "value"

"body": "Expected Body"

}

https://github.com/jmartin82/mmock Mocking for e2e-tests with dockertest

var db *sql.DB

func TestMain(m *testing.M) {
	// uses a sensible default on windows (tcp/http) and linux/osx (socket)
	pool, err := dockertest.NewPool("")
	if err != nil {
		log.Fatalf("Could not connect to docker: %s", err)
	}

	// pulls an image, creates a container based on it and runs it
	resource, err := pool.Run("mysql", "5.7", []string{"MYSQL_ROOT_PASSWORD=secret"})
	if err != nil {
		log.Fatalf("Could not start resource: %s", err)
	}

	// exponential backoff-retry, because the application in the container might not be ready to accept connections yet
	if err := pool.Retry(func() error {
		var err error
		db, err = sql.Open("mysql", fmt.Sprintf("root:secret@(localhost:%s)/mysql", resource.GetPort("3306/tcp")))
		if err != nil {
			return err
		}
		return db.Ping()
	}); err != nil {
		log.Fatalf("Could not connect to docker: %s", err)
	}

	code := m.Run()
	
	// You can't defer this because os.Exit doesn't care for defer
	if err := pool.Purge(resource); err != nil {
		log.Fatalf("Could not purge resource: %s", err)
	}
	
	os.Exit(code)
}

func TestSomething(t *testing.T) {
	// db.Query()
}

var db *sql.DB

func TestMain(m *testing.M) {

// uses a sensible default on windows (tcp/http) and linux/osx (socket)

pool, err := dockertest.NewPool("")

if err != nil {

log.Fatalf("Could not connect to docker: %s", err)

}

// pulls an image, creates a container based on it and runs it

resource, err := pool.Run("mysql", "5.7", []string{"MYSQL_ROOT_PASSWORD=secret"})

if err != nil {

log.Fatalf("Could not start resource: %s", err)

}

// exponential backoff-retry, because the application in the container might not be ready to accept connections yet

if err := pool.Retry(func() error {

var err error

db, err = sql.Open("mysql", fmt.Sprintf("root:secret@(localhost:%s)/mysql", resource.GetPort("3306/tcp")))

if err != nil {

return err

}

return db.Ping()

}); err != nil {

log.Fatalf("Could not connect to docker: %s", err)

}

code := m.Run()

// You can't defer this because os.Exit doesn't care for defer

if err := pool.Purge(resource); err != nil {

log.Fatalf("Could not purge resource: %s", err)

}

os.Exit(code)

}

func TestSomething(t *testing.T) {

// db.Query()

}

https://github.com/ory/dockertest

Install go2 beta on MacOS

A quick guide on installing golang v2 development revision. It works for my MacOS Big Sur (with an Intel processor).

Using DataDog on your CentOs

DD_AGENT_MAJOR_VERSION=7 DD_API_KEY=<your-key> DD_SITE="datadoghq.eu" bash -c "$(curl -L https://s3.amazonaws.com/dd-agent/scripts/install_script.sh)"

sudo nano /etc/datadog-agent/datadog.yaml
# uncomment logs_enabled: true
sudo systemctl restart datadog-agent
sudo mkdir /etc/datadog-agent/conf.d/go.d
sudo nano /etc/datadog-agent/conf.d/go.d/conf.yaml
# and add contents from the url below

DD_AGENT_MAJOR_VERSION=7 DD_API_KEY=<your-key> DD_SITE="datadoghq.eu" bash -c "$(curl -L https://s3.amazonaws.com/dd-agent/scripts/install_script.sh)"

sudo nano /etc/datadog-agent/datadog.yaml

# uncomment logs_enabled: true

sudo systemctl restart datadog-agent

sudo mkdir /etc/datadog-agent/conf.d/go.d

sudo nano /etc/datadog-agent/conf.d/go.d/conf.yaml

# and add contents from the url below

https://app.datadoghq.eu/logs/onboarding/server https://docs.datadoghq.com/agent/faq/error-restarting-agent-already-listening-on-a-configured-port/ Or, if you want a dockerized version of DataDog agent, and you want to listen to docker logs, you can do this:

docker run -d --name datadog-agent \
           -e DD_API_KEY=<DATADOG_API_KEY> \
           -e DD_LOGS_ENABLED=true \
           -e DD_LOGS_CONFIG_CONTAINER_COLLECT_ALL=true \
           -e DD_CONTAINER_EXCLUDE_LOGS="name:datadog-agent" \
           -e DD_SITE="datadoghq.eu"  \
           -v /var/run/docker.sock:/var/run/docker.sock:ro \
           -v /proc/:/host/proc/:ro \
           -v /opt/datadog-agent/run:/opt/datadog-agent/run:rw \
           -v /sys/fs/cgroup/:/host/sys/fs/cgroup:ro \
           datadog/agent:latest

docker run -d --name datadog-agent \

-e DD_API_KEY=<DATADOG_API_KEY> \

-e DD_LOGS_ENABLED=true \

-e DD_LOGS_CONFIG_CONTAINER_COLLECT_ALL=true \

-e DD_CONTAINER_EXCLUDE_LOGS="name:datadog-agent" \

-e DD_SITE="datadoghq.eu" \

-v /var/run/docker.sock:/var/run/docker.sock:ro \

-v /proc/:/host/proc/:ro \

-v /opt/datadog-agent/run:/opt/datadog-agent/run:rw \

-v /sys/fs/cgroup/:/host/sys/fs/cgroup:ro \

datadog/agent:latest

More info here — https://docs.datadoghq.com/agent/docker/log/?tab=containerinstallation Then you will see your logs here — https://app.datadoghq.eu/logs

Install go with gvm on MacOS Big Sur

I used to use gvm for this — https://github.com/moovweb/gvm. But initial installation of go from scratch does not work for Big Sur as 1.4 does not have a binary for this OS (this command fails — gvm install go1.4 -B). To use gvm, you can do this: 0. Install gvm — bash <

Complete list of swagger options to protobuf file

Here is an example with many options that help generate proper swagger out of protofile. Original URL — https://raw.githubusercontent.com/grpc-ecosystem/grpc-gateway/master/examples/internal/proto/examplepb/a_bit_of_everything.proto.

Refactor your code and know it’s completely backward compatible

The idea is to run 2 versions of code on production: 1 (control group) — is your old implementation which will be returned to the end user 2 (the experiment) — is a new implementation which results will be checked against the control group and logged It will be run asynchronously and only for a …

GRPC fallback to Rest API with custom field names

When you generate JSON for Rest API from proto-file, protoc-gen-gofast generates field names for JSON in lowerCamelCase format while most of the Rest APIs use snake_case for that. And if you want to replace some legacy API with your new implementation without breaking backward compatibility, you need to fix it. There could be different ways …

Integration tests with testcontainers-go

Here is the go library that simplifies integration tests with docker containers — https://github.com/testcontainers/testcontainers-go. That’s how you can use it to test sql — https://github.com/testcontainers/testcontainers-go/blob/master/docs/examples/cockroachdb.md. Project documentation — https://golang.testcontainers.org/features/docker_compose/ The idea is to prepare the environment, build your app, then make requests and check the DB state.

«Awesome go» resources

A list of good resources for each go specific part — https://github.com/avelino/awesome-go. A list of good tools for go performance or just fast libs — https://github.com/cristaloleg/awesome-go-perf Security lists — https://github.com/guardrailsio/awesome-golang-security, https://github.com/Binject/awesome-go-security