Golang patterns. Worker pool

func worker(id int, jobs <-chan int, results chan<- int) {
    for j := range jobs {
        fmt.Println("worker", id, "started  job", j)
        time.Sleep(time.Second)
        fmt.Println("worker", id, "finished job", j)
        results <- j * 2
    }
}

func main() {

    // In order to use our pool of workers we need to send
    // them work and collect their results. We make 2
    // channels for this.
    jobs := make(chan int, 100)
    results := make(chan int, 100)

    // This starts up 3 workers, initially blocked
    // because there are no jobs yet.
    for w := 1; w <= 3; w++ {
        go worker(w, jobs, results)
    }

    // Here we send 5 `jobs` and then `close` that
    // channel to indicate that's all the work we have.
    for j := 1; j <= 5; j++ {
        jobs <- j
    }
    close(jobs)

    for a := 1; a <= 5; a++ {
        <-results
    }
}

func worker(id int, jobs <-chan int, results chan<- int) {

for j := range jobs {

fmt.Println("worker", id, "started job", j)

time.Sleep(time.Second)

fmt.Println("worker", id, "finished job", j)

results <- j * 2

}

func main() {

// In order to use our pool of workers we need to send

// them work and collect their results. We make 2

// channels for this.

jobs := make(chan int, 100)

results := make(chan int, 100)

// This starts up 3 workers, initially blocked

// because there are no jobs yet.

for w := 1; w <= 3; w++ {

go worker(w, jobs, results)

}

// Here we send 5 `jobs` and then `close` that

// channel to indicate that's all the work we have.

for j := 1; j <= 5; j++ {

jobs <- j

}

close(jobs)

for a := 1; a <= 5; a++ {

<-results

}

Further reading: https://gobyexample.com/worker-pools

PgBouncer and prepared statements

In our system, we use connection pooler called PgBouncer as a proxy to PostgreSQL server. PgBouncer has two main modes Session pooling mode This mode is less performant (it’s a default mode). When a client connects, a server connection will be assigned to it for the whole duration it stays connected. So it does not …

Golang: testing http and grpc servers

HTTP server is quite easy to test — here is a nice video about it:

Count lines in git repository

git ls-files | xargs wc -l

1	git ls-files \| xargs wc -l

https://stackoverflow.com/questions/4822471/count-number-of-lines-in-a-git-repository

Algorithms to find fastest route through network

This is a great article on that — https://www.redblobgames.com/pathfinding/a-star/introduction.html (in russain). It tells about Dijkstra, Breadth-first, Greedy and A* algorithms. It’s all about optimization of one travel. To optimize your entire network you should try to use Aint Colony algorithm — http://rain.ifmo.ru/cat/data/theory/unsorted/ant-algo-2006/article.pdf (it’s in russian). Another link to this PDF file — aca

How go code is being compiled to assembler code

https://go.godbolt.org/z/31FyJ3 I was interested in comparing line 15 vs line 17 of the following code:

package main

import "fmt"

type kv struct {
	key   []byte
	value []byte
}

type sliceMap []kv

func (sm *sliceMap) Add(k, v []byte) {
	kvs := *sm
	if cap(kvs) > len(kvs) {
		kvs = kvs[:len(kvs)+1]
	} else {
		kvs = append(kvs, kv{})
	}
	kv := &kvs[len(kvs)-1]
	kv.key = append(kv.key[:0], k...)
	kv.value = append(kv.value[:0], v...)
	*sm = kvs
}

func main() {
	sm := sliceMap{}
	sm.Add([]byte("foo"), []byte("bar"))
	fmt.Println(sm)
}

package main

import "fmt"

type kv struct {

key []byte

value []byte

}

type sliceMap []kv

func (sm *sliceMap) Add(k, v []byte) {

kvs := *sm

if cap(kvs) > len(kvs) {

kvs = kvs[:len(kvs)+1]

} else {

kvs = append(kvs, kv{})

}

kv := &kvs[len(kvs)-1]

kv.key = append(kv.key[:0], k...)

kv.value = append(kv.value[:0], v...)

*sm = kvs

}

func main() {

sm := sliceMap{}

sm.Add([]byte("foo"), []byte("bar"))

fmt.Println(sm)

}

E2E tests in go

I tried it via ginkgo and gomega. It has Agouti with WebDriver support out of the box, but I didn’t use it as we have grpc API. That’s the example (table tests):

var _ = Describe("GetRoute", func() {
	var (
		srClient sr.Client
		srErr    error
	)

	JustBeforeEach(func() {
		srClient, srErr = GetClient()
	})

	DescribeTable("positive cases",
		func(fromGeoPointID, toGeoPointID int, expectedUID string, expectedCostGreaterThan float64) {
			Expect(srErr).NotTo(HaveOccurred())
			Expect(srClient).NotTo(BeNil())

			response, err := srClient.GetRoute(context.TODO(), &sr.GetRouteRequest{
				FromGeoPointId: int64(fromGeoPointID),
				ToGeoPointId:   int64(toGeoPointID),
				Quantity:       1,
				Volume:         1,
				Weight:         1,
			})

			Expect(err).To(Succeed())
			Expect(response).NotTo(BeNil())
			Expect(response.RouteCost).NotTo(BeNil())
			Expect(response.RouteCost.RouteUid).To(HaveSuffix(expectedUID))
			Expect(response.RouteCost.GetCost()).To(BeNumerically(">", expectedCostGreaterThan))
		},
		Entry("from(geo=3)->to(geo=2)", 3, 2, "-3-2", float64(10)),
	)

var _ = Describe("GetRoute", func() {

var (

srClient sr.Client

srErr error

)

JustBeforeEach(func() {

srClient, srErr = GetClient()

})

DescribeTable("positive cases",

func(fromGeoPointID, toGeoPointID int, expectedUID string, expectedCostGreaterThan float64) {

Expect(srErr).NotTo(HaveOccurred())

Expect(srClient).NotTo(BeNil())

response, err := srClient.GetRoute(context.TODO(), &sr.GetRouteRequest{

FromGeoPointId: int64(fromGeoPointID),

ToGeoPointId: int64(toGeoPointID),

Quantity: 1,

Volume: 1,

Weight: 1,

})

Expect(err).To(Succeed())

Expect(response).NotTo(BeNil())

Expect(response.RouteCost).NotTo(BeNil())

Expect(response.RouteCost.RouteUid).To(HaveSuffix(expectedUID))

Expect(response.RouteCost.GetCost()).To(BeNumerically(">", expectedCostGreaterThan))

Entry("from(geo=3)->to(geo=2)", 3, 2, "-3-2", float64(10)),

)

What is teamlead’s first duty

Sarah Mei «We think awful code is written by awful devs. But in reality, it’s written by reasonable devs in awful circumstances.» (source) So, teamlead is there to improve circumstances.

ORM for go

https://github.com/Masterminds/squirrel Never used it as we always write SQL queries manually. But looks okay for ORM (well, actually a query-builder).

Kubernetes’ cronjobs

That’s what it is — https://kubernetes.io/docs/concepts/workloads/controllers/cron-jobs/ Get all cronjobs in env:

kubectl --kubeconfig ... --namespace platform -o wide get cronjobs

1	kubectl --kubeconfig ... --namespace platform -o wide get cronjobs

Get info on exact cronjob:

kubectl --kubeconfig ... --namespace platform describe cronjob/smartrouting-cron-import-from-bi

1	kubectl --kubeconfig ... --namespace platform describe cronjob/smartrouting-cron-import-from-bi

Get pods with this cronjob:

kubectl --kubeconfig ... --namespace platform  -o wide get pods|grep "rou"

1	kubectl --kubeconfig ... --namespace platform -o wide get pods\|grep "rou"

Get logs of cronjob

kubectl --kubeconfig ... --namespace platform logs smartrouting-cron-import-from-bi-1555060200-dqg5j

1	kubectl --kubeconfig ... --namespace platform logs smartrouting-cron-import-from-bi-1555060200-dqg5j

you can add -f for live updating logs and -p for watching logs of previous pod.