Category Archives: Programming

Installing protobuf tools on MacOS

Installing protoc

brew install protobuf

1	brew install protobuf

Or follow different instructions. Installing grpc_cli Option 1. Easy way.

brew tap grpc/grpc
brew install --with-plugins grpc

1 2	brew tap grpc/grpc brew install --with-plugins grpc

It is described here — https://github.com/grpc/homebrew-grpc. Option 2. Hard way — using cmake and make. NOT RECOMMENDED.

brew install autoconf automake libtool shtool cmake

git clone -b $(curl -L https://grpc.io/release) https://github.com/grpc/grpc
cd grpc
git submodule update --init

mkdir -p cmake/build
cd cmake/build
cmake ../..
make

gRPC_INSTALL=ON make install
# and continue accordgin to https://github.com/grpc/grpc/blob/master/BUILDING.md

brew install autoconf automake libtool shtool cmake

git clone -b $(curl -L https://grpc.io/release) https://github.com/grpc/grpc

cd grpc

git submodule update --init

mkdir -p cmake/build

cd cmake/build

cmake ../..

make

gRPC_INSTALL=ON make install

# and continue accordgin to https://github.com/grpc/grpc/blob/master/BUILDING.md

Or follow these instructions

Nice talk about concurrency in Go

Another good talk. It’s not only about channels, but also about atomics and patterns around ti. https://www.youtube.com/watch?v=YEKjSzIwAdA

Make your bot for telegram using go

It’s really not that difficult. Here are the docs: https://core.telegram.org/bots https://core.telegram.org/bots/api Here is a simple library in go for telegram — https://github.com/go-telegram-bot-api/telegram-bot-api/ (too simple, from my point of view, does not cover all functionality, but okay). And here is a skeletton for making your bots if you want it as just standalone binary — https://github.com/nezorflame/example-telegram-bot/

Gitlab-ci: build go app with docker as a docker image

Preparations [TO BE UPDATED LATER] My gitlab-ci.yml

variables:
  PACKAGE_PATH: /go/src/gitlab.com/[username_for_gitlab]/[project_name]

  # https://docs.gitlab.com/ee/ci/docker/using_docker_build.html
  CONTAINER_TEST_IMAGE: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG
  CONTAINER_RELEASE_IMAGE: $CI_REGISTRY_IMAGE:latest

stages:
  - dep
  - test
  - build

# A hack to make Golang-in-Gitlab happy
.anchors:
  - &inject-gopath
      mkdir -p $(dirname ${PACKAGE_PATH})
      && ln -s ${CI_PROJECT_DIR} ${PACKAGE_PATH}
      && cd ${PACKAGE_PATH}

dep:
  stage: dep
  image: golang:1.13.1-alpine
  before_script:
    - *inject-gopath
  script: go mod tidy && go mod vendor
  artifacts:
    name: "vendor-$CI_PIPELINE_ID"
    paths:
      - vendor/
    expire_in: 1 hour

test:
  stage: test
  dependencies:
    - dep
  image: docker:17
  services:
    - docker:dind
  before_script:
    - *inject-gopath
  script: docker build --target=test .

lint:
  stage: test
  dependencies:
    - dep
  image: docker:17
  services:
    - docker:dind
  before_script:
    - *inject-gopath
  script: docker build --target=lint .

build:
  stage: build
  dependencies:
    - dep
    - lint
    - test
  image: docker:17
  services:
    - docker:dind
  before_script:
    - *inject-gopath
    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
registry
  script:
    - docker build --target=release --pull -t $CONTAINER_TEST_IMAGE .
    - docker tag $CONTAINER_TEST_IMAGE $CONTAINER_RELEASE_IMAGE
    - docker push $CONTAINER_TEST_IMAGE

variables:

PACKAGE_PATH: /go/src/gitlab.com/[username_for_gitlab]/[project_name]

# https://docs.gitlab.com/ee/ci/docker/using_docker_build.html

CONTAINER_TEST_IMAGE: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG

CONTAINER_RELEASE_IMAGE: $CI_REGISTRY_IMAGE:latest

stages:

- dep

- test

- build

# A hack to make Golang-in-Gitlab happy

.anchors:

- &inject-gopath

mkdir -p $(dirname ${PACKAGE_PATH})

&& ln -s ${CI_PROJECT_DIR} ${PACKAGE_PATH}

&& cd ${PACKAGE_PATH}

dep:

stage: dep

image: golang:1.13.1-alpine

before_script:

- *inject-gopath

script: go mod tidy && go mod vendor

artifacts:

name: "vendor-$CI_PIPELINE_ID"

paths:

- vendor/

expire_in: 1 hour

test:

stage: test

dependencies:

- dep

image: docker:17

services:

- docker:dind

before_script:

- *inject-gopath

script: docker build --target=test .

lint:

stage: test

dependencies:

- dep

image: docker:17

services:

- docker:dind

before_script:

- *inject-gopath

script: docker build --target=lint .

build:

stage: build

dependencies:

- dep

- lint

- test

image: docker:17

services:

- docker:dind

before_script:

- *inject-gopath

- docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY

registry

script:

- docker build --target=release --pull -t $CONTAINER_TEST_IMAGE .

- docker tag $CONTAINER_TEST_IMAGE $CONTAINER_RELEASE_IMAGE

- docker push $CONTAINER_TEST_IMAGE

My Dockerfile

FROM golang:1.13.1-alpine AS build
WORKDIR /go/src/gitlab.com/[username_for_gitlab]/[project_name]
COPY . .
RUN go get -u github.com/jessevdk/go-assets-builder
RUN apk add --no-cache make
RUN make build #some pareparations and after that: go build -i -o ./[path_to_bin] ./cmd/[...]

FROM alpine:3.10 AS release
COPY --from=build /go/src/gitlab.com/[username_for_gitlab]/[project_name]/[path_to_bin] /usr/local/bin/[appname]
RUN apk add --no-cache make bash curl tzdata
EXPOSE 3000 3001
ENTRYPOINT ["/usr/local/bin/[appname]"]

FROM golang:1.13.1-alpine AS test
ENV CGO_ENABLED=0
WORKDIR /go/src/gitlab.com/[username_for_gitlab]/[project_name]
COPY . .
RUN apk add --no-cache make bash
RUN make test

FROM golangci/golangci-lint:v1.20 AS lint
WORKDIR /go/src/gitlab.com/[username_for_gitlab]/[project_name]
COPY . .
RUN make lint-full

# to check things locally
# docker build --target=check_container .
# docker images
# docker run -it <hash> /bin/bash
FROM golang:1.13.1-alpine AS check_container
WORKDIR /go/src/gitlab.com/[username_for_gitlab]/[project_name]
COPY . .
RUN apk add --no-cache make bash
RUN /bin/bash

FROM release

FROM golang:1.13.1-alpine AS build

WORKDIR /go/src/gitlab.com/[username_for_gitlab]/[project_name]

COPY . .

RUN go get -u github.com/jessevdk/go-assets-builder

RUN apk add --no-cache make

RUN make build #some pareparations and after that: go build -i -o ./[path_to_bin] ./cmd/[...]

FROM alpine:3.10 AS release

COPY --from=build /go/src/gitlab.com/[username_for_gitlab]/[project_name]/[path_to_bin] /usr/local/bin/[appname]

RUN apk add --no-cache make bash curl tzdata

EXPOSE 3000 3001

ENTRYPOINT ["/usr/local/bin/[appname]"]

FROM golang:1.13.1-alpine AS test

ENV CGO_ENABLED=0

WORKDIR /go/src/gitlab.com/[username_for_gitlab]/[project_name]

COPY . .

RUN apk add --no-cache make bash

RUN make test

FROM golangci/golangci-lint:v1.20 AS lint

WORKDIR /go/src/gitlab.com/[username_for_gitlab]/[project_name]

COPY . .

RUN make lint-full

# to check things locally

# docker build --target=check_container .

# docker images

# docker run -it <hash> /bin/bash

FROM golang:1.13.1-alpine AS check_container

WORKDIR /go/src/gitlab.com/[username_for_gitlab]/[project_name]

COPY . .

RUN apk add --no-cache make bash

RUN /bin/bash

FROM release

Pulling from your private registry Create deploy token as described here. After that, you can do this:

docker login -u [deploy_username] -p [deploy_token] registry.gitlab.com
docker pull registry.gitlab.com/[username_for_gitlab]/[project_name]:master
docker run -d --name [container_name_you_like] --env-file [path-to-.env] --net host --rm registry.gitlab.com/[username_for_gitlab]/[project_name]:master

docker login -u [deploy_username] -p [deploy_token] registry.gitlab.com

docker pull registry.gitlab.com/[username_for_gitlab]/[project_name]:master

docker run -d --name [container_name_you_like] --env-file [path-to-.env] --net host --rm registry.gitlab.com/[username_for_gitlab]/[project_name]:master

Useful links: https://dev.to/hypnoglow/how-to-make-friends-with-golang-docker-and-gitlab-ci-4bil https://docs.gitlab.com/ee/ci/docker/using_docker_build.html https://medium.com/@tonistiigi/advanced-multi-stage-build-patterns-6f741b852fae https://docs.gitlab.com/ee/ci/yaml/ (full description of gitlab-ci.yml) https://about.gitlab.com/blog/2017/11/27/go-tools-and-gitlab-how-to-do-continuous-integration-like-a-boss/ https://gitlab.com/hypnoglow/example-go-docker-gitlab/blob/master/.gitlab-ci.yml Somewhat useful: https://blog.lwolf.org/post/how-to-build-tiny-golang-docker-images-with-gitlab-ci/ https://angristan.xyz/build-push-docker-images-gitlab-ci/ https://docs.gitlab.com/ee/user/packages/container_registry/#use-images-from-gitlab-container-registry https://www.digitalocean.com/community/tutorials/how-to-install-and-use-docker-on-centos-7 (it says to do sudo usermod -aG docker …

Free weather APIs

https://openweathermap.org/api — old, odd, but works https://www.weatherbit.io/api https://developer.accuweather.com/accuweather-forecast-api/apis to be discovered https://darksky.net/dev/docs to be discovered http://api.weather2020.com/ to be discovered, looks abandoned https://www.aerisweather.com/support/docs/api/ works, but keys are for valid for 3 months only https://www.wunderground.com/api died (https://apicommunity.wunderground.com/weatherapi/topics/end-of-service-for-the-weather-underground-api)

Another cool article on using PostgreSQL+pgBouncer with go

Extremely interesting and very practical talk about problems occurring with go+pgBouncer (in Russian, sorry). https://habr.com/ru/company/oleg-bunin/blog/461935/ And a video on that: https://www.youtube.com/watch?v=Uojy57I-xP0

env-file parser for Goland

You can use env-file parser for running and debugging your app in Jetbrains’ products like Goland. It’s pretty simple and works. https://github.com/Ashald/EnvFile

How golang garbage collection works

It is a Mark-and-Sweep GC. Phases Mark Stop-the-World: Set write barrier (to know how much was allocated during maark phase) Concurrent: Mark all memory which is still in use by the app Stop-the-World: Remove write barrier Concurrent: Sweep (it actually happens on new allocations) Details/Algorithm It’s a tri-color process: grey for objects to check, black …

Style guide for golang project file structure

Highly recommended. It has a couple of odds like not explaining about /test dir (I assume, they wanted to say to put there only e2e and other integrational tests, not unit-test as the latter should be kept together with the code itself in files like …_test.go). But it’s awesome in general. https://github.com/golang-standards/project-layout

Golang patterns. Worker pool

func worker(id int, jobs <-chan int, results chan<- int) {
    for j := range jobs {
        fmt.Println("worker", id, "started  job", j)
        time.Sleep(time.Second)
        fmt.Println("worker", id, "finished job", j)
        results <- j * 2
    }
}

func main() {

    // In order to use our pool of workers we need to send
    // them work and collect their results. We make 2
    // channels for this.
    jobs := make(chan int, 100)
    results := make(chan int, 100)

    // This starts up 3 workers, initially blocked
    // because there are no jobs yet.
    for w := 1; w <= 3; w++ {
        go worker(w, jobs, results)
    }

    // Here we send 5 `jobs` and then `close` that
    // channel to indicate that's all the work we have.
    for j := 1; j <= 5; j++ {
        jobs <- j
    }
    close(jobs)

    for a := 1; a <= 5; a++ {
        <-results
    }
}

func worker(id int, jobs <-chan int, results chan<- int) {

for j := range jobs {

fmt.Println("worker", id, "started job", j)

time.Sleep(time.Second)

fmt.Println("worker", id, "finished job", j)

results <- j * 2

}

func main() {

// In order to use our pool of workers we need to send

// them work and collect their results. We make 2

// channels for this.

jobs := make(chan int, 100)

results := make(chan int, 100)

// This starts up 3 workers, initially blocked

// because there are no jobs yet.

for w := 1; w <= 3; w++ {

go worker(w, jobs, results)

}

// Here we send 5 `jobs` and then `close` that

// channel to indicate that's all the work we have.

for j := 1; j <= 5; j++ {

jobs <- j

}

close(jobs)

for a := 1; a <= 5; a++ {

<-results

}

Further reading: https://gobyexample.com/worker-pools