Category Archives: Databases

PgBouncer and prepared statements

In our system, we use connection pooler called PgBouncer as a proxy to PostgreSQL. PgBouncer has two main modes Session pooling mode This mode is less performant (it’s a default mode). When a client connects, a server connection will be assigned to it for the whole duration it stays connected. So it does not utilize …

ORM for go

https://github.com/Masterminds/squirrel Never used it as we always write SQL queries manually. But looks okay for ORM (well, actually a query-builder).

PostgreSQL: TOP slow queries

SELECT rolname,
    calls,
    total_time,
    mean_time,
    max_time,
    stddev_time,
    rows,
    query,
    regexp_replace(query, '[ \t\n]+', ' ', 'g') AS query_oneline
FROM pg_stat_statements
JOIN pg_roles r ON r.oid = userid
WHERE calls > 100
AND rolname NOT LIKE '%backup'
ORDER BY mean_time DESC
LIMIT 15;

SELECT rolname,

calls,

total_time,

mean_time,

max_time,

stddev_time,

rows,

query,

regexp_replace(query, '[ \t\n]+', ' ', 'g') AS query_oneline

FROM pg_stat_statements

JOIN pg_roles r ON r.oid = userid

WHERE calls > 100

AND rolname NOT LIKE '%backup'

ORDER BY mean_time DESC

LIMIT 15;

Golang, PostgreSQL and array_agg

If you have SQL like this:

Postgresql: style guide for DataGrip

Place it into xml file and import into DataGrip or another jetbrains’ product

PostgreSQL: using indices on json fields

It’s obviously a really bad idea. But if you really need it, that’s what you can do.

Postgresql: show all locked queries

WITH RECURSIVE l AS (
 SELECT pid, locktype, mode, granted,
ROW(locktype,database,relation,page,tuple,virtualxid,transactionid,classid,objid,objsubid) obj
 FROM pg_locks
), pairs AS (
 SELECT w.pid waiter, l.pid locker, l.obj, l.mode
 FROM l w
 JOIN l ON l.obj IS NOT DISTINCT FROM w.obj AND l.locktype=w.locktype AND NOT l.pid=w.pid AND l.granted
 WHERE NOT w.granted
), tree AS (
 SELECT l.locker pid, l.locker root, NULL::record obj, NULL AS mode, 0 lvl, locker::text path, array_agg(l.locker) OVER () all_pids
 FROM ( SELECT DISTINCT locker FROM pairs l WHERE NOT EXISTS (SELECT 1 FROM pairs WHERE waiter=l.locker) ) l
 UNION ALL
 SELECT w.waiter pid, tree.root, w.obj, w.mode, tree.lvl+1, tree.path||'.'||w.waiter, all_pids || array_agg(w.waiter) OVER ()
 FROM tree JOIN pairs w ON tree.pid=w.locker AND NOT w.waiter = ANY ( all_pids )
)
SELECT (clock_timestamp() - a.xact_start)::interval(3) AS ts_age,
      replace(a.state, 'idle in transaction', 'idletx') state,
      (clock_timestamp() - state_change)::interval(3) AS change_age,
      a.datname,tree.pid,a.usename,a.client_addr,lvl,
      (SELECT count(*) FROM tree p WHERE p.path ~ ('^'||tree.path) AND NOT p.path=tree.path) blocked,
      repeat(' .', lvl)||' '||left(regexp_replace(query, '\s+', ' ', 'g'),100) query
FROM tree
JOIN pg_stat_activity a USING (pid)
ORDER BY path;

WITH RECURSIVE l AS (

SELECT pid, locktype, mode, granted,

ROW(locktype,database,relation,page,tuple,virtualxid,transactionid,classid,objid,objsubid) obj

FROM pg_locks

), pairs AS (

SELECT w.pid waiter, l.pid locker, l.obj, l.mode

FROM l w

JOIN l ON l.obj IS NOT DISTINCT FROM w.obj AND l.locktype=w.locktype AND NOT l.pid=w.pid AND l.granted

WHERE NOT w.granted

), tree AS (

SELECT l.locker pid, l.locker root, NULL::record obj, NULL AS mode, 0 lvl, locker::text path, array_agg(l.locker) OVER () all_pids

FROM ( SELECT DISTINCT locker FROM pairs l WHERE NOT EXISTS (SELECT 1 FROM pairs WHERE waiter=l.locker) ) l

UNION ALL

SELECT w.waiter pid, tree.root, w.obj, w.mode, tree.lvl+1, tree.path||'.'||w.waiter, all_pids || array_agg(w.waiter) OVER ()

FROM tree JOIN pairs w ON tree.pid=w.locker AND NOT w.waiter = ANY ( all_pids )

)

SELECT (clock_timestamp() - a.xact_start)::interval(3) AS ts_age,

replace(a.state, 'idle in transaction', 'idletx') state,

(clock_timestamp() - state_change)::interval(3) AS change_age,

a.datname,tree.pid,a.usename,a.client_addr,lvl,

(SELECT count(*) FROM tree p WHERE p.path ~ ('^'||tree.path) AND NOT p.path=tree.path) blocked,

repeat(' .', lvl)||' '||left(regexp_replace(query, '\s+', ' ', 'g'),100) query

FROM tree

JOIN pg_stat_activity a USING (pid)

ORDER BY path;

docker-compose example for ceph, kafka, postgres cluster

That is how it can be done:

Книги по PostgreSQL

Хорошая бесплатная книга на русском языке про оптимизацию — Работа с PostgreSQLнастройка и масштабирование, А. Ю. Васильев. Про разработку и синтаксис — PostgreSQL Development Essentials. Manpreet Kaur, Baji Shaik: на google books можно ознакомиться и скачать на scribd

SQL: выборка разреженных данных (не больше раза в час)

Допустим, мы сохраняем данные в базу раз в 10 минут. Нужно достать данные из базы, начинающиеся с определённого времени и не чаще одного раза за час. В качестве базы используется sqlite3, но идея реализуется так же для других SQL.