PostgreSQL/PgBouncer: Difference between revisions

From Chorke Wiki
Jump to navigation Jump to search
No edit summary
No edit summary
 
(One intermediate revision by the same user not shown)
Line 1: Line 1:
{|class='wikitable' style='width:100%;margin:-11px 0 6px 0'
{|class='wikitable'
|valign='top' style='width:50%'|
|valign='top' style='width:50%'|
<syntaxhighlight style='margin:3px 0' lang='bash'>
<syntaxhighlight lang='bash'>
cat <<-'EXE'|sudo bash
cat <<-'EXE'|sudo bash
apt-get update;echo
apt-get update;echo
Line 10: Line 10:


|valign='top' style='width:50%'|
|valign='top' style='width:50%'|
<syntaxhighlight style='margin:3px 0' lang='bash'>
<syntaxhighlight lang='bash'>
     systemctl status  pgbouncer.service
     systemctl status  pgbouncer.service


Line 19: Line 19:
|-
|-
|valign='top'|
|valign='top'|
<syntaxhighlight style='margin:3px 0' lang='bash'>
<syntaxhighlight lang='bash'>
  ls -lah /etc/pgbouncer/
  ls -lah /etc/pgbouncer/


Line 28: Line 28:


|valign='top'|
|valign='top'|
|-
|valign='top' colspan='2'|
{|class='wikitable mw-collapsible mw-collapsed'
!scope='col' style='text-align:left'|
Diagram
|-
|valign='top'|
<kroki lang='plantuml'>
@startuml
!theme plain
' Styling for Transparency and Layout
skinparam backgroundColor transparent
skinparam DefaultFontName Helvetica
skinparam componentStyle  rectangle
skinparam packageStyle    rectangle
skinparam actorStyle      awesome
skinparam monochrome      true
skinparam shadowing      false
actor "Application\n(Clients)" as App
participant "PgBouncer\n(Connection Pooler)" as PGB
database "PostgreSQL\n(Backend)" as DB
== Connection Phase ==
App -> PGB : Connect (TCP/Unix Socket)
PGB -> PGB : Check Pool for available\nbackend connection
alt No idle connection in pool
    PGB -> DB : Establish new backend connection
    DB -> PGB : Connection Ready
end
== Execution Phase ==
App -> PGB : SQL Query (SELECT/INSERT...)
PGB -> DB : Forward Query on pooled connection
DB -> PGB : Result Set
PGB -> App : Return Results
== Disconnection Phase ==
App -> PGB : Close Connection
note right of PGB
  PgBouncer keeps the DB connection
  OPEN in its pool for the next client.
end note
PGB -> PGB : Return backend connection to pool
@enduml
</kroki>
|}
|}
|}


==Configure==
==Configure==
{|class='wikitable mw-collapsible' style='width:100%;margin:3px 0'
{|class='wikitable mw-collapsible'
!scope='col' style='text-align:left' colspan='2'|
!scope='col' style='text-align:left' colspan='2'|
Configure
Configure
|-
|-
|valign='top' style='width:50%'|
|valign='top' style='width:50%'|
<syntaxhighlight style='margin:3px 0' lang='bash'>
<syntaxhighlight lang='bash'>
sudo rsync -avz /etc/pgbouncer/pgbouncer.ini /etc/pgbouncer/pgbouncer.ini.bkp
sudo rsync -avz /etc/pgbouncer/pgbouncer.ini /etc/pgbouncer/pgbouncer.ini.bkp
sudo rsync -avz /etc/pgbouncer/userlist.txt  /etc/pgbouncer/userlist.txt.bkp
sudo rsync -avz /etc/pgbouncer/userlist.txt  /etc/pgbouncer/userlist.txt.bkp
Line 49: Line 100:


===Configure » Pool » Database===
===Configure » Pool » Database===
{|class='wikitable mw-collapsible' style='width:100%;margin:3px 0'
{|class='wikitable mw-collapsible'
!scope='col' style='text-align:left'|
!scope='col' style='text-align:left'|
Configure » Pool » Database
Configure » Pool » Database
|-
|-
|valign='top'|
|valign='top'|
<syntaxhighlight style='margin:3px 0' lang='bash'>
<syntaxhighlight lang='bash'>
# find pgbouncer db pool
# find pgbouncer db pool
PG_DB_POOL_FIND=$(cat <<QRY
PG_DB_POOL_FIND=$(cat <<QRY
Line 81: Line 132:


===Configure » Pool » User===
===Configure » Pool » User===
{|class='wikitable mw-collapsible' style='width:100%;margin:3px 0'
{|class='wikitable mw-collapsible'
!scope='col' style='text-align:left'|
!scope='col' style='text-align:left'|
Configure » Pool » Database
Configure » Pool » Database
|-
|-
|valign='top'|
|valign='top'|
<syntaxhighlight style='margin:3px 0' lang='bash'>
<syntaxhighlight lang='bash'>
# find pgbouncer user pool
# find pgbouncer user pool
PG_USER_POOL_FIND=$(cat <<QRY
PG_USER_POOL_FIND=$(cat <<QRY
Line 110: Line 161:


===Configure » User » Secret===
===Configure » User » Secret===
{|class='wikitable mw-collapsible' style='width:100%;margin:3px 0'
{|class='wikitable mw-collapsible'
!scope='col' style='text-align:left' colspan='2'|
!scope='col' style='text-align:left' colspan='2'|
Configure
Configure
|-
|-
|valign='top' style='width:50%'|
|valign='top' style='width:50%'|
<syntaxhighlight style='margin:3px 0' lang='bash'>
<syntaxhighlight lang='bash'>
cat << USR | sudo tee -a /etc/pgbouncer/userlist.txt >/dev/null
cat << USR | sudo tee -a /etc/pgbouncer/userlist.txt >/dev/null
"chorke" "md5$(printf '%s%s' 'p@$$w0rd' 'chorke'|md5sum|awk '{print $1}')"
"chorke" "md5$(printf '%s%s' 'p@$$w0rd' 'chorke'|md5sum|awk '{print $1}')"
Line 123: Line 174:


|valign='top' style='width:50%'|
|valign='top' style='width:50%'|
<syntaxhighlight style='margin:3px 0' lang='bash'>
<syntaxhighlight lang='bash'>
     systemctl status  pgbouncer.service
     systemctl status  pgbouncer.service


Line 131: Line 182:
|-
|-
|valign='top'|
|valign='top'|
<syntaxhighlight style='margin:3px 0' lang='bash'>
<syntaxhighlight lang='bash'>
psql -hlocalhost -p6432 -Uchorke -dacademia_users_staging
psql -hlocalhost -p6432 -Uchorke -dacademia_users_staging
psql -hlocalhost -p6432 -Uchorke -dacademia_audit_staging
psql -hlocalhost -p6432 -Uchorke -dacademia_audit_staging
Line 141: Line 192:


==PgBouncer.ini==
==PgBouncer.ini==
{|class='wikitable mw-collapsible' style='width:100%;margin:3px 0'
{|class='wikitable mw-collapsible'
!scope='col' style='text-align:left' colspan='2'|
!scope='col' style='text-align:left' colspan='2'|
PgBouncer.ini
PgBouncer.ini
|-
|-
|valign='top' colspan='2'|
|valign='top' colspan='2'|
<syntaxhighlight style='margin:3px 0' lang="ini">
<syntaxhighlight lang="ini">
[databases]
[databases]
academia_flair_staging = host=127.0.0.1 port=5432 user=chorke password='sadaqah!' client_encoding=UNICODE datestyle=ISO connect_query='SELECT 1'
academia_flair_staging = host=127.0.0.1 port=5432 user=chorke password='sadaqah!' client_encoding=UNICODE datestyle=ISO connect_query='SELECT 1'
Line 172: Line 223:
|-
|-
|valign='top' style='width:50%'|
|valign='top' style='width:50%'|
<syntaxhighlight style='margin:3px 0' lang='bash'>
<syntaxhighlight lang='bash'>
sudo su
sudo su
su postgres
su postgres
Line 181: Line 232:


==Monitoring==
==Monitoring==
{|class='wikitable mw-collapsible' style='width:100%;margin:3px 0'
{|class='wikitable mw-collapsible'
!scope='col' style='text-align:left' colspan='2'|
!scope='col' style='text-align:left' colspan='2'|
Monitoring
Monitoring
|-
|-
|valign='top' style='width:50%'|
|valign='top' style='width:50%'|
<syntaxhighlight style='margin:3px 0' lang="sql">
<syntaxhighlight lang='sql'>
SHOW max_connections;
SHOW max_connections;
RESET max_connections;
RESET max_connections;
Line 198: Line 249:
|-
|-
|valign='top'|
|valign='top'|
<syntaxhighlight style='margin:3px 0' lang="sql">
<syntaxhighlight lang='sql'>
-- pg_lsclusters
-- pg_lsclusters
SHOW data_directory;
SHOW data_directory;
Line 207: Line 258:
|-
|-
|valign='top'|
|valign='top'|
<syntaxhighlight style='margin:3px 0' lang="sql">
<syntaxhighlight lang='sql'>
WITH conn_limit AS (
WITH conn_limit AS (
     SELECT setting::int "max_conn_size"  
     SELECT setting::int "max_conn_size"  
Line 223: Line 274:


==Playground==
==Playground==
{|class='wikitable mw-collapsible' style='width:100%;margin:3px 0'
{|class='wikitable mw-collapsible'
!scope='col' style='text-align:left' colspan='2'|
!scope='col' style='text-align:left' colspan='2'|
Playground
Playground
|-
|-
|valign='top' style='width:50%'|
|valign='top' style='width:50%'|
<syntaxhighlight style='margin:3px 0' lang='bash'>
<syntaxhighlight lang='bash'>
ls -lah /etc/pgbouncer/
ls -lah /etc/pgbouncer/
ls -lah /var/log/postgresql/
ls -lah /var/log/postgresql/
Line 235: Line 286:


|valign='top' style='width:50%'|
|valign='top' style='width:50%'|
<syntaxhighlight style='margin:3px 0' lang="ini">
<syntaxhighlight lang="ini">
[pgbouncer]
[pgbouncer]
; ERROR: unsupported startup parameter
; ERROR: unsupported startup parameter
Line 243: Line 294:


==References==
==References==
{|class='wikitable mw-collapsible' style='width:100%;margin:3px 0'
{|class='wikitable mw-collapsible'
!scope='col' style='text-align:left' colspan='3'|
!scope='col' style='text-align:left' colspan='3'|
References
References
Line 286: Line 337:


|valign='top'|
|valign='top'|
|}
|}

Latest revision as of 14:40, 25 January 2026

cat <<-'EXE'|sudo bash
apt-get update;echo
apt list -a --upgradable
apt-get install -y pgbouncer
EXE
     systemctl status  pgbouncer.service

sudo systemctl stop    pgbouncer.service
sudo systemctl start   pgbouncer.service
sudo systemctl restart pgbouncer.service
 ls -lah /etc/pgbouncer/

sudo cat /etc/pgbouncer/userlist.txt
sudo cat /etc/pgbouncer/pgbouncer.ini
sudo cat /etc/pgbouncer/pgbouncer.ini|less

Diagram

Configure

Configure

sudo rsync -avz /etc/pgbouncer/pgbouncer.ini /etc/pgbouncer/pgbouncer.ini.bkp
sudo rsync -avz /etc/pgbouncer/userlist.txt  /etc/pgbouncer/userlist.txt.bkp

# restore default configuration
sudo rsync -avz /etc/pgbouncer/pgbouncer.ini.bkp /etc/pgbouncer/pgbouncer.ini
       sudo cat /etc/pgbouncer/pgbouncer.ini

Configure » Pool » Database

Configure » Pool » Database

# find pgbouncer db pool
PG_DB_POOL_FIND=$(cat <<QRY
\[databases\]\n\
\n\
;; foodb over Unix socket\n\
;foodb =
QRY
)

# fill pgbouncer db pool
PG_DB_POOL_FILL=$(cat <<UPD
[databases]\n\
\n\
;; future db pool here\n\
academia_flair_staging = host=10.110.110.155 port=5432 user=chorke password='sadaqah!' client_encoding=UNICODE datestyle=ISO connect_query='SELECT 1'\n\
academia_audit_staging = host=10.110.110.155 port=5432 user=chorke password='sadaqah!' client_encoding=UNICODE datestyle=ISO connect_query='SELECT 1'\n\
academia_qoute_staging = host=10.110.110.155 port=5432 user=chorke password='sadaqah!' client_encoding=UNICODE datestyle=ISO connect_query='SELECT 1'\n\
academia_users_staging = host=10.110.110.155 port=5432 user=chorke password='sadaqah!' client_encoding=UNICODE datestyle=ISO connect_query='SELECT 1'
UPD
)

sudo sed -z "s|${PG_DB_POOL_FIND}|$(echo "${PG_DB_POOL_FILL}")|" -i /etc/pgbouncer/pgbouncer.ini

Configure » Pool » User

Configure » Pool » Database

# find pgbouncer user pool
PG_USER_POOL_FIND=$(cat <<QRY
\[users\]\n\
\n\
;user1 = pool_mode=transaction max_user_connections=10
QRY
)

# fill pgbouncer user pool
PG_USER_POOL_FILL=$(cat <<UPD
[users]\n\
\n\
;; future user pool here\n\
chorke = pool_mode=transaction max_user_connections=10
UPD
)

sudo sed -z "s|${PG_USER_POOL_FIND}|$(echo "${PG_USER_POOL_FILL}")|" -i /etc/pgbouncer/pgbouncer.ini
sudo sed -z "s|auth_type = trust|auth_type = md5|" -i /etc/pgbouncer/pgbouncer.ini

Configure » User » Secret

Configure

cat << USR | sudo tee -a /etc/pgbouncer/userlist.txt >/dev/null
"chorke" "md5$(printf '%s%s' 'p@$$w0rd' 'chorke'|md5sum|awk '{print $1}')"
"shahed" "md5$(printf '%s%s' 'sadaqah!' 'shahed'|md5sum|awk '{print $1}')"
USR
     systemctl status  pgbouncer.service

sudo systemctl reload  pgbouncer.service
sudo systemctl restart pgbouncer.service
psql -hlocalhost -p6432 -Uchorke -dacademia_users_staging
psql -hlocalhost -p6432 -Uchorke -dacademia_audit_staging
psql -hlocalhost -p6432 -Uchorke -dacademia_flair_staging

PgBouncer.ini

PgBouncer.ini

[databases]
academia_flair_staging = host=127.0.0.1 port=5432 user=chorke password='sadaqah!' client_encoding=UNICODE datestyle=ISO connect_query='SELECT 1'
academia_audit_staging = host=127.0.0.1 port=5432 user=chorke password='sadaqah!' client_encoding=UNICODE datestyle=ISO connect_query='SELECT 1'
academia_qoute_staging = host=127.0.0.1 port=5432 user=chorke password='sadaqah!' client_encoding=UNICODE datestyle=ISO connect_query='SELECT 1'
academia_users_staging = host=127.0.0.1 port=5432 user=chorke password='sadaqah!' client_encoding=UNICODE datestyle=ISO connect_query='SELECT 1'

[users]
chorke = pool_mode=transaction max_user_connections=20

[pgbouncer]
auth_type = md5
auth_file = /etc/pgbouncer/userlist.txt

pool_mode = session
ignore_startup_parameters = extra_float_digits

min_pool_size = 5
reserve_pool_size = 5
reserve_pool_timeout = 5

max_db_connections = 10
max_user_connections = 10
sudo su
su postgres
pgbouncer -R /etc/pgbouncer/pgbouncer.ini

Monitoring

Monitoring

SHOW max_connections;
RESET max_connections;
ALTER SYSTEM SET max_connections = 200;
SELECT current_setting('max_connections');
SELECT sum(numbackends) FROM pg_stat_database;
SELECT * FROM  pg_settings WHERE name = 'max_connections';
SELECT state, count(*) FROM pg_stat_activity WHERE pid <> pg_backend_pid() GROUP BY 1 ORDER BY 1;
-- pg_lsclusters
SHOW data_directory;
SELECT setting FROM pg_settings WHERE name = 'data_directory';
SELECT name, setting FROM pg_settings WHERE setting LIKE '/%';
WITH conn_limit AS (
    SELECT setting::int "max_conn_size" 
    FROM pg_settings WHERE name=$$max_connections$$
),
conn_super AS (
    SELECT setting::int "max_conn_admin"
    FROM pg_settings WHERE name=$$superuser_reserved_connections$$
)
SELECT count(*) "max_conn_used", (SELECT * FROM conn_super), (SELECT * FROM conn_limit)
FROM pg_stat_activity;

Playground

Playground

ls -lah /etc/pgbouncer/
ls -lah /var/log/postgresql/
ls -lah /var/run/postgresql/
[pgbouncer]
; ERROR: unsupported startup parameter
ignore_startup_parameters = extra_float_digits

References

References