Commit 62a25ac6 authored by qhoangxuan's avatar qhoangxuan

wi6 scripted postgres

parent a47a16f6
......@@ -16,34 +16,72 @@
# limitations under the License.
"""Backend Deploy."""
import logging
import os
import time
import yaml
from typing import Iterable, Callable, Dict, Any, Union
import docker
from docker import Client
from utils.DockerContainerParameter import DockerContainerParameter
from utils.docker_container_parameter import DockerContainerParameter #pylint: disable=import-error, no-name-in-module
class ZoeBackendDeploy():
""" Class deploy zoe backend """
def __init__(self, dockerUrl, dockerComposePath):
self.cli = Client(base_url=dockerUrl)
self.zoe_api = ''
self.zoe_master = ''
self.zoe_opts = []
self.docker_compose_file = dockerComposePath
self.typeDeploy = 1 if 'prod' in dockerComposePath else 0
self.type_deploy = 1 if 'prod' in dockerComposePath else 0
self.parse_docker_compose()
self.previousImage=''
self.previous_img = ''
self.zoe_postgres = 'zoe-postgres-test'
self.postgres_conf = 'postgres'
self.postgre_ip = ''
def deploy_postgres(self):
""" deploy postgres and wait until it is ready """
opts = DockerContainerParameter()
opts.set_name(self.zoe_postgres)
opts.set_hostname(self.zoe_postgres)
opts.set_image(self.postgres_conf + ':latest')
host_config = self.cli.create_host_config(network_mode='bridge')
self.cli.create_container(image=opts.get_image(),
hostname=opts.get_hostname(),
name=opts.get_name(),
tty=True,
host_config=host_config,
environment={'POSTGRES_USER': self.postgres_conf,
'POSTGRES_DB': self.postgres_conf,
'POSTGRES_PASSWORD': self.postgres_conf})
self.cli.start(self.zoe_postgres)
self.postgres_ip = self.cli.inspect_container(self.zoe_postgres)['NetworkSettings']['Networks']['bridge']['IPAddress'] #pylint: disable=attribute-defined-outside-init
res = os.popen('scripts/pg_isready -h ' + self.postgres_ip).read()
while 'no response' in res:
time.sleep(1)
print('Waiting for postgres to get ready')
res = os.popen('scripts/pg_isready -h ' + self.postgres_ip).read()
for opts in self.zoe_opts:
cmd = opts.get_command()
conn = '--dbhost ' + self.postgres_ip
opts.set_command(cmd.replace('--dbhost', conn))
def parse_docker_compose(self):
""" parse docker-compose to get docker configuration """
content = ''
with open(self.docker_compose_file, 'r') as stream:
content = list(yaml.load(stream)['services'].items())
for i in range(0,2):
for i in range(0, 2):
opts = DockerContainerParameter()
......@@ -76,8 +114,8 @@ class ZoeBackendDeploy():
self.zoe_opts.append(opts)
def create_container(self, opts, image):
def create_container(self, opts, image): #pylint: disable=too-many-locals
""" create container """
port_binds = {}
volume_binds = {}
ports = []
......@@ -88,35 +126,37 @@ class ZoeBackendDeploy():
if len(opts.get_ports()) != 0:
ports_list = opts.get_ports()[0]
for p in ports_list:
splitted = p.split(":")
for prt in ports_list:
splitted = prt.split(":")
port_binds[int(splitted[0])] = int(splitted[1])
ports.append(int(splitted[0]))
if len(opts.get_volumes()) != 0:
volumes_list = opts.get_volumes()[0]
for v in volumes_list:
splitted = v.split(":")
for vol in volumes_list:
splitted = vol.split(":")
bind = {'bind' : splitted[1], 'mode': 'rw'}
volume_binds[splitted[0]] = bind
volumes.append(splitted[1])
if opts.get_gelf() != '':
log_config = docker.utils.LogConfig(type = 'gelf', config = {'gelf-address': opts.get_gelf()})
log_config = docker.utils.LogConfig(type='gelf', config={'gelf-address': opts.get_gelf()})
else:
log_config = None
host_config = self.cli.create_host_config(
network_mode='bridge',
port_bindings= port_binds,
binds= volume_binds,
log_config = log_config)
port_bindings=port_binds,
binds=volume_binds,
log_config=log_config)
ctn = self.cli.create_container(
image = opts.get_image(),
command = opts.get_command(),
hostname = opts.get_hostname(),
name = opts.get_name(),
image=opts.get_image(),
command=opts.get_command(),
hostname=opts.get_hostname(),
name=opts.get_name(),
tty=True,
ports=ports,
volumes=volumes,
......@@ -126,40 +166,45 @@ class ZoeBackendDeploy():
return ctn
def export_master_ip(self):
def export_ip(self, ctn_name, to_ctn):
""" get ip of container and export to host file of other container """
try:
ip_zoe_master = self.cli.inspect_container(self.zoe_master)['NetworkSettings']['Networks']['bridge']['IPAddress']
hostEntry = ip_zoe_master + '\t' + self.zoe_master
add_to_host = 'bash -c "echo ' + "'" + hostEntry + "'" + ' >> /etc/hosts"'
id = self.cli.exec_create(self.zoe_api, add_to_host)
self.cli.exec_start(id)
ip = self.cli.inspect_container(ctn_name)['NetworkSettings']['Networks']['bridge']['IPAddress']
host_entry = ip + '\t' + ctn_name
add_to_host = 'bash -c "echo ' + "'" + host_entry + "'" + ' >> /etc/hosts"'
identifier = self.cli.exec_create(to_ctn, add_to_host)
self.cli.exec_start(identifier)
except Exception as ex:
print(ex)
def deploy(self, image):
# '''deploy with docker-compose, if success, return, else, fallback '''
try:
""" deploy with docker-compose, if success, return, else, fallback """
try: #pylint: disable=too-many-nested-blocks
retcode = 1
for s in [self.zoe_api, self.zoe_master]:
res = self.cli.containers(all=True, filters={'name': s})
if len(res) > 0:
for r in res:
name = r['Names'][0].split("/")[1]
if self.typeDeploy == 0:
if name == s:
for ctn_name in [self.zoe_api, self.zoe_master, self.zoe_postgres]:
result = self.cli.containers(all=True, filters={'name': ctn_name})
print(result)
if len(result) > 0:
for res in result:
name = res['Names'][0].split("/")[1]
if self.type_deploy == 0:
if name == ctn_name:
self.cli.remove_container(name, force=True)
else:
if name == s:
imgID = self.cli.inspect_container(s)['Image']
self.previousImage = self.cli.inspect_image(imgID)['RepoTags'][0]
if name == ctn_name:
img_id = self.cli.inspect_container(ctn_name)['Image']
self.previous_img = self.cli.inspect_image(img_id)['RepoTags'][0]
self.cli.remove_container(name, force=True)
print('Removed ' + name + ' container')
if self.type_deploy == 0:
print('Deploying postgres...')
self.deploy_postgres()
print('deploying with ' + image)
for opts in self.zoe_opts:
ctn = self.create_container(opts, image)
self.create_container(opts, image)
print('Deploying zoe backend...')
......@@ -168,9 +213,9 @@ class ZoeBackendDeploy():
print('Started latest ' + self.zoe_api + ' container...')
if not self.cli.inspect_container(self.zoe_api)['State']['Running']:
retcode = -1
time.sleep(5)
#start zoe_master
self.cli.start(self.zoe_master)
print('Started latest ' + self.zoe_master + ' container...')
......@@ -178,15 +223,14 @@ class ZoeBackendDeploy():
retcode = 0
#export zoe-master ip address to hosts file of zoe-api
self.export_master_ip()
self.export_ip(self.zoe_master, self.zoe_api)
except Exception as ex:
print(ex)
retcode = 0
pass
return retcode
def fallback(self, image):
def fallback(self):
""" fallback to previous success images """
return
......@@ -18,25 +18,25 @@
"""Frontend deploy."""
from docker import Client
import sys
class ZoeFrontendDeploy():
""" Zoe frontend deploy class """
def __init__(self, dockerUrl, apache2):
self.src = 'prod.tar'
self.srcBackup = 'backup.tar'
self.src_backup = 'backup.tar'
self.dst = '/var/www/'
self.dstBackup = '/var/www/prod'
self.dst_backup = '/var/www/prod'
self.cli = Client(base_url=dockerUrl)
self.apache2 = apache2
return
def deploy(self):
# """ Put new frontend folder behind apache2 container """
try:
""" Put new frontend folder behind apache2 container """
try:
retcode = 1
#do backup
strm, stat = self.cli.get_archive(container=self.apache2, path=self.dstBackup)
filebackup = open(self.srcBackup, 'wb')
strm, stat = self.cli.get_archive(container=self.apache2, path=self.dst_backup)
print(stat)
filebackup = open(self.src_backup, 'wb')
filebackup.write(strm.read())
filedata = open(self.src, 'rb').read()
......@@ -49,9 +49,10 @@ class ZoeFrontendDeploy():
return retcode
def fallback(self):
""" Fallback to previous successfull build """
try:
retcode = 1
filebackup = open(self.srcBackup, 'rb').read()
filebackup = open(self.src_backup, 'rb').read()
res = self.cli.put_archive(container=self.apache2, path=self.dst, data=filebackup)
if res is False:
retcode = 0
......
......@@ -4,23 +4,13 @@ services:
image: 192.168.12.2:5000/zoe:reducetime
volumes:
- /var/run/docker.sock:/var/run/docker.sock
command: python3 zoe-api.py --debug --swarm consul://192.168.12.2 --deployment-name test --dbuser postgres --dbhost 192.168.12.2 --dbport 4321 --dbname postgres --dbpass postgres --overlay-network-name my-net --master-url tcp://zoe-master-test:4850 --auth-type text --proxy-type apache --proxy-container apache2 --proxy-config-file /etc/apache2/sites-available/all.conf --proxy-path fsdna.on.kpmg.de/zoe --listen-port 5100 --gelf-address udp://192.168.12.2:5004
command: python3 zoe-api.py --debug --swarm consul://192.168.12.2 --deployment-name test --dbuser postgres --dbhost --dbport 5432 --dbname postgres --dbpass postgres --overlay-network-name my-net --master-url tcp://zoe-master-test:4850 --auth-type text --listen-port 5100
ports:
- "5100:5100"
logging:
driver: "gelf"
options:
gelf-address: "udp://192.168.12.2:5004"
tag: "zoe-api"
zoe-master-test:
image: 192.168.12.2:5000/zoe:reducetime
volumes:
- /var/run/docker.sock:/var/run/docker.sock
command: python3 zoe-master.py --debug --swarm consul://192.168.12.2 --deployment-name test --dbuser postgres --dbhost 192.168.12.2 --dbport 4321 --dbname postgres --dbpass postgres --overlay-network-name my-net --auth-type text --proxy-type apache --proxy-container apache2 --proxy-config-file /etc/apache2/sites-available/all.conf --proxy-path fsdna.on.kpmg.de/zoe --listen-port 5100 --gelf-address udp://192.168.12.2:5004 --gelf-address udp://192.168.12.2:5004
command: python3 zoe-master.py --debug --swarm consul://192.168.12.2 --deployment-name test --dbuser postgres --dbhost --dbport 5432 --dbname postgres --dbpass postgres --overlay-network-name my-net --auth-type text
depends_on:
- zoe-api
logging:
driver: "gelf"
options:
gelf-address: "udp://192.168.12.2:5004"
tag: "zoe-master"
......@@ -17,9 +17,10 @@
"""Container Parameter class"""
from typing import Iterable, Callable, Dict, Any, Union
from typing import Iterable
class DockerContainerParameter():
""" Class holding Docker Container configuration """
def __init__(self):
self.image = ''
self.volumes = []
......@@ -32,24 +33,31 @@ class DockerContainerParameter():
self.hostname = ''
def set_gelf(self, gelf_address):
""" setter gelf_address """
self.gelf_address = gelf_address
def get_gelf(self) -> str:
return self.gelf_address
""" getter gelf_addres """
return self.gelf_address
def set_ports(self, ports):
""" setter ports """
self.ports.append(ports)
def get_ports(self) -> Iterable[str]:
""" getter ports """
return self.ports
def set_image(self, image) -> str:
""" setter image """
self.image = image
def get_image(self) -> str:
""" getter image """
return self.image
def set_volumes(self, volumes):
""" setter volume """
self.volumes.append(volumes)
def get_volumes(self) -> Iterable[str]:
......@@ -65,14 +73,17 @@ class DockerContainerParameter():
return self.command
def set_name(self, name) -> str:
""" setter for name """
self.name = name
def get_name(self) -> str:
""" getter for name """
return self.name
def set_hostname(self, hostname) -> str:
""" setter for hostname """
self.hostname = hostname
def get_hostname(self) -> str:
""" getter for hostname """
return self.hostname
......@@ -17,85 +17,84 @@
"""ZOE CI entry point."""
import yaml
import sys
from typing import Iterable, Callable, Dict, Any, Union
import docker
from docker import Client
from utils.DockerContainerParameter import DockerContainerParameter
from deploy.frontenddeploy import ZoeFrontendDeploy
from deploy.backenddeploy import ZoeBackendDeploy
from deploy.frontenddeploy import ZoeFrontendDeploy #pylint: disable=import-error
from deploy.backenddeploy import ZoeBackendDeploy #pylint: disable=import-error
class ZoeDeploy():
class ZoeDeploy():
""" Zoe deploy class """
def __init__(self, dockerUrl, dockerComposePath, image):
self.currentImage = image
self.typeDeploy = 1 if 'prod' in dockerComposePath else 0
self.current_image = image
self.type_deploy = 1 if 'prod' in dockerComposePath else 0
self.backend = ZoeBackendDeploy(dockerUrl, dockerComposePath)
self.frontend = ZoeFrontendDeploy(dockerUrl, 'apache2')
def deploy(self):
""" Deploy frontend and backend """
try:
retBE = self.backend.deploy(self.currentImage)
ret_be = self.backend.deploy(self.current_image)
print('Deployed BE with latest image...')
if self.typeDeploy == 1 and retBE == 0:
if self.type_deploy == 1 and ret_be == 0:
print('Redeploy BE with previous image')
self.backend.deploy(self.backend.previousImage)
retFE = 1
if self.typeDeploy == 1:
#retFE = self.frontend.deploy()
self.backend.deploy(self.backend.previousImage)
ret_fe = 1
if self.type_deploy == 1:
#ret_fe = self.frontend.deploy()
print('Deployed FE with latest codes...')
if retFE == 0 or retBE == 0:
retFE = self.frontend.fallback()
if ret_fe == 0 or ret_be == 0:
ret_fe = self.frontend.fallback()
except Exception as ex:
print(ex)
retBE = 0
return (retBE and retFE)
ret_be = 0
return ret_be and ret_fe
class ZoeImage():
""" Zoe build/push image class """
def __init__(self, dockerUrl, tag):
self.cli = Client(base_url=dockerUrl)
self.tag = tag
def build(self):
# """ Build docker image """
ret = 1
""" Build docker image """
build_ret = 1
for line in self.cli.build(path='.', tag=self.tag, rm=True):
print(line)
if 'error' in str(line):
ret = 0
return ret
build_ret = 0
return build_ret
def push(self):
# """ Push docker image """
ret = 1
""" Push docker image """
push_ret = 1
for line in self.cli.push(self.tag, stream=True):
print(line)
if 'error' in str(line):
ret = 0
return ret
push_ret = 0
return push_ret
if __name__ == '__main__':
if len(sys.argv) < 4:
sys.exit(1)
else:
if sys.argv[1] == '0':
deployer = ZoeDeploy(sys.argv[2], sys.argv[3], sys.argv[4])
ret = deployer.deploy()
deployer = ZoeDeploy(sys.argv[2], sys.argv[3], sys.argv[4]) # pylint: disable=invalid-name
ret = deployer.deploy() # pylint: disable=invalid-name
if ret == 0:
sys.exit(1)
elif sys.argv[1] == '1':
imghandler = ZoeImage(sys.argv[2], sys.argv[3])
ret = imghandler.build()
imghandler = ZoeImage(sys.argv[2], sys.argv[3]) # pylint: disable=invalid-name
ret = imghandler.build() # pylint: disable=invalid-name
if ret == 0:
sys.exit(1)
elif sys.argv[1] == '2':
imghandler = ZoeImage(sys.argv[2], sys.argv[3])
ret = imghandler.push()
imghandler = ZoeImage(sys.argv[2], sys.argv[3]) # pylint: disable=invalid-name
ret = imghandler.push() # pylint: disable=invalid-name
if ret == 0:
sys.exit(1)
......@@ -2,5 +2,5 @@
set -e
pylint --ignore old_swarm *.py zoe_*
pylint --ignore old_swarm *.py zoe_* ci
doc8 docs/
#!/usr/bin/perl -w
# Call a PostgreSQL client program with the version, cluster and default
# database specified in ~/.postgresqlrc or
# /etc/postgresql-common/user_clusters.
#
# (C) 2005-2009 Martin Pitt <mpitt@debian.org>
# (C) 2013-2014 Christoph Berg <myon@debian.org>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
use strict;
use POSIX;
use PgCommon;
my ($version, $cluster, $db, $port, $host);
$host = $ENV{'PGHOST'};
# Check for PGCLUSTER in %ENV
if (defined $ENV{'PGCLUSTER'}) {
($version, $cluster) = split ('/', $ENV{'PGCLUSTER'}, 2);
error 'Invalid version specified with $PGCLUSTER' unless version_exists $version;
error 'No cluster specified with $PGCLUSTER' unless $cluster;
}
# Check for --cluster argument and filter it out, and check if --port is specified
my $port_specified = exists $ENV{'PGPORT'};
for (my $i = 0; $i <= $#ARGV; ++$i) {
last if $ARGV[$i] eq '--';
if ($ARGV[$i] eq '--cluster') {
error '--cluster option needs an argument (<version>/<cluster>)' if ($i >= $#ARGV);
($version, $cluster) = split ('/', $ARGV[$i+1], 2);
$host = undef; # --cluster overrides $PGHOST env var
error 'Invalid version specified with --cluster' unless version_exists $version;
error 'No cluster specified with --cluster' unless $cluster;
splice @ARGV, $i, 2;
last;
} elsif ($ARGV[$i] =~ /^--cluster=(\d+\.\d)\/(.+)/) {
($version, $cluster) = ($1, $2);
$host = undef; # --cluster overrides $PGHOST env var
error 'Invalid version specified with --cluster' unless version_exists $version;
error 'No cluster specified with --cluster' unless $cluster;
splice @ARGV, $i, 1;
last;
}
$port_specified = 1 if $ARGV[$i] =~ /^--port\b/ || $ARGV[$i] =~ /^-\w*p\w*\d*$/;
$host = '.from.commandline' if $ARGV[$i] =~ /^--host\b/ || $ARGV[$i] =~ /^-\w*h\w*$/;
}
# Determine $version, $cluster, $db, $port from map files
($version, $cluster, $db) = user_cluster_map() unless $cluster;
# check if we have a network cluster
if (!$host && $cluster && !cluster_exists $version, $cluster) {
if ($cluster =~ /^(\S+):(\d*)$/) {
$host = $1;
$port = $2 || 5432;
} else {
error 'Specified cluster does not exist locally and does not specify a remote cluster';
}
}
if (!$host && $cluster) {
$port = get_cluster_port($version, $cluster);
unless ($ENV{'PGHOST'}) {
# default to cluster specific Unix socket directory
$ENV{'PGHOST'} = get_cluster_socketdir $version, $cluster;
}
}
$ENV{'PGSYSCONFDIR'} = '/etc/postgresql-common' if !$ENV{'PGSYSCONFDIR'};
$ENV{'PGPORT'} = $port if $port && !$ENV{'PGPORT'};
$ENV{'PGDATABASE'} = $db if $db && !$ENV{'PGDATABASE'};
$ENV{'PGHOST'} = $host if $host && $host ne '.from.commandline';
# if we only have a port, but no version here, use the latest version
# TODO: this could be improved by better argument parsing and mapping back the
# port to a cluster version/name
if (!$version and $port_specified) {
$version = get_newest_version;
}
unless ($version) {
if (get_versions) {
error 'No existing local cluster is suitable as a default target. Please see man pg_wrapper(1) how to specify one.';
} else {
error 'You must install at least one postgresql-client-<version> package.';
}
}
error 'Invalid PostgreSQL cluster version' unless -d "$PgCommon::binroot$version";
my $cmdname = (split '/', $0)[-1];
my $cmd;
# for psql we always want the latest version, as this is backwards compatible
# to every major version that that we support
if ($cmdname eq 'pg_wrapper') {
error "pg_wrapper should not be called directly, but through a symlink";
} elsif ($cmdname =~ /^(psql|pg_archivecleanup|pg_isready)$/) {
$cmd = get_program_path ($cmdname, get_newest_version);
} else {
$cmd = get_program_path ($cmdname, $version);
}
# libreadline is a lot better than libedit, so prefer that
if ($cmdname eq 'psql' and not $PgCommon::rpm) {
my @readlines;
# non-multiarch path
@readlines = sort(</lib/libreadline.so.?>);
unless (@readlines) {
# get multiarch dir for our architecture
if (open PS, '-|', '/usr/bin/ldd', $cmd) {
my $out;
read PS, $out, 10000;
close PS;
if ($out =~ m!/libreadline.so!) {
# already linked against libreadline
@readlines = ();
}
else
{
my ($lib_path) = $out =~ m!(/lib/.*)/libedit.so!;
@readlines = sort(<$lib_path/libreadline.so.?>);
}
}
}
if (@readlines) {
$ENV{'LD_PRELOAD'} = ($ENV{'LD_PRELOAD'} or '') . ':' . $readlines[-1];
}
}
error "pg_wrapper: $cmdname was not found in $PgCommon::binroot$version/bin" unless $cmd;
unshift @ARGV, $cmd;
exec @ARGV;
__END__
=head1 NAME
pg_wrapper - wrapper for PostgreSQL client commands
=head1 SYNOPSIS
I<client-program> [B<--cluster> I<version>/I<cluster>] [...]
(I<client-program>: B<psql>, B<createdb>, B<dropuser>, and all other client
programs installed in C</usr/lib/postgresql/>I<version>C</bin>).
=head1 DESCRIPTION
This program is run only as a link to names which correspond to PostgreSQL
programs in C</usr/lib/postgresql/>I<version>C</bin>. It determines the
configured cluster and database for the user and calls the appropriate version
of the desired program to connect to that cluster and database, supplying any
specified options to that command.
The target cluster is selected by the following means, in descending order of
precedence:
=over
=item 1.
explicit specification with the B<--host> option
=item 2.
explicit specification with the B<--cluster> option
=item 3.
if the B<PGHOST> environment variable is set, no further cluster selection is
performed. The default PostgreSQL version and port number (from the command
line, the environment variable B<PGPORT>, or default 5432) will be used.
=item 4.
explicit specification with the B<PGCLUSTER> environment variable
=item 5.
matching entry in C<~/.postgresqlrc> (see L<postgresqlrc(5)>), if that
file exists
=item 6.
matching entry in C</etc/postgresql-common/user_clusters> (see
L<user_clusters(5)>), if that file exists
=item 7.
If only one local cluster exists, that one will be selected.
=item 8.
If several local clusters exist, the one listening on the default port 5432
will be selected.
=back
If none of these rules match, B<pg_wrapper> aborts with an error.